Add test leg to the PR build to run libraries tests on iOS x64 Simulators (#37476)

Forked from #37057

Adds the iOS test leg that will execute selected BCL tests on an iOS x64 Simulator (13.5) using the `OSX.1015.Amd64.Open` queue which is our on-prem Mac Mini infrastructure (60 machines).

To prepare the Helix job, we are using the Arcade Helix SDK, more specifically the [XHarness Helix SDK](https://github.com/dotnet/arcade/blob/master/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md) which uses the [XHarness tool](https://github.com/dotnet/xharness) to control the test run.

Author: premun

eng/pipelines/common/xplat-setup.yml

@@ -112,16 +112,8 @@ jobs:
           name: NetCoreInternal-Pool
           queue: BuildPool.Ubuntu.1604.Amd64
 
-        # Public OSX Build Pool
-        ${{ if eq(parameters.osGroup, 'OSX') }}:
-          vmImage: 'macOS-10.14'
-
-        # Public OSX Build Pool
-        ${{ if eq(parameters.osGroup, 'iOS') }}:
-          vmImage: 'macOS-10.14'
-
-        # Public OSX Build Pool
-        ${{ if eq(parameters.osGroup, 'tvOS') }}:
+        # OSX Build Pool (we don't have on-prem OSX BuildPool
+        ${{ if in(parameters.osGroup, 'OSX', 'iOS', 'tvOS') }}:
           vmImage: 'macOS-10.14'
 
         ${{ if eq(parameters.osGroup, 'Browser') }}:

eng/pipelines/libraries/helix-queues-setup.yml

@@ -91,6 +91,10 @@ jobs:
         - OSX.1014.Amd64.Open
         - OSX.1015.Amd64.Open
 
+    # iOS x64/x86
+    - ${{ if in(parameters.platform, 'iOS_x64', 'iOS_x86') }}:
+      - OSX.1015.Amd64.Open
+
     # Windows_NT x64
     - ${{ if eq(parameters.platform, 'Windows_NT_x64') }}:
       # netcoreapp

eng/pipelines/runtime.yml

@@ -290,7 +290,6 @@ jobs:
     - Android_arm
     - tvOS_arm64
     - iOS_arm
-    - iOS_x64
     - Linux_musl_x64
     jobParameters:
       testGroup: innerloop
@@ -313,6 +312,7 @@ jobs:
     buildConfig: Release
     runtimeFlavor: mono
     platforms:
+    - iOS_x64
     - Browser_wasm
     variables:
       # map dependencies variables to local variables

eng/testing/AppleRunnerTemplate.sh

@@ -1,11 +1,9 @@
 #!/usr/bin/env bash
 
 EXECUTION_DIR=$(dirname $0)
-TEST_NAME=$1
-TARGET_ARCH=$2
-TARGET=
-SCHEME_SDK=
+[[RunCommands]]
 
+# "Release" in SCHEME_SDK is what xcode produces (see "bool Optimized" property in AppleAppBuilderTask)
 if [ "$TARGET_ARCH" == "arm" ]; then
     TARGET=ios-device
     SCHEME_SDK=Release-iphoneos
@@ -23,8 +21,6 @@ else
     exit 1
 fi
 
-# "Release" in SCHEME_SDK is what xcode produces (see "bool Optimized" property in AppleAppBuilderTask)
-
 APP_BUNDLE=$EXECUTION_DIR/$TEST_NAME/$SCHEME_SDK/$TEST_NAME.app
 
 # it doesn't support parallel execution yet, so, here is a hand-made semaphore:
@@ -39,14 +35,17 @@ while true; do
     fi
 done
 
-XHARNESS_OUT="$EXECUTION_DIR/xharness-output"
+XCODE_PATH="`xcode-select -p`/../.."
+export XHARNESS_OUT="$EXECUTION_DIR/xharness-output"
 
-dotnet xharness ios test --app="$APP_BUNDLE" \
-    --targets=$TARGET \
+dotnet xharness ios test  \
+    --targets="$TARGET"   \
+    --app="$APP_BUNDLE"   \
+    --xcode="$XCODE_PATH" \
     --output-directory=$XHARNESS_OUT
 
 _exitCode=$?
 
-echo "Xharness artifacts: $XHARNESS_OUT"
+echo "XHarness artifacts: $XHARNESS_OUT"
 
 exit $_exitCode

eng/testing/tests.mobile.targets

@@ -17,6 +17,11 @@
     <RunScriptCommand>$HARNESS_RUNNER wasm test --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js -v --output-directory=$XHARNESS_OUT -- --run WasmTestRunner.dll $(AssemblyName).dll</RunScriptCommand>
   </PropertyGroup>
 
+  <ItemGroup>
+    <RunScriptCommands Include="TEST_NAME=$(TestProjectName)" />
+    <RunScriptCommands Include="TARGET_ARCH=$(TargetArchitecture)" />
+  </ItemGroup>
+
   <!-- Generate a self-contained app bundle for Android with tests. -->
   <UsingTask Condition="'$(TargetOS)' == 'Android'"
              TaskName="AndroidAppBuilderTask"
@@ -72,6 +77,7 @@
     <Error Condition="!Exists('$(MicrosoftNetCoreAppRuntimePackRidDir)')" Text="MicrosoftNetCoreAppRuntimePackRidDir=$(MicrosoftNetCoreAppRuntimePackRidDir) doesn't exist" />
     <Error Condition="('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm') and '$(DevTeamProvisioning)' == ''"
            Text="'DevTeamProvisioning' needs to be set for device builds. Set it to 'UBF8T346G9' if you're part of the Microsoft team account." />
+    <Error Condition="'$(TestArchiveTestsDir)' == ''" Text="TestArchiveTestsDir property to archive the test folder must be set." />
 
     <WriteLinesToFile File="$(PublishDir)xunit-excludes.txt" Lines="$(_withoutCategories.Replace(';', '%0dcategory='))" />
 
@@ -114,7 +120,16 @@
     </AppleAppBuilderTask>
     <Message Importance="High" Text="Xcode: $(XcodeProjectPath)"/>
     <Message Importance="High" Text="App: $(AppBundlePath)"/>
-    <!-- This app is now can be consumed by xharness CLI to deploy on a device or simulator -->
+
+    <ItemGroup>
+      <_appFiles Include="$(AppBundlePath)/../**/*" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(_appFiles)"
+          DestinationFolder="$(TestArchiveTestsDir)/%(RecursiveDir)"
+          SkipUnchangedFiles="true"
+          Condition="'$(ArchiveTests)' == 'true' and '$(IgnoreForCI)' != 'true'" />
+
   </Target>
 
   <UsingTask TaskName="WasmAppBuilder"

src/libraries/sendtohelixhelp.proj

@@ -12,10 +12,10 @@
     <HelixBuild Condition="'$(HelixBuild)' == ''">$(BUILD_BUILDNUMBER)</HelixBuild>
     <HelixBuild Condition="'$(HelixBuild)' == ''">default</HelixBuild>
 
-    <!-- Normal jobs have a 30 minute (1800 sec.) timeout for arm/arm64, and 15 minute (900 sec.) timeout otherwise.
+    <!-- Normal jobs have a 30 minute timeout for arm/arm64, and 15 minute timeout otherwise.
          Stress modes can take considerably longer, so double those numbers. And GCStress is even slower.
     -->
-    <TimeoutInSeconds Condition="
+    <_workItemTimeout Condition="
         '$(Scenario)' == 'gcstress0x3' or
         '$(Scenario)' == 'gcstress0xc' or
         '$(Scenario)' == 'heapverify1' or
@@ -24,13 +24,12 @@
         '$(Scenario)' == 'gcstress0xc_zapdisable_heapverify1' or
         '$(Scenario)' == 'gcstress0xc_jitstress1' or
         '$(Scenario)' == 'gcstress0xc_jitstress2' or
-        '$(Scenario)' == 'gcstress0xc_jitminopts_heapverify1'">5400</TimeoutInSeconds>
-    <TimeoutInSeconds Condition="'$(Scenario)' == '' and '$(TimeoutInSeconds)' == '' and ('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm')">1800</TimeoutInSeconds>
-    <TimeoutInSeconds Condition="'$(Scenario)' != '' and '$(TimeoutInSeconds)' == '' and ('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm')">3600</TimeoutInSeconds>
-    <TimeoutInSeconds Condition="'$(Scenario)' == '' and '$(TimeoutInSeconds)' == ''">900</TimeoutInSeconds>
-    <TimeoutInSeconds Condition="'$(Scenario)' != '' and '$(TimeoutInSeconds)' == ''">1800</TimeoutInSeconds>
-
-    <_timeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</_timeoutSpan>
+        '$(Scenario)' == 'gcstress0xc_jitminopts_heapverify1'">01:30:00</_workItemTimeout>
+    <_workItemTimeout Condition="'$(_workItemTimeout)' == '' and ('$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'tvOS')">00:30:00</_workItemTimeout>
+    <_workItemTimeout Condition="'$(Scenario)' == '' and '$(_workItemTimeout)' == '' and ('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm')">00:45:00</_workItemTimeout>
+    <_workItemTimeout Condition="'$(Scenario)' != '' and '$(_workItemTimeout)' == '' and ('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm')">01:00:00</_workItemTimeout>
+    <_workItemTimeout Condition="'$(Scenario)' == '' and '$(_workItemTimeout)' == ''">00:15:00</_workItemTimeout>
+    <_workItemTimeout Condition="'$(Scenario)' != '' and '$(_workItemTimeout)' == ''">00:30:00</_workItemTimeout>
 
     <!-- We need to enable xunit reporter so that it parses test results
          Package testing doesn't run on xunit. -->
@@ -73,7 +72,7 @@
     <HelixType Condition="'$(TestPackages)' == 'true'">test/functional/packaging/</HelixType>
   </PropertyGroup>
 
-  <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+  <PropertyGroup Condition="'$(TargetOS)' == 'Browser' or '$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'tvOS'">
     <IncludeXHarnessCli>true</IncludeXHarnessCli>
   </PropertyGroup>
 
@@ -162,9 +161,22 @@
     <Message Importance="High" Text="Using HelixCommand: $(HelixCommand)" />
     <Message Importance="High" Text="Using HelixType: $(HelixType)" />
     <Message Importance="High" Text="Using WorkItemArchiveWildCard: $(WorkItemArchiveWildCard)" />
-    <Message Importance="High" Text="Using Timeout: $(_timeoutSpan)" />
+    <Message Importance="High" Text="Using Timeout: $(_workItemTimeout)" />
+
+    <ItemGroup Condition="'$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'tvOS'">
+      <!-- Find all directories named *.app -->
+      <XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('$(TestArchiveTestsRoot)', '*.app', System.IO.SearchOption.AllDirectories))">
+        <Targets Condition="'$(TargetArchitecture)' == 'arm'">ios-device</Targets>
+        <Targets Condition="'$(TargetArchitecture)' == 'arm64'">ios-device</Targets>
+        <Targets Condition="'$(TargetArchitecture)' == 'x64'">ios-simulator-64</Targets>
+        <Targets Condition="'$(TargetArchitecture)' == 'x86'">ios-simulator-32</Targets>
+
+        <TestTimeout>$(_workItemTimeout)</TestTimeout>
+        <LaunchTimeout>$(_workItemTimeout)</LaunchTimeout>
+      </XHarnessAppBundleToTest>
+    </ItemGroup>
 
-    <ItemGroup>
+    <ItemGroup Condition="'$(TargetOS)' != 'iOS' and '$(TargetOS)' != 'tvOS'">
       <HelixCorrelationPayload Include="$(HelixCorrelationPayload)"
                                Condition="'$(IncludeHelixCorrelationPayload)' == 'true'" />
 
@@ -173,12 +185,13 @@
       <HelixWorkItem Include="@(_WorkItem -> '%(FileName)')">
         <PayloadArchive>%(Identity)</PayloadArchive>
         <Command>$(HelixCommand)</Command>
-        <Timeout>$(_timeoutSpan)</Timeout>
+        <Timeout>$(_workItemTimeout)</Timeout>
       </HelixWorkItem>
     </ItemGroup>
 
     <Message Condition="'$(Scenario)' != ''" Importance="High" Text="Done building Helix work items for scenario $(Scenario). Work item count: @(_WorkItem->Count())" />
-    <Message Condition="'$(Scenario)' == ''" Importance="High" Text="Done building Helix work items. Work item count: @(_WorkItem->Count())" />
+    <Message Condition="'$(Scenario)' == '' and ('$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'tvOS')" Importance="High" Text="Done building Helix work items. Work item count: @(XHarnessAppBundleToTest->Count())" />
+    <Message Condition="'$(Scenario)' == '' and '$(TargetOS)' != 'iOS' and '$(TargetOS)' != 'tvOS'" Importance="High" Text="Done building Helix work items. Work item count: @(_WorkItem->Count())" />
 
   </Target>