feat: CI runs for runtime and buildtime init on mobile (#1944)

Author: bitsandfoxes

Diff for: .github/workflows/android-smoke-test.yml

@@ -7,6 +7,9 @@ on:
       api-level:
         required: true
         type: string
+      init-type:
+        required: true
+        type: string
     # Map the workflow outputs to job outputs
     outputs:
       status:
@@ -33,7 +36,7 @@ jobs:
       - name: Download test app artifact
         uses: actions/download-artifact@v4
         with:
-          name: testapp-Android-compiled-${{ inputs.unity-version }}
+          name: testapp-Android-compiled-${{ inputs.unity-version }}-${{ inputs.init-type }}
           path: samples/IntegrationTest/Build
 
       # See https://github.blog/changelog/2023-02-23-hardware-accelerated-android-virtualization-on-actions-windows-and-linux-larger-hosted-runners/
@@ -83,4 +86,4 @@ jobs:
         with:
           name: testapp-android-logs-${{ inputs.api-level }}-${{ inputs.unity-version }}
           path: ${{ env.ARTIFACTS_PATH }}
-          retention-days: 14
+          retention-days: 14

Diff for: .github/workflows/ci.yml

@@ -339,17 +339,45 @@ jobs:
         run: |
           # Note: remove local.properties file that contains Android SDK & NDK paths in the Unity installation.
           rm -rf samples/IntegrationTest/Build/*_BackUpThisFolder_ButDontShipItWithYourGame
-          tar -cvzf test-app.tar.gz samples/IntegrationTest/Build
+          tar -cvzf test-app-runtime.tar.gz samples/IntegrationTest/Build
 
+      # Upload runtime initialization build
       - name: Upload test app
         uses: actions/upload-artifact@v4
         with:
-          name: testapp-${{ matrix.platform }}-${{ matrix.unity-version }}
+          name: testapp-${{ matrix.platform }}-${{ matrix.unity-version }}-runtime
           if-no-files-found: error
-          path: test-app.tar.gz
-          # Lower retention period - we only need this to retry CI.
+          path: test-app-runtime.tar.gz
           retention-days: 14
 
+      - name: Configure Sentry for mobile platforms (build-time initialization)
+        if: ${{ matrix.platform == 'iOS' || matrix.platform == 'Android' }}
+        run: |
+          $optionsPath = "samples/IntegrationTest/Assets/Scripts/OptionsConfiguration.cs"
+          $content = Get-Content $optionsPath -Raw
+          $content = $content -replace 'AndroidNativeInitializationType = NativeInitializationType.Runtime', 'AndroidNativeInitializationType = NativeInitializationType.BuildTime'
+          $content = $content -replace 'IosNativeInitializationType = NativeInitializationType.Runtime', 'IosNativeInitializationType = NativeInitializationType.BuildTime'
+          Set-Content $optionsPath $content
+
+      - name: Build Project for mobile platforms (build-time initialization)
+        if: ${{ matrix.platform == 'iOS' || matrix.platform == 'Android' }}
+        run: ./test/Scripts.Integration.Test/build-project.ps1 -UnityPath "${{ env.UNITY_PATH }}" -Platform ${{ matrix.build_platform }} -CheckSymbols:$${{ matrix.check_symbols }} -UnityVersion "${{ matrix.unity-version }}"
+
+      - name: Create archive (build-time initialization)
+        shell: bash
+        run: |
+          rm -rf samples/IntegrationTest/Build/*_BackUpThisFolder_ButDontShipItWithYourGame
+          tar -cvzf test-app-buildtime.tar.gz samples/IntegrationTest/Build
+
+      # Upload build-time initialization build
+      - name: Upload test app (build-time initialization)
+        uses: actions/upload-artifact@v4
+        with:
+          name: testapp-${{ matrix.platform }}-${{ matrix.unity-version }}-buildtime
+          if-no-files-found: error
+          path: test-app-buildtime.tar.gz
+          retention-days: 14
+          
       - name: Upload IntegrationTest project on failure
         if: ${{ failure() }}
         uses: actions/upload-artifact@v4
@@ -437,27 +465,30 @@ jobs:
   android-smoke-test-run:
     if: ${{ !startsWith(github.ref, 'refs/heads/release/') }}
     needs: [mobile-smoke-test-compile]
-    name: ${{ matrix.unity-version }} Android ${{ matrix.api-level }} Run Smoke Test
+    name: ${{ matrix.unity-version }} Android ${{ matrix.api-level }} ${{ matrix.init-type }} Run Smoke Test
     uses: ./.github/workflows/android-smoke-test.yml
     with:
       unity-version: ${{ matrix.unity-version }}
       api-level: ${{ matrix.api-level }}
+      init-type: ${{ matrix.init-type }}
     strategy:
       fail-fast: false
       matrix:
         api-level: [30, 31, 34] # last updated January 2025
+        init-type: ["runtime", "buildtime"]
         unity-version: ["2019", "6000"]
 
   mobile-smoke-test-compile:
     if: ${{ !startsWith(github.ref, 'refs/heads/release/') }}
     needs: [smoke-test-build]
-    name: ${{ matrix.unity-version }} ${{ matrix.platform }}  Compile Smoke Test
+    name: ${{ matrix.unity-version }} ${{ matrix.platform }} ${{ matrix.init-type }} Compile Smoke Test
     runs-on: ${{ matrix.platform == 'iOS' && 'macos-latest' || 'ubuntu-latest-4-cores' }}
     strategy:
       fail-fast: false
       matrix:
         unity-version: ["2019", "2022", "6000"]
         platform: ["Android", "iOS"]
+        init-type: ["runtime", "buildtime"]
         include:
           # See supported version in https://docs.unity3d.com/6000.0/Documentation/Manual/android-sdksetup.html
           - unity-version: "2019"
@@ -474,10 +505,10 @@ jobs:
       - name: Download app project
         uses: actions/download-artifact@v4
         with:
-          name: testapp-${{ matrix.platform }}-${{ matrix.unity-version }}
+          name: testapp-${{ matrix.platform }}-${{ matrix.unity-version }}-${{ matrix.init-type }}
 
       - name: Extract app project
-        run: tar -xvzf test-app.tar.gz
+        run: tar -xvzf test-app-${{ matrix.init-type }}.tar.gz
 
       - name: Setup Android
         uses: android-actions/setup-android@7c5672355aaa8fde5f97a91aa9a99616d1ace6bc # pin@v2
@@ -526,7 +557,7 @@ jobs:
         if: ${{ failure() }}
         uses: actions/upload-artifact@v4
         with:
-          name: failed-project-${{ matrix.platform }}-${{ matrix.unity-version }}-but-compiled
+          name: failed-project-${{ matrix.platform }}-${{ matrix.unity-version }}-${{ matrix.init-type }}-but-compiled
           path: |
             samples/IntegrationTest
             !samples/IntegrationTest/Build/*_BackUpThisFolder_ButDontShipItWithYourGame
@@ -539,7 +570,7 @@ jobs:
         if: |
           !(matrix.platform == 'Android' && matrix.unity-version == '2022') || matrix.platform == 'iOS'
         with:
-          name: testapp-${{ matrix.platform }}-compiled-${{ matrix.unity-version }}
+          name: testapp-${{ matrix.platform }}-compiled-${{ matrix.unity-version }}-${{ matrix.init-type }}
           # Collect app but ignore the files that are not required for the test.
           path: |
             samples/IntegrationTest/Build/*.apk
@@ -552,7 +583,7 @@ jobs:
   ios-smoke-test-run:
     if: ${{ !startsWith(github.ref, 'refs/heads/release/') }}
     needs: [mobile-smoke-test-compile]
-    name: ${{ matrix.unity-version }} iOS ${{ matrix.ios }} Run Smoke Test
+    name: ${{ matrix.unity-version }} iOS ${{ matrix.ios }} ${{ matrix.init-type }} Run Smoke Test
     runs-on: macos-13 # Pinning to get the oldest, supported version of iOS simulator
     strategy:
       fail-fast: false
@@ -568,6 +599,7 @@ jobs:
         # Also make sure to match the versions available here:
         #  - https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md
         ios: ["16.1", latest] # last updated October 2024
+        init-type: ["runtime", "buildtime"]
 
     steps:
       - name: Checkout
@@ -576,7 +608,7 @@ jobs:
       - name: Download app artifact
         uses: actions/download-artifact@v4
         with:
-          name: testapp-iOS-compiled-${{ matrix.unity-version }}
+          name: testapp-iOS-compiled-${{ matrix.unity-version }}-${{ matrix.init-type }}
           path: samples/IntegrationTest/Build
 
       - name: Set Xcode for iOS version ${{matrix.ios}}
@@ -614,10 +646,10 @@ jobs:
         uses: actions/download-artifact@v4
         id: download
         with:
-          name: testapp-${{ matrix.platform }}-${{ matrix.unity-version }}
+          name: testapp-${{ matrix.platform }}-${{ matrix.unity-version }}-runtime
 
       - name: Extract test app
-        run: tar -xvzf test-app.tar.gz
+        run: tar -xvzf test-app-runtime.tar.gz
 
       - name: Run (WebGL)
         if: ${{ matrix.platform == 'WebGL' }}

Diff for: scripts/smoke-test-android.ps1

@@ -424,8 +424,8 @@ $results.hasntCrashedTestPassed = RunTestWithRetry -Name "hasnt-crashed" -Succes
 try
 {
     CrashTestWithServer -SuccessString "POST /api/12345/envelope/ HTTP/1.1`" 200 -b'1f8b08000000000000" -CrashTestCallback {
-        $results.crashTestPassed = RunTestWithRetry -Name "crash" -SuccessString "CRASH TEST: Issuing a native crash" -FailureString "CRASH TEST: FAIL" -MaxRetries 3
-        $results.hasCrashTestPassed = RunTestWithRetry -Name "has-crashed" -SuccessString "HAS-CRASHED TEST: PASS" -FailureString "HAS-CRASHED TEST: FAIL" -MaxRetries 3
+        $results.crashTestPassed = RunTest -Name "crash" -SuccessString "CRASH TEST: Issuing a native crash" -FailureString "CRASH TEST: FAIL"
+        $results.hasCrashTestPassed = RunTest -Name "has-crashed" -SuccessString "HAS-CRASHED TEST: PASS" -FailureString "HAS-CRASHED TEST: FAIL"
     }
 }
 catch

Diff for: src/Sentry.Unity.Android/SentryJava.cs

@@ -8,8 +8,8 @@ namespace Sentry.Unity.Android;
 
 internal interface ISentryJava
 {
-    public bool IsEnabled(IJniExecutor jniExecutor);
-    public bool Init(IJniExecutor jniExecutor, SentryUnityOptions options, TimeSpan timeout);
+    public bool IsEnabled(IJniExecutor jniExecutor, TimeSpan timeout);
+    public void Init(IJniExecutor jniExecutor, SentryUnityOptions options, TimeSpan timeout);
     public string? GetInstallationId(IJniExecutor jniExecutor);
     public bool? CrashedLastRun(IJniExecutor jniExecutor);
     public void Close(IJniExecutor jniExecutor);
@@ -45,16 +45,16 @@ internal class SentryJava : ISentryJava
 {
     private static AndroidJavaObject GetSentryJava() => new AndroidJavaClass("io.sentry.Sentry");
 
-    public bool IsEnabled(IJniExecutor jniExecutor)
+    public bool IsEnabled(IJniExecutor jniExecutor, TimeSpan timeout)
     {
         return jniExecutor.Run(() =>
         {
             using var sentry = GetSentryJava();
             return sentry.CallStatic<bool>("isEnabled");
-        });
+        }, timeout);
     }
 
-    public bool Init(IJniExecutor jniExecutor, SentryUnityOptions options, TimeSpan timeout)
+    public void Init(IJniExecutor jniExecutor, SentryUnityOptions options, TimeSpan timeout)
     {
         jniExecutor.Run(() =>
         {
@@ -97,8 +97,6 @@ public bool Init(IJniExecutor jniExecutor, SentryUnityOptions options, TimeSpan
                 androidOptions.Call("setEnableScopePersistence", false);
             }, options.DiagnosticLogger));
         }, timeout);
-
-        return IsEnabled(jniExecutor);
     }
 
     internal class AndroidOptionsConfiguration : AndroidJavaProxy

Diff for: src/Sentry.Unity.Android/SentryNativeAndroid.cs

@@ -28,33 +28,50 @@ public static void Configure(SentryUnityOptions options, ISentryUnityInfo sentry
             return;
         }
 
+        options.DiagnosticLogger?.LogDebug("Checking whether the Android SDK is present.");
+
         if (!SentryJava.IsSentryJavaPresent())
         {
             options.DiagnosticLogger?.LogError("Android Native Support has been enabled but the " +
-                                               "Sentry Java SDK is missing. This could have been caused by a mismatching" +
+                                               "Android SDK is missing. This could have been caused by a mismatching" +
                                                "build time / runtime configuration. Please make sure you have " +
                                                "Android Native Support enabled during build time.");
             return;
         }
 
         JniExecutor ??= new JniExecutor(options.DiagnosticLogger);
 
-        if (SentryJava.IsEnabled(JniExecutor))
+        options.DiagnosticLogger?.LogDebug("Checking whether the Android SDK has already been initialized");
+
+        if (SentryJava.IsEnabled(JniExecutor, TimeSpan.FromMilliseconds(200)))
         {
             options.DiagnosticLogger?.LogDebug("The Android SDK is already initialized");
         }
-        // Local testing had Init at an average of about 25ms.
-        else if (!SentryJava.Init(JniExecutor, options, TimeSpan.FromMilliseconds(200)))
+        else
         {
-            options.DiagnosticLogger?.LogError("Failed to initialize Android Native Support");
-            return;
+            options.DiagnosticLogger?.LogInfo("Initializing the Android SDK");
+
+            // Local testing had Init at an average of about 25ms.
+            SentryJava.Init(JniExecutor, options, TimeSpan.FromMilliseconds(200));
+
+            options.DiagnosticLogger?.LogDebug("Validating Android SDK initialization");
+
+            if (!SentryJava.IsEnabled(JniExecutor, TimeSpan.FromMilliseconds(200)))
+            {
+                options.DiagnosticLogger?.LogError("Failed to initialize Android Native Support");
+                return;
+            }
         }
 
+        options.DiagnosticLogger?.LogDebug("Configuring scope sync");
+
         options.NativeContextWriter = new NativeContextWriter(JniExecutor, SentryJava);
         options.ScopeObserver = new AndroidJavaScopeObserver(options, JniExecutor);
         options.EnableScopeSync = true;
         options.CrashedLastRun = () =>
         {
+            options.DiagnosticLogger?.LogDebug("Checking for `CrashedLastRun`");
+
             var crashedLastRun = SentryJava.CrashedLastRun(JniExecutor);
             if (crashedLastRun is null)
             {
@@ -89,6 +106,8 @@ public static void Configure(SentryUnityOptions options, ISentryUnityInfo sentry
 
         options.NativeSupportCloseCallback = () => Close(options, sentryUnityInfo);
 
+        options.DiagnosticLogger?.LogDebug("Fetching installation ID");
+
         options.DefaultUserId = SentryJava.GetInstallationId(JniExecutor);
         if (string.IsNullOrEmpty(options.DefaultUserId))
         {

Diff for: test/Scripts.Integration.Test/Scripts/OptionsConfiguration.cs

@@ -40,6 +40,10 @@ public override void Configure(SentryUnityOptions options)
         // If an ANR triggers while the smoke test runs, the test would fail because we expect exact order of events.
         options.DisableAnrIntegration();
 
-        Debug.Log("Sentry: BuildTimeOptions::Configure() finished");
+        // These options will get overwritten by CI. This allows us to create artifacts for both initialization types.
+        options.AndroidNativeInitializationType = NativeInitializationType.Runtime;
+        options.IosNativeInitializationType = NativeInitializationType.Runtime;
+
+        Debug.Log("Sentry: OptionsConfig::Configure() finished");
     }
 }

Diff for: test/Sentry.Unity.Android.Tests/SentryNativeAndroidTests.cs

@@ -122,7 +122,7 @@ public void Configure_DefaultConfigurationSentryJavaNotPresent_LogsErrorAndRetur
 
         Assert.IsTrue(_logger.Logs.Any(log =>
             log.logLevel == SentryLevel.Error &&
-            log.message.Contains("Sentry Java SDK is missing.")));
+            log.message.Contains("Android SDK is missing.")));
 
         Assert.Null(_options.ScopeObserver);
     }

Diff for: test/Sentry.Unity.Android.Tests/TestSentryJava.cs

@@ -10,9 +10,9 @@ internal class TestSentryJava : ISentryJava
     public string? InstallationId { get; set; }
     public bool? IsCrashedLastRun { get; set; }
 
-    public bool IsEnabled(IJniExecutor jniExecutor) => Enabled;
+    public bool IsEnabled(IJniExecutor jniExecutor, TimeSpan timeout) => Enabled;
 
-    public bool Init(IJniExecutor jniExecutor, SentryUnityOptions options, TimeSpan timeout) => InitSuccessful;
+    public void Init(IJniExecutor jniExecutor, SentryUnityOptions options, TimeSpan timeout) { }
 
     public string? GetInstallationId(IJniExecutor jniExecutor) => InstallationId;