Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/firebase-runtime-failure-backlog.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Firebase Runtime Failure Backlog

This backlog tracks binding drifts that are concrete candidates for the runtime-failure remediation loop.

## Ready

| Case id | Target/member | Audit finding reference | Expected runtime failure mechanism | Proof status | Fix PR | Merged commit |
| --- | --- | --- | --- | --- | --- | --- |
| `cloudfirestore-getquerynamed` | `Firebase.CloudFirestore.Firestore.GetQueryNamed` | `output/firebase-binding-audit/report.json` -> `CloudFirestore` -> `GetQueryNamed` (`signature-drift`) | The binding currently exports `getQueryNamed:completion:` as `NSInputStream` instead of `NSString`, so native Firestore receives `__NSCFInputStream` and throws an Objective-C exception when it treats the argument like a string. | Proved locally on the unfixed binding. Evidence: `ObjCRuntime.ObjCException`, `NSInvalidArgumentException`, selector `-[__NSCFInputStream length]`, artifact `tests/E2E/Firebase.Foundation/artifacts/firebase-foundation-result-cloudfirestore-getquerynamed-failure.json`, log `tests/E2E/Firebase.Foundation/artifacts/firebase-foundation-sim-cloudfirestore-getquerynamed-failure.log`. | Pending | - |

## Investigating

No additional runtime-failure candidates are currently promoted.

## Done

None yet.
2 changes: 1 addition & 1 deletion source/Firebase/CloudFirestore/ApiDefinition.cs
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ interface Firestore

// - (void)getQueryNamed:(NSString *)name completion:(void (^)(FIRQuery *_Nullable query))completion
[Export ("getQueryNamed:completion:")]
void GetQueryNamed (NSInputStream bundleStream, Action<Query> completion);
void GetQueryNamed (string name, Action<Query> completion);
}

// @interface FIRTimestamp : NSObject <NSCopying>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,13 @@
<IsPackable>false</IsPackable>
<ProvisioningType>manual</ProvisioningType>
<EnableNullabilityValidation>false</EnableNullabilityValidation>
<RuntimeDriftCase></RuntimeDriftCase>
<RuntimeDriftCaseMethod></RuntimeDriftCaseMethod>
<RuntimeDriftCasePropsPath></RuntimeDriftCasePropsPath>
</PropertyGroup>

<Import Project="$(RuntimeDriftCasePropsPath)" Condition="'$(RuntimeDriftCasePropsPath)' != '' and Exists('$(RuntimeDriftCasePropsPath)')" />

<PropertyGroup Condition="'$(EnableNullabilityValidation)' == 'true'">
<DefineConstants>$(DefineConstants);ENABLE_NULLABILITY_VALIDATION</DefineConstants>
</PropertyGroup>
Expand All @@ -42,6 +47,17 @@
<BundleResource Include="GoogleService-Info.plist" Condition="Exists('GoogleService-Info.plist')" />
</ItemGroup>

<ItemGroup Condition="'$(RuntimeDriftCase)' != ''">
<AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute">
<_Parameter1>RuntimeDriftCase</_Parameter1>
<_Parameter2>$(RuntimeDriftCase)</_Parameter2>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute" Condition="'$(RuntimeDriftCaseMethod)' != ''">
<_Parameter1>RuntimeDriftCaseMethod</_Parameter1>
<_Parameter2>$(RuntimeDriftCaseMethod)</_Parameter2>
</AssemblyAttribute>
</ItemGroup>

<ItemGroup>
<PackageReference Include="AdamE.Firebase.iOS.Analytics" Version="12.6.0" />
<PackageReference Include="AdamE.Firebase.iOS.Core" Version="12.6.0" />
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
using System.Reflection;

#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_GETQUERYNAMED
using Firebase.CloudFirestore;
using Foundation;
using ObjCRuntime;
#endif

namespace FirebaseFoundationE2E;

static class FirebaseRuntimeDriftCases
{
static readonly TimeSpan AsyncTimeout = TimeSpan.FromSeconds(5);

public static string? GetConfiguredCaseId()
{
return GetAssemblyMetadataValue("RuntimeDriftCase");
}

public static async Task<string> ExecuteConfiguredCaseAsync()
{
var caseId = GetConfiguredCaseId();
if (string.IsNullOrWhiteSpace(caseId))
{
throw new InvalidOperationException("Runtime drift mode was requested without a RuntimeDriftCase value.");
}

var methodName = GetAssemblyMetadataValue("RuntimeDriftCaseMethod");
if (string.IsNullOrWhiteSpace(methodName))
{
throw new InvalidOperationException($"Runtime drift case '{caseId}' is missing RuntimeDriftCaseMethod metadata.");
}

var method = typeof(FirebaseRuntimeDriftCases).GetMethod(methodName, BindingFlags.Static | BindingFlags.NonPublic);
if (method is null)
{
throw new InvalidOperationException($"Runtime drift case '{caseId}' points at missing method '{methodName}'.");
}

if (method.Invoke(null, null) is not Task<string> task)
{
throw new InvalidOperationException($"Runtime drift case '{caseId}' method '{methodName}' did not return Task<string>.");
}

return await task;
}

static string? GetAssemblyMetadataValue(string key)
{
return typeof(FirebaseRuntimeDriftCases)
.Assembly
.GetCustomAttributes<AssemblyMetadataAttribute>()
.FirstOrDefault(attribute => string.Equals(attribute.Key, key, StringComparison.Ordinal))
?.Value;
}

#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_GETQUERYNAMED
static async Task<string> VerifyCloudFirestoreGetQueryNamedAsync()
{
const string selector = "getQueryNamed:completion:";

var firestore = Firestore.SharedInstance;
if (firestore is null)
{
throw new InvalidOperationException("Firebase.CloudFirestore.Firestore.SharedInstance returned null after App.Configure().");
}

var queryName = "codex-firestore-missing-query";
var callbackInvoked = false;
var returnedQueryWasNull = false;
NSException? marshaledException = null;
MarshalObjectiveCExceptionMode? marshaledExceptionMode = null;
var completionSource = new TaskCompletionSource<Query?>(TaskCreationOptions.RunContinuationsAsynchronously);

void OnMarshalObjectiveCException(object? sender, MarshalObjectiveCExceptionEventArgs args)
{
marshaledException ??= args.Exception;
marshaledExceptionMode ??= args.ExceptionMode;
}

Runtime.MarshalObjectiveCException += OnMarshalObjectiveCException;
try
{
try
{
firestore.GetQueryNamed(queryName, query =>
{
callbackInvoked = true;
returnedQueryWasNull = query is null;
completionSource.TrySetResult(query);
});
}
catch (ObjCException ex)
{
throw new InvalidOperationException(
$"Selector '{selector}' should not throw after the binding fix, but observed {ex.GetType().FullName}. " +
$"Runtime argument type: {queryName.GetType().FullName}. " +
$"NSException.Name: {FormatDetail(marshaledException?.Name?.ToString())}. " +
$"NSException.Reason: {FormatDetail(marshaledException?.Reason)}. " +
$"Marshal mode: {FormatDetail(marshaledExceptionMode?.ToString())}.",
ex);
}

var completedTask = await Task.WhenAny(completionSource.Task, Task.Delay(AsyncTimeout));
if (completedTask != completionSource.Task)
{
throw new TimeoutException(
$"Selector '{selector}' did not invoke its completion callback within {AsyncTimeout.TotalSeconds} seconds after the binding fix.");
}

var returnedQuery = await completionSource.Task;
if (!callbackInvoked)
{
throw new InvalidOperationException(
$"Selector '{selector}' completed without throwing, but the completion callback was never marked as invoked.");
}

if (marshaledException is not null)
{
throw new InvalidOperationException(
$"Selector '{selector}' completed, but Runtime.MarshalObjectiveCException captured unexpected NSException.Name '{marshaledException.Name}'. " +
$"Reason: {FormatDetail(marshaledException.Reason)}. Marshal mode: {FormatDetail(marshaledExceptionMode?.ToString())}.");
}

return
$"Selector '{selector}' completed without ObjC exception after the binding fix. " +
$"Runtime argument type: {queryName.GetType().FullName}. " +
$"CallbackInvoked: {callbackInvoked}. " +
$"ReturnedQueryWasNull: {returnedQueryWasNull}. " +
$"ReturnedQueryType: {returnedQuery?.GetType().FullName ?? "<null>"}.";
}
finally
{
Runtime.MarshalObjectiveCException -= OnMarshalObjectiveCException;
}
}
#endif

static string FormatDetail(string? value)
{
return string.IsNullOrWhiteSpace(value) ? "<empty>" : value;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ public static async Task RunAsync(StatusViewController statusViewController)
#if ENABLE_NULLABILITY_VALIDATION
await statusViewController.AppendLineAsync("Firebase nullability validation mode enabled.");
#endif
var runtimeDriftCase = FirebaseRuntimeDriftCases.GetConfiguredCaseId();
if (!string.IsNullOrWhiteSpace(runtimeDriftCase))
{
await statusViewController.AppendLineAsync($"Runtime drift case mode enabled: {runtimeDriftCase}");
}

try
{
Expand All @@ -44,74 +49,82 @@ await ExecuteCaseAsync(result, statusViewController, "ConfigureApp", async () =>
return $"Configured app '{defaultApp.Name}'.";
});

await ExecuteCaseAsync(result, statusViewController, "CoreSurface", async () =>
if (!string.IsNullOrWhiteSpace(runtimeDriftCase))
{
var defaultApp = App.DefaultInstance ?? throw new InvalidOperationException("Firebase.Core.App.DefaultInstance returned null.");
var firebaseVersion = App.FirebaseVersion;

if (string.IsNullOrWhiteSpace(firebaseVersion))
await ExecuteCaseAsync(result, statusViewController, $"RuntimeDrift:{runtimeDriftCase}", () =>
FirebaseRuntimeDriftCases.ExecuteConfiguredCaseAsync());
}
else
{
await ExecuteCaseAsync(result, statusViewController, "CoreSurface", async () =>
{
throw new InvalidOperationException("Firebase.Core.App.FirebaseVersion returned an empty value.");
}
var defaultApp = App.DefaultInstance ?? throw new InvalidOperationException("Firebase.Core.App.DefaultInstance returned null.");
var firebaseVersion = App.FirebaseVersion;

result.FirebaseVersion = firebaseVersion;
return $"Firebase version: {firebaseVersion}; app name: {defaultApp.Name}.";
});
if (string.IsNullOrWhiteSpace(firebaseVersion))
{
throw new InvalidOperationException("Firebase.Core.App.FirebaseVersion returned an empty value.");
}

result.FirebaseVersion = firebaseVersion;
return $"Firebase version: {firebaseVersion}; app name: {defaultApp.Name}.";
});

#if ENABLE_NULLABILITY_VALIDATION
await ExecuteCaseAsync(result, statusViewController, "CoreNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyCoreNullabilityAsync());
await ExecuteCaseAsync(result, statusViewController, "CoreNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyCoreNullabilityAsync());

await ExecuteCaseAsync(result, statusViewController, "AnalyticsNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyAnalyticsNullabilityAsync());
await ExecuteCaseAsync(result, statusViewController, "AnalyticsNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyAnalyticsNullabilityAsync());

await ExecuteCaseAsync(result, statusViewController, "AppCheckNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyAppCheckNullabilityAsync());
await ExecuteCaseAsync(result, statusViewController, "AppCheckNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyAppCheckNullabilityAsync());

await ExecuteCaseAsync(result, statusViewController, "CloudMessagingNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyCloudMessagingNullabilityAsync());
await ExecuteCaseAsync(result, statusViewController, "CloudMessagingNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyCloudMessagingNullabilityAsync());

await ExecuteCaseAsync(result, statusViewController, "CloudFirestoreNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyCloudFirestoreNullabilityAsync());
await ExecuteCaseAsync(result, statusViewController, "CloudFirestoreNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyCloudFirestoreNullabilityAsync());

await ExecuteCaseAsync(result, statusViewController, "CrashlyticsNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyCrashlyticsNullabilityAsync());
await ExecuteCaseAsync(result, statusViewController, "CrashlyticsNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyCrashlyticsNullabilityAsync());

await ExecuteCaseAsync(result, statusViewController, "DatabaseNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyDatabaseNullabilityAsync());
await ExecuteCaseAsync(result, statusViewController, "DatabaseNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyDatabaseNullabilityAsync());

await ExecuteCaseAsync(result, statusViewController, "PerformanceNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyPerformanceNullabilityAsync());
await ExecuteCaseAsync(result, statusViewController, "PerformanceNullabilitySurface", () =>
FirebaseNullabilityValidation.VerifyPerformanceNullabilityAsync());
#endif

await ExecuteCaseAsync(result, statusViewController, "InstallationsSurface", async () =>
{
var installations = Installations.DefaultInstance ?? throw new InvalidOperationException("Firebase.Installations.Installations.DefaultInstance returned null.");
var installationId = await GetInstallationIdAsync(installations);
result.InstallationsIdPreview = installationId.Length > 16
? installationId[..8] + "..." + installationId[^8..]
: installationId;
return $"Installation id: {result.InstallationsIdPreview}.";
});

await ExecuteCaseAsync(result, statusViewController, "AnalyticsSurface", async () =>
{
Analytics.SetAnalyticsCollectionEnabled(true);
Analytics.SetConsent(new Dictionary<ConsentType, ConsentStatus>
await ExecuteCaseAsync(result, statusViewController, "InstallationsSurface", async () =>
{
[ConsentType.AnalyticsStorage] = ConsentStatus.Granted,
[ConsentType.AdStorage] = ConsentStatus.Denied,
var installations = Installations.DefaultInstance ?? throw new InvalidOperationException("Firebase.Installations.Installations.DefaultInstance returned null.");
var installationId = await GetInstallationIdAsync(installations);
result.InstallationsIdPreview = installationId.Length > 16
? installationId[..8] + "..." + installationId[^8..]
: installationId;
return $"Installation id: {result.InstallationsIdPreview}.";
});

Analytics.LogEvent(EventNamesConstants.ScreenView.ToString(), new Dictionary<object, object>
await ExecuteCaseAsync(result, statusViewController, "AnalyticsSurface", async () =>
{
[ParameterNamesConstants.ScreenName] = new NSString("firebase_nuget_e2e"),
[ParameterNamesConstants.ScreenClass] = new NSString(nameof(FirebaseSelfTestRunner)),
[ParameterNamesConstants.Success] = NSNumber.FromBoolean(true),
Analytics.SetAnalyticsCollectionEnabled(true);
Analytics.SetConsent(new Dictionary<ConsentType, ConsentStatus>
{
[ConsentType.AnalyticsStorage] = ConsentStatus.Granted,
[ConsentType.AdStorage] = ConsentStatus.Denied,
});

Analytics.LogEvent(EventNamesConstants.ScreenView.ToString(), new Dictionary<object, object>
{
[ParameterNamesConstants.ScreenName] = new NSString("firebase_nuget_e2e"),
[ParameterNamesConstants.ScreenClass] = new NSString(nameof(FirebaseSelfTestRunner)),
[ParameterNamesConstants.Success] = NSNumber.FromBoolean(true),
});

return "Analytics collection and event APIs completed without throwing.";
});

return "Analytics collection and event APIs completed without throwing.";
});
}
}
catch (Exception ex)
{
Expand Down
12 changes: 12 additions & 0 deletions tests/E2E/Firebase.Foundation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,3 +80,15 @@ dotnet tool run dotnet-cake -- --target=nuget --names="Firebase.Analytics,Fireba
```sh
tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --enable-nullability-validation
```

## Targeted runtime drift mode

The harness also supports a targeted runtime-drift lane for one binding drift at a time. This mode runs only `ConfigureApp` plus the selected drift case, so each remediation PR can prove the unfixed runtime failure locally, then keep only the success-oriented regression test in the final diff.

The checked-in case manifest lives at [`runtime-drift-cases.json`](./runtime-drift-cases.json), and the backlog/queue is tracked in [`docs/firebase-runtime-failure-backlog.md`](../../docs/firebase-runtime-failure-backlog.md).
Comment thread
AdamEssenmacher marked this conversation as resolved.

Run a specific drift case with:

```sh
tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --runtime-drift-case cloudfirestore-getquerynamed
```
15 changes: 15 additions & 0 deletions tests/E2E/Firebase.Foundation/runtime-drift-cases.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"cases": [
{
"id": "cloudfirestore-getquerynamed",
"method": "VerifyCloudFirestoreGetQueryNamedAsync",
"bindingPackage": "AdamE.Firebase.iOS.CloudFirestore",
"packages": [
{
"id": "AdamE.Firebase.iOS.CloudFirestore",
"version": "12.6.0"
}
]
}
]
}
Loading
Loading