Add file-backed cert store so CertHelper works on Mac (#5257)

* Add file-backed cert store so CertHelper works on Mac

On macOS the CurrentUser\My X509Store maps to the login Keychain, which
cannot export a private key back out without an interactive password
prompt. As a result the iphone/Mac queues bypassed CertHelper entirely
and relied on hand-provisioned static PFX files that were never rotated.

Introduce an ILocalCertStore abstraction with two implementations:
- X509LocalCertStore: existing Windows/Linux behavior over the OS store.
- FileLocalCertStore: persists PFX files (thumbprint-named) in a known
  directory on Mac, loaded Exportable so no Keychain round-trip is needed.

A factory selects the implementation by OS. LocalCert now loads through
the abstraction, and Program.Main rotates via ILocalCertStore.Rotate and
exports the in-memory Key Vault certificates to stdout instead of
round-tripping through the OS store. common.py's get_certificates now
runs CertHelper on all platforms (removing the Mac static-file path), so
Mac hosts get the same bootstrap + Key Vault rotation as Linux.

Update LocalCertTests to mock ILocalCertStore and add FileLocalCertStore
round-trip tests.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Address PR review: reliable X509 rotation and restrict Mac key files

- X509LocalCertStore now opens the store read-write by default and no
  longer re-opens/closes it in Rotate, so rotation works reliably on
  Windows/Linux and reads remain valid.
- FileLocalCertStore restricts written PFX files to owner read/write
  (0600) on macOS so the private keys are not left world-readable.

Note: the reviewer's suggestion to load PFXs with EphemeralKeySet is not
applied because EphemeralKeySet throws PlatformNotSupportedException on
macOS, which is the only platform that uses FileLocalCertStore.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: DrewScoggins

scripts/performance/common.py

@@ -210,24 +210,17 @@ def retry_on_exception(
 
 def get_certificates() -> list[str]:
     '''
-    Gets the certificates from the certhelper tool and on Mac uses find-certificate.
+    Gets the certificates from the certhelper tool. CertHelper handles per-OS certificate
+    storage internally (X509 store on Windows/Linux, file-backed store on Mac).
     '''
-    if ismac():
-        certs: list[str] = []
-        with open("/Users/helix-runner/certs/LabCert1.pfx", "rb") as f:
-            certs.append(base64.b64encode(f.read()).decode())
-        with open("/Users/helix-runner/certs/LabCert2.pfx", "rb") as f:
-            certs.append(base64.b64encode(f.read()).decode())
-        return certs
-    else:
-        cmd_line = [(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))]
-        cert_helper = RunCommand(cmd_line, None, True, False, 0)
-        try:
-            return cert_helper.run_and_get_stdout().splitlines()
-        except Exception as ex:
-            getLogger().error("Failed to get certificates")
-            getLogger().error('{0}: {1}'.format(type(ex), str(ex)))
-            return []
+    cmd_line = [(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))]
+    cert_helper = RunCommand(cmd_line, None, True, False, 0)
+    try:
+        return cert_helper.run_and_get_stdout().splitlines()
+    except Exception as ex:
+        getLogger().error("Failed to get certificates")
+        getLogger().error('{0}: {1}'.format(type(ex), str(ex)))
+        return []
 
 
 def __write_pipeline_variable(name: str, value: str):

src/tools/CertHelper/ILocalCertStore.cs

@@ -0,0 +1,145 @@
+using System.Runtime.InteropServices;
+using System.Security.Cryptography.X509Certificates;
+
+namespace CertHelper;
+
+/// <summary>
+/// Abstraction over where the local perf certificates live. On Windows and Linux this is
+/// backed by the CurrentUser\My <see cref="X509Store"/>. On macOS the CurrentUser\My store maps
+/// to the login Keychain, which cannot export a private key back out without an interactive
+/// password prompt, so a file-backed store is used instead.
+/// </summary>
+public interface ILocalCertStore
+{
+    /// <summary>
+    /// Returns the certificates currently persisted in the local store.
+    /// </summary>
+    X509Certificate2Collection GetCertificates();
+
+    /// <summary>
+    /// Replaces <paramref name="certsToRemove"/> with <paramref name="certsToAdd"/> in the local store.
+    /// </summary>
+    void Rotate(X509Certificate2Collection certsToRemove, X509Certificate2Collection certsToAdd);
+}
+
+/// <summary>
+/// Picks the appropriate <see cref="ILocalCertStore"/> for the current operating system.
+/// </summary>
+public static class LocalCertStoreFactory
+{
+    public static ILocalCertStore Create()
+    {
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+        {
+            return new FileLocalCertStore();
+        }
+
+        return new X509LocalCertStore();
+    }
+}
+
+/// <summary>
+/// <see cref="ILocalCertStore"/> backed by the CurrentUser\My <see cref="X509Store"/>.
+/// Used on Windows and Linux where the managed store fully supports adding certificates with
+/// private keys and exporting them back out.
+/// </summary>
+public class X509LocalCertStore : ILocalCertStore
+{
+    private readonly IX509Store _store;
+
+    public X509LocalCertStore(IX509Store? store = null)
+    {
+        // Open read-write by default so rotation works. TestableX509Store opens the store in its
+        // constructor, so GetCertificates and Rotate both operate on the same already-open handle.
+        _store = store ?? new TestableX509Store(OpenFlags.ReadWrite);
+    }
+
+    public X509Certificate2Collection GetCertificates()
+    {
+        return _store.Certificates;
+    }
+
+    public void Rotate(X509Certificate2Collection certsToRemove, X509Certificate2Collection certsToAdd)
+    {
+        var store = _store.GetX509Store();
+        store.RemoveRange(certsToRemove);
+        store.AddRange(certsToAdd);
+    }
+}
+
+/// <summary>
+/// <see cref="ILocalCertStore"/> that persists certificates as PFX files in a known directory.
+/// Used on macOS where the OS certificate store (login Keychain) cannot export a private key
+/// without an interactive password prompt. Files are named by thumbprint so that the on-disk
+/// state can be reconciled exactly with the desired set on rotation.
+/// </summary>
+public class FileLocalCertStore : ILocalCertStore
+{
+    // Matches the directory the perf Mac hosts historically used for pre-provisioned certificates.
+    public const string DefaultDirectory = "/Users/helix-runner/certs";
+
+    private readonly string _directory;
+
+    public FileLocalCertStore(string? directory = null)
+    {
+        _directory = directory ?? DefaultDirectory;
+    }
+
+    public X509Certificate2Collection GetCertificates()
+    {
+        var certificates = new X509Certificate2Collection();
+        if (!Directory.Exists(_directory))
+        {
+            return certificates;
+        }
+
+        foreach (var file in Directory.EnumerateFiles(_directory, "*.pfx"))
+        {
+            try
+            {
+                var bytes = File.ReadAllBytes(file);
+#if NET9_0_OR_GREATER
+                certificates.Add(X509CertificateLoader.LoadPkcs12(bytes, "", X509KeyStorageFlags.Exportable));
+#else
+                certificates.Add(new X509Certificate2(bytes, "", X509KeyStorageFlags.Exportable));
+#endif
+            }
+            catch (Exception ex)
+            {
+                // Skip files that cannot be loaded; they will be treated as missing and trigger bootstrap/rotation.
+                Console.Error.WriteLine($"Failed to load certificate from {file}: {ex.Message}");
+            }
+        }
+
+        return certificates;
+    }
+
+    public void Rotate(X509Certificate2Collection certsToRemove, X509Certificate2Collection certsToAdd)
+    {
+        Directory.CreateDirectory(_directory);
+
+        // Clear the existing PFX files so the on-disk state matches the desired set exactly.
+        foreach (var file in Directory.EnumerateFiles(_directory, "*.pfx"))
+        {
+            File.Delete(file);
+        }
+
+        foreach (var cert in certsToAdd)
+        {
+            var path = Path.Combine(_directory, $"{cert.Thumbprint}.pfx");
+            File.WriteAllBytes(path, cert.Export(X509ContentType.Pfx));
+            RestrictToOwner(path);
+        }
+    }
+
+    // Private keys are written to disk as unprotected PFX files. On macOS (the only platform that
+    // uses this file-backed store) restrict them to owner read/write so the keys are not exposed if
+    // the directory permissions or umask are permissive.
+    private static void RestrictToOwner(string path)
+    {
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+        {
+            File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite);
+        }
+    }
+}

src/tools/CertHelper/LocalCert.cs

@@ -13,19 +13,19 @@ public class LocalCert : ILocalCert
 {
     public X509Certificate2Collection Certificates { get; set; }
     public bool RequiresBootstrap { get; private set; }
-    internal IX509Store LocalMachineCerts { get; set; }
+    internal ILocalCertStore LocalCertStore { get; set; }
 
-    public LocalCert(IX509Store? store = null)
+    public LocalCert(ILocalCertStore? store = null)
     {
-        LocalMachineCerts = store ?? new TestableX509Store();
+        LocalCertStore = store ?? LocalCertStoreFactory.Create();
         Certificates = new X509Certificate2Collection();
         RequiresBootstrap = false;
         GetLocalCerts();
     }
 
     private void GetLocalCerts()
     {
-        foreach (var cert in LocalMachineCerts.Certificates.Find(X509FindType.FindBySubjectName, "dotnetperf.microsoft.com", false))
+        foreach (var cert in LocalCertStore.GetCertificates().Find(X509FindType.FindBySubjectName, "dotnetperf.microsoft.com", false))
         {
             if (cert.Subject == "CN=dotnetperf.microsoft.com")
             {

src/tools/CertHelper/Program.cs

@@ -15,19 +15,18 @@ internal class Program
 
     static async Task<int> Main(string[] args)
     {
+        var certStore = LocalCertStoreFactory.Create();
+        X509Certificate2Collection? certsToExport = null;
         try
         {
-            var kvc = new KeyVaultCert();
+            var localCerts = new LocalCert(certStore);
+            var kvc = new KeyVaultCert(localCerts: localCerts);
             await kvc.LoadKeyVaultCertsAsync();
             if (kvc.ShouldRotateCerts())
             {
-                using (var localMachineCerts = new X509Store(StoreName.My, StoreLocation.CurrentUser))
-                {
-                    localMachineCerts.Open(OpenFlags.ReadWrite);
-                    localMachineCerts.RemoveRange(kvc.LocalCerts.Certificates);
-                    localMachineCerts.AddRange(kvc.KeyVaultCertificates);
-                }
+                certStore.Rotate(kvc.LocalCerts.Certificates, kvc.KeyVaultCertificates);
             }
+            certsToExport = kvc.KeyVaultCertificates;
             var bcc = new BlobContainerClient(new Uri("https://pvscmdupload.blob.core.windows.net/certstatus"),
                 new ClientCertificateCredential(TENANT_ID, CERT_CLIENT_ID, kvc.KeyVaultCertificates.First(), new() {SendCertificateChain = true}));
             var currentKeyValutCertThumbprints = "";
@@ -55,12 +54,14 @@ static async Task<int> Main(string[] args)
             Console.Error.WriteLine(ex.StackTrace);
         }
 
-        using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser, OpenFlags.ReadOnly))
+        // Export the current certificates to stdout. Prefer the in-memory Key Vault certificates
+        // (loaded as exportable) so we never round-trip through the OS store, which cannot export
+        // private keys on macOS. If Key Vault auth failed above, fall back to whatever is currently
+        // persisted locally so callers can still attempt to use existing certificates.
+        var exportSource = certsToExport ?? certStore.GetCertificates();
+        foreach (var cert in exportSource.Find(X509FindType.FindBySubjectName, "dotnetperf.microsoft.com", false))
         {
-            foreach(var cert in store.Certificates.Find(X509FindType.FindBySubjectName, "dotnetperf.microsoft.com", false))
-            {
-                Console.WriteLine(Convert.ToBase64String(cert.Export(X509ContentType.Pfx)));
-            }
+            Console.WriteLine(Convert.ToBase64String(cert.Export(X509ContentType.Pfx)));
         }
         return 0;
     }

src/tools/CertHelperTests/FileLocalCertStoreTests.cs

@@ -0,0 +1,88 @@
+using System.Security.Cryptography;
+using System.Security.Cryptography.X509Certificates;
+using Xunit;
+
+namespace CertHelper.Tests;
+
+public class FileLocalCertStoreTests : IDisposable
+{
+    private readonly string _directory;
+
+    public FileLocalCertStoreTests()
+    {
+        _directory = Path.Combine(Path.GetTempPath(), "certhelper-tests-" + Guid.NewGuid().ToString("N"));
+    }
+
+    public void Dispose()
+    {
+        if (Directory.Exists(_directory))
+        {
+            Directory.Delete(_directory, recursive: true);
+        }
+    }
+
+    private static X509Certificate2 MakeCert(string subject = "CN=dotnetperf.microsoft.com")
+    {
+        using var rsa = RSA.Create();
+        var req = new CertificateRequest(subject, rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
+        return req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(1));
+    }
+
+    [Fact]
+    public void GetCertificates_ShouldReturnEmpty_WhenDirectoryMissing()
+    {
+        var store = new FileLocalCertStore(_directory);
+
+        var result = store.GetCertificates();
+
+        Assert.Empty(result);
+    }
+
+    [Fact]
+    public void Rotate_ShouldWriteCertificates_ThatCanBeReloaded()
+    {
+        var store = new FileLocalCertStore(_directory);
+        var cert1 = MakeCert();
+        var cert2 = MakeCert();
+        var toAdd = new X509Certificate2Collection { cert1, cert2 };
+
+        store.Rotate(new X509Certificate2Collection(), toAdd);
+
+        var reloaded = store.GetCertificates();
+        Assert.Equal(2, reloaded.Count);
+        var thumbprints = reloaded.Cast<X509Certificate2>().Select(c => c.Thumbprint).ToHashSet();
+        Assert.Contains(cert1.Thumbprint, thumbprints);
+        Assert.Contains(cert2.Thumbprint, thumbprints);
+    }
+
+    [Fact]
+    public void Rotate_ShouldReplaceExistingCertificates()
+    {
+        var store = new FileLocalCertStore(_directory);
+        var oldCert = MakeCert();
+        store.Rotate(new X509Certificate2Collection(), new X509Certificate2Collection { oldCert });
+
+        var newCert = MakeCert();
+        store.Rotate(new X509Certificate2Collection { oldCert }, new X509Certificate2Collection { newCert });
+
+        var reloaded = store.GetCertificates();
+        Assert.Single(reloaded);
+        Assert.Equal(newCert.Thumbprint, reloaded[0].Thumbprint);
+    }
+
+    [Fact]
+    public void Rotate_ReloadedCertificate_ShouldBeExportableWithPrivateKey()
+    {
+        var store = new FileLocalCertStore(_directory);
+        var cert = MakeCert();
+
+        store.Rotate(new X509Certificate2Collection(), new X509Certificate2Collection { cert });
+
+        var reloaded = store.GetCertificates();
+        Assert.Single(reloaded);
+        Assert.True(reloaded[0].HasPrivateKey);
+        // Must be exportable so Program.Main can emit the PFX to stdout without an OS keystore round-trip.
+        var exported = reloaded[0].Export(X509ContentType.Pfx);
+        Assert.NotEmpty(exported);
+    }
+}