Move to TOML configuration for defaults (apple#1425)

- Discussion topic apple#1336.
- This change migrates away from using `UserDefaults`,
  instead providing a TOML configuration mechanism for
  user configurable settings. All existing system property
  settings keys are supported in the new configuration
  file. However, users will have to migrate any settings
  they have configured in the `UserDefaults` into TOML
  for these settings to take effect.
- Breaking changes:
  * `container system property get` is removed in favor of
    users directly utilizing `container system property list --format toml | jq<>`.
  * `container system property set` is removed since the TOML
    configuration is effectively immutable during the lifetime of the
    `container` daemon. Uses can edit the TOML they have in their home
    directory, however no changes will take effect until the daemon is
    restarted via `container system stop && container system start`
* `container system property list --format table` is removed as
    generating tabular format is non-trivial and the new TOML format is
    intended to be human readable

Author: noah-thor

BUILDING.md

@@ -80,10 +80,11 @@ to prepare your build environment.
     >
     > **Note:** If you have already run `swift package edit`, whether intentionally or by accident, follow the steps in the next section to restore the normal `containerization` dependency. Otherwise, the modified `Package.swift` file will not work, and the project may fail to build.
 
-5. If you want `container` to use any changes you made in the `vminit` subproject of Containerization, update the system property to use the locally built init filesystem image:
+5. If you want `container` to use any changes you made in the `vminit` subproject of Containerization, set the init image in your runtime configuration file at `~/.config/container/runtime-config.toml`:
 
-    ```bash
-    container system property set image.init vminit:latest 
+    ```toml
+    [vminit]
+    image = "vminit:latest"
     ```
 
 6. Build `container`.
@@ -101,11 +102,7 @@ to prepare your build environment.
 
 To revert to using the Containerization dependency from your `Package.swift`:
 
-1. If you were using the local init filesystem, revert the system property to its default value:
-
-    ```bash
-    container system property clear image.init
-    ```
+1. If you were using the local init filesystem, remove the `init` override from your `~/.config/container/runtime-config.toml` (or delete the `[vminit]` section if no other image settings are present).
 
 2. Use the Swift package manager to restore the normal `containerization` dependency and update your `Package.resolved` file. If you are using Xcode, revert your `Package.swift` change instead of using `swift package unedit`.
 
@@ -133,14 +130,20 @@ To test changes that require the `container-builder-shim` project:
 
 1. Clone the [container-builder-shim](https://github.com/apple/container-builder-shim) repository and navigate to its directory.
 
-2. After making the necessary changes, build the custom builder image, set it as the active builder image, and remove the existing `buildkit` container so the new image will be used:
+2. After making the necessary changes, build the custom builder image, set it as the active builder image in `~/.config/container/runtime-config.toml`, and remove the existing `buildkit` container so the new image will be used:
 
 ```bash
 container build -t builder .
-container system property set image.builder builder:latest
 container rm -f buildkit
 ```
 
+Add the following to your `~/.config/container/runtime-config.toml`:
+
+```toml
+[build]
+image = "builder:latest"
+```
+
 3. Run the `container` build as usual:
 
 ```bash

Package.resolved

@@ -80,8 +80,10 @@ let package = Package(
                 .product(name: "ContainerizationArchive", package: "containerization"),
                 .product(name: "ContainerizationExtras", package: "containerization"),
                 .product(name: "ContainerizationOS", package: "containerization"),
+                .product(name: "TOML", package: "swift-toml"),
                 "ContainerBuild",
                 "ContainerLog",
+                "ContainerPersistence",
                 "ContainerResource",
                 "Yams",
             ],
@@ -93,6 +95,7 @@ let package = Package(
                 .product(name: "ArgumentParser", package: "swift-argument-parser"),
                 .product(name: "Logging", package: "swift-log"),
                 .product(name: "SwiftProtobuf", package: "swift-protobuf"),
+                .product(name: "TOML", package: "swift-toml"),
                 .product(name: "Containerization", package: "containerization"),
                 .product(name: "ContainerizationOCI", package: "containerization"),
                 .product(name: "ContainerizationOS", package: "containerization"),
@@ -237,6 +240,7 @@ let package = Package(
                 .product(name: "SystemPackage", package: "swift-system"),
                 "ContainerImagesService",
                 "ContainerLog",
+                "ContainerPersistence",
                 "ContainerPlugin",
                 "ContainerVersion",
                 "ContainerXPC",
@@ -285,6 +289,7 @@ let package = Package(
                 "ContainerLog",
                 "ContainerNetworkService",
                 "ContainerNetworkServiceClient",
+                "ContainerPersistence",
                 "ContainerPlugin",
                 "ContainerResource",
                 "ContainerVersion",
@@ -401,6 +406,7 @@ let package = Package(
                 .product(name: "Logging", package: "swift-log"),
                 .product(name: "Containerization", package: "containerization"),
                 .product(name: "SystemPackage", package: "swift-system"),
+                .product(name: "TOML", package: "swift-toml"),
                 "CVersion",
                 "ContainerVersion",
             ]

Package.swift

@@ -50,6 +50,9 @@ extension APIServer {
         var logRoot = LogRoot.path
 
         func run() async throws {
+            let containerSystemConfig: ContainerSystemConfig = try SystemRuntimeOptions.loadConfig(
+                configFile: SystemRuntimeOptions.configFileFromAppRoot(ApplicationRoot.url)
+            )
             let commandName = APIServer._commandName
             let logPath = logRoot.map { $0.appending("\(commandName).log") }
             let log = ServiceLogger.bootstrap(category: "APIServer", debug: debug, logPath: logPath)
@@ -62,15 +65,18 @@ extension APIServer {
                 log.info("configuring XPC server")
                 var routes = [XPCRoute: XPCServer.RouteHandler]()
                 let pluginLoader = try initializePluginLoader(log: log)
+
                 try await initializePlugins(pluginLoader: pluginLoader, log: log, routes: &routes)
                 let containersService = try initializeContainersService(
                     pluginLoader: pluginLoader,
+                    containerSystemConfig: containerSystemConfig,
                     log: log,
                     routes: &routes
                 )
                 let networkService = try await initializeNetworksService(
                     pluginLoader: pluginLoader,
                     containersService: containersService,
+                    containerSystemConfig: containerSystemConfig,
                     log: log,
                     routes: &routes
                 )
@@ -259,12 +265,18 @@ extension APIServer {
             routes[XPCRoute.getDefaultKernel] = harness.getDefaultKernel
         }
 
-        private func initializeContainersService(pluginLoader: PluginLoader, log: Logger, routes: inout [XPCRoute: XPCServer.RouteHandler]) throws -> ContainersService {
+        private func initializeContainersService(
+            pluginLoader: PluginLoader,
+            containerSystemConfig: ContainerSystemConfig,
+            log: Logger,
+            routes: inout [XPCRoute: XPCServer.RouteHandler]
+        ) throws -> ContainersService {
             log.info("initializing containers service")
 
             let service = try ContainersService(
                 appRoot: appRoot,
                 pluginLoader: pluginLoader,
+                containerSystemConfig: containerSystemConfig,
                 log: log,
                 debugHelpers: debug
             )
@@ -292,6 +304,7 @@ extension APIServer {
         private func initializeNetworksService(
             pluginLoader: PluginLoader,
             containersService: ContainersService,
+            containerSystemConfig: ContainerSystemConfig,
             log: Logger,
             routes: inout [XPCRoute: XPCServer.RouteHandler]
         ) async throws -> NetworksService {
@@ -316,8 +329,8 @@ extension APIServer {
                 let config = try NetworkConfiguration(
                     id: NetworkClient.defaultNetworkName,
                     mode: .nat,
-                    ipv4Subnet: try? DefaultsStore.getOptional(key: .defaultSubnet).map { try CIDRv4($0) },
-                    ipv6Subnet: try? DefaultsStore.getOptional(key: .defaultIPv6Subnet).map { try CIDRv6($0) },
+                    ipv4Subnet: containerSystemConfig.network.subnet,
+                    ipv6Subnet: containerSystemConfig.network.subnetv6,
                     labels: try .init([ResourceLabelKeys.role: ResourceRoleValues.builtin]),
                     pluginInfo: NetworkPluginInfo(plugin: "container-network-vmnet")
                 )

Sources/APIServer/APIServer+Start.swift

@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 import ContainerAPIClient
+import ContainerPersistence
 import Containerization
 import ContainerizationOCI
 import Foundation
@@ -27,12 +28,16 @@ struct BuildImageResolver: BuildPipelineHandler {
     let quiet: Bool
     let output: FileHandle
     let pull: Bool
+    let containerSystemConfig: ContainerSystemConfig
 
-    public init(_ contentStore: ContentStore, quiet: Bool = false, output: FileHandle = FileHandle.standardError, pull: Bool = false) throws {
+    public init(_ contentStore: ContentStore, quiet: Bool = false, output: FileHandle = FileHandle.standardError, pull: Bool = false, containerSystemConfig: ContainerSystemConfig)
+        throws
+    {
         self.contentStore = contentStore
         self.quiet = quiet
         self.output = output
         self.pull = pull
+        self.containerSystemConfig = containerSystemConfig
     }
 
     func accept(_ packet: ServerStream) throws -> Bool {
@@ -75,10 +80,10 @@ struct BuildImageResolver: BuildPipelineHandler {
             progress.start()
 
             if self.pull {
-                return try await ClientImage.pull(reference: ref, platform: platform, progressUpdate: progress.handler)
+                return try await ClientImage.pull(reference: ref, platform: platform, containerSystemConfig: containerSystemConfig, progressUpdate: progress.handler)
             }
             // Use fetch() which checks cache first, then pulls if needed
-            return try await ClientImage.fetch(reference: ref, platform: platform, progressUpdate: progress.handler)
+            return try await ClientImage.fetch(reference: ref, platform: platform, containerSystemConfig: containerSystemConfig, progressUpdate: progress.handler)
         }()
 
         let index: Index = try await img.index()

Sources/ContainerBuild/BuildImageResolver.swift

@@ -30,7 +30,13 @@ public actor BuildPipeline {
             [
                 try BuildFSSync(URL(filePath: config.contextDir)),
                 try BuildRemoteContentProxy(config.contentStore),
-                try BuildImageResolver(config.contentStore, quiet: config.quiet, output: config.terminal?.handle ?? FileHandle.standardError, pull: config.pull),
+                try BuildImageResolver(
+                    config.contentStore,
+                    quiet: config.quiet,
+                    output: config.terminal?.handle ?? FileHandle.standardError,
+                    pull: config.pull,
+                    containerSystemConfig: config.containerSystemConfig
+                ),
                 try BuildStdio(quiet: config.quiet, output: config.terminal?.handle ?? FileHandle.standardError),
             ]
     }

Sources/ContainerBuild/BuildPipelineHandler.swift

@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 import ContainerAPIClient
+import ContainerPersistence
 import Containerization
 import ContainerizationOCI
 import ContainerizationOS
@@ -279,6 +280,7 @@ public struct Builder: Sendable {
         public let cacheIn: [String]
         public let cacheOut: [String]
         public let pull: Bool
+        public let containerSystemConfig: ContainerSystemConfig
 
         public init(
             buildID: String,
@@ -298,7 +300,8 @@ public struct Builder: Sendable {
             exports: [BuildExport],
             cacheIn: [String],
             cacheOut: [String],
-            pull: Bool
+            pull: Bool,
+            containerSystemConfig: ContainerSystemConfig
         ) {
             self.buildID = buildID
             self.contentStore = contentStore
@@ -318,6 +321,7 @@ public struct Builder: Sendable {
             self.cacheIn = cacheIn
             self.cacheOut = cacheOut
             self.pull = pull
+            self.containerSystemConfig = containerSystemConfig
         }
     }
 

Sources/ContainerBuild/Builder.swift

@@ -18,6 +18,8 @@ import ArgumentParser
 import ContainerAPIClient
 import ContainerBuild
 import ContainerImagesServiceClient
+import ContainerPersistence
+import ContainerPlugin
 import Containerization
 import ContainerizationError
 import ContainerizationOCI
@@ -147,6 +149,9 @@ extension Application {
         var pull: Bool = false
 
         public func run() async throws {
+            let containerSystemConfig: ContainerSystemConfig = try SystemRuntimeOptions.loadConfig(
+                configFile: SystemRuntimeOptions.configFileFromAppRoot(ApplicationRoot.url)
+            )
             do {
                 let timeout: Duration = .seconds(300)
                 let progressConfig = try ProgressConfig(
@@ -190,7 +195,8 @@ extension Application {
                                     memory: memory,
                                     log: log,
                                     dnsNameservers: dnsNameservers,
-                                    progressUpdate: progress.handler
+                                    progressUpdate: progress.handler,
+                                    containerSystemConfig: containerSystemConfig,
                                 )
 
                                 // wait (seconds) for builder to start listening on vsock
@@ -330,7 +336,9 @@ extension Application {
                         return results
                     }()
                     group.addTask {
-                        [terminal, buildArg, secretsData, contextDir, ignoreFileData, label, noCache, target, quiet, cacheIn, cacheOut, pull, exports, imageNames, tempURL, log] in
+                        [
+                            terminal, buildArg, secretsData, contextDir, ignoreFileData, label, noCache, target, quiet, cacheIn, cacheOut, pull, exports, imageNames, tempURL, log,
+                        ] in
                         let config = Builder.BuildConfig(
                             buildID: buildID,
                             contentStore: RemoteContentStoreClient(),
@@ -349,7 +357,8 @@ extension Application {
                             exports: exports,
                             cacheIn: cacheIn,
                             cacheOut: cacheOut,
-                            pull: pull
+                            pull: pull,
+                            containerSystemConfig: containerSystemConfig,
                         )
                         progress.finish()