R2 artifact registry: host + install native CLI binaries (install.json/install.sh, deps, presign)

cmd/publish-server/main.go

@@ -32,7 +32,9 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"regexp"
 	"strings"
+	"time"
 
 	"github.com/pilot-protocol/app-template/internal/publish"
 )
@@ -49,6 +51,8 @@ type server struct {
 	adminToken string
 	origins    []string
 	registrar  publish.BrokerRegistrar // registers managed apps with the broker on approval
+	r2         *publish.R2             // artifact registry (nil = uploads disabled)
+	selfBase   string                  // public base URL of THIS server, for proxy artifact URLs
 }
 
 func main() {
@@ -79,7 +83,14 @@ func main() {
 		adminToken: os.Getenv("ADMIN_TOKEN"),
 		// CORS: only the production website may call the API. ALLOWED_ORIGINS
 		// overrides (e.g. add a local origin for testing); default is prod.
-		origins: splitOrigins(allowedOriginsEnv()),
+		origins:  splitOrigins(allowedOriginsEnv()),
+		r2:       publish.R2FromEnv(),
+		selfBase: strings.TrimRight(os.Getenv("PUBLISH_SELF_URL"), "/"),
+	}
+	if s.r2 != nil {
+		log.Printf("artifact registry: R2 bucket %q (public base %q)", s.r2.Bucket, s.r2.PublicBase)
+	} else {
+		log.Printf("artifact registry: disabled (set R2_ENDPOINT/R2_BUCKET + AWS keys to enable uploads)")
 	}
 	// Managed-app approval registers the app with the broker by writing its
 	// registry file (BROKER_REGISTRY). Unset = managed registration is logged
@@ -100,6 +111,10 @@ func main() {
 	})
 	mux.HandleFunc("/api/preview", s.cors(s.apiPreview))
 	mux.HandleFunc("/api/submit", s.cors(s.apiSubmit))
+	mux.HandleFunc("/api/artifact/presign", s.cors(s.apiArtifactPresign))
+	// Signing proxy: install-time GET of an artifact when no public domain is set.
+	// Unauthenticated by design (the daemon fetches it); R2 holds the real bytes.
+	mux.HandleFunc("GET /artifact/", s.artifactProxy)
 	// Self-contained admin assets (embedded). The dashboard depends on nothing
 	// from the website — its CSS ships in this binary and is served from here.
 	mux.Handle("GET /static/", http.FileServer(http.FS(assets)))
@@ -210,6 +225,91 @@ func (s *server) apiSubmit(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, 202, map[string]any{"case_id": c.CaseID, "status": c.Status})
 }
 
+// ── artifact registry (R2) ────────────────────────────────────────────────────
+
+// presignReq is the website Artifacts step's request for a direct-to-R2 upload
+// slot: it identifies the app + target platform + filename, and gets back a
+// short-lived PUT URL plus the stable public URL to record in the submission.
+type presignReq struct {
+	ID       string `json:"id"`
+	Version  string `json:"version"`
+	OS       string `json:"os"`
+	Arch     string `json:"arch"`
+	Filename string `json:"filename"`
+}
+
+var (
+	reArtifactID   = regexp.MustCompile(`^io\.pilot\.[a-z0-9]([a-z0-9-]*[a-z0-9])?$`)
+	reArtifactVer  = regexp.MustCompile(`^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$`)
+	reArtifactFile = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
+	okArtifactOS   = map[string]bool{"linux": true, "darwin": true}
+	okArtifactArch = map[string]bool{"amd64": true, "arm64": true}
+)
+
+func (s *server) apiArtifactPresign(w http.ResponseWriter, r *http.Request) {
+	if s.r2 == nil {
+		writeJSON(w, 503, map[string]any{"error": "artifact uploads are not configured on this server"})
+		return
+	}
+	var req presignReq
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeJSON(w, 400, map[string]any{"error": "bad json: " + err.Error()})
+		return
+	}
+	var errs []string
+	if !reArtifactID.MatchString(req.ID) {
+		errs = append(errs, "id must be io.pilot.<name>")
+	}
+	if !reArtifactVer.MatchString(req.Version) {
+		errs = append(errs, "version must be semver")
+	}
+	if !okArtifactOS[req.OS] {
+		errs = append(errs, "os must be linux or darwin")
+	}
+	if !okArtifactArch[req.Arch] {
+		errs = append(errs, "arch must be amd64 or arm64")
+	}
+	if !reArtifactFile.MatchString(req.Filename) {
+		errs = append(errs, "filename must be a plain name (letters, digits, . _ -)")
+	}
+	if len(errs) > 0 {
+		writeJSON(w, 422, map[string]any{"errors": errs})
+		return
+	}
+	key := publish.ArtifactKey(req.ID, req.Version, req.OS, req.Arch, req.Filename)
+	putURL, err := s.r2.PresignPut(key, 15*time.Minute, time.Now())
+	if err != nil {
+		writeJSON(w, 500, map[string]any{"error": "presign: " + err.Error()})
+		return
+	}
+	writeJSON(w, 200, map[string]any{
+		"key":        key,
+		"put_url":    putURL,
+		"public_url": s.r2.PublicURL(key, s.selfBase),
+		"expires_in": 900,
+	})
+}
+
+// artifactProxy 302-redirects an install-time GET to a fresh presigned R2 GET, so
+// installs work off a stable URL even when the bucket has no public domain.
+func (s *server) artifactProxy(w http.ResponseWriter, r *http.Request) {
+	if s.r2 == nil {
+		http.Error(w, "artifact registry not configured", http.StatusServiceUnavailable)
+		return
+	}
+	key := strings.TrimPrefix(r.URL.Path, "/artifact/")
+	if key == "" || strings.Contains(key, "..") {
+		http.Error(w, "bad key", http.StatusBadRequest)
+		return
+	}
+	getURL, err := s.r2.PresignGet(key, 10*time.Minute, time.Now())
+	if err != nil {
+		http.Error(w, "presign: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+	http.Redirect(w, r, getURL, http.StatusFound)
+}
+
 // adminBuild kicks off the async bundle build for a submitted (or previously
 // failed) case. Admin-token gated, same as approve/reject. The build runs in a
 // background goroutine; the case flips submitted/build_failed → building →

docs/NATIVE-APPS.md

@@ -1,6 +1,16 @@
 # Native (binary-delivery) apps — design
 
-> Status: DESIGN + TODO. Native/CLI apps are **Coming soon** — blocked at the
+> **SUPERSEDED (2026-06-22) for the delivery model.** This doc proposed delivering
+> native binaries *by reference* (customer-hosted URL, "we never store the bytes").
+> The shipped implementation instead **hosts the bytes in a Pilot-run Cloudflare
+> R2 artifact registry**: the publisher uploads per-OS/arch binaries in the
+> publish form's Artifacts step, and the generated cli adapter fetches + verifies
+> + stages + execs them at install (with install order + optional args). See
+> **`docs/R2-ARTIFACT-REGISTRY.md`** for the canonical, implemented design. The
+> `assets[]` schema and the daemon-side staging notes below remain useful
+> background, but where they disagree with R2-ARTIFACT-REGISTRY.md, that doc wins.
+
+> Status (original): DESIGN + TODO. Native/CLI apps are **Coming soon** — blocked at the
 > wizard's type step; only HTTP (translation-only) apps ship today. Decision
 > (2026-06-17): native apps deliver the real binary via a **customer-hosted URL +
 > per-OS/arch sha256**, pinned in the signed manifest and **fetched + verified +

docs/PUBLISHING-SMOLMACHINES.md

@@ -0,0 +1,104 @@
+# Publishing Smol Machines (io.pilot.smolmachines) to the Pilot app store
+
+End-to-end runbook: build every platform artifact, host them in the R2 registry,
+produce the catalogue entry, and land it. The app is a passthrough cli adapter
+over the `smolvm` binary (no enumerated methods).
+
+## 0. Identity & descriptions (fixed)
+
+- **id:** `io.pilot.smolmachines`  ·  **version:** `1.2.0`  ·  **namespace/method prefix:** `smolmachines`
+- **command:** `smolvm`  ·  **method:** `smolmachines.exec` (passthrough) + auto `smolmachines.help`
+- **short** (catalogue / `appstore list`): the one-liner.
+- **long** (`appstore view` → metadata `description_md`): the full bullet description.
+- See `submissions/io.pilot.smolmachines/submission.json` for the exact text.
+
+## 1. Build all platform artifacts (the binaries the publisher uploads)
+
+smolvm releases per-platform tarballs. Re-host them in the registry under the app id:
+
+```bash
+for p in "darwin arm64 arm64" "linux arm64 arm64" "linux amd64 x86_64"; do
+  set -- $p; OS=$1 PARCH=$2 SARCH=$3
+  T=smolvm-1.2.0-$OS-$SARCH.tar.gz
+  gh release download v1.2.0 --repo smol-machines/smolvm --pattern "$T" --clobber
+  aws s3 cp "$T" "s3://pilot-artifacts-prod/io.pilot.smolmachines/1.2.0/$OS-$PARCH/$T" \
+    --endpoint-url=https://<ACCOUNT_ID>.r2.cloudflarestorage.com
+done
+```
+Record each `sha256` (computed in the browser at upload time, or `shasum -a 256`).
+The artifact for each platform is the **tar.gz** with `unpack: tar.gz`,
+`exec_path: smolvm-1.2.0-<os>-<arch>/smolvm`, `order: 1`.
+
+> Note: smolvm needs the whole tarball (wrapper + `smolvm-bin` + `lib/` + sparse VM
+> images), not just one file — hence `unpack: tar.gz`.
+
+## 2. Submit + build the adapter bundle
+
+POST the submission to the publish-server (or drive the website Artifacts step):
+
+```bash
+curl -X POST $PUBLISH_API/api/submit -H 'Content-Type: application/json' \
+  --data @submissions/io.pilot.smolmachines/submission.json   # → {case_id, status:submitted}
+# admin builds (per platform): scaffold adapter → sign manifest → emit install.json/install.sh → self-verify
+```
+`BuildBundle` cross-compiles the adapter for all four targets and **self-verifies each
+through the catalogue gate** — a green build IS the §7.1 preflight passing.
+
+Each bundle contains: signed `manifest.json` (`proc.exec→smolvm`, `net.dial <r2-host>`,
+`fs.write $APP`, `fs.read $APP/install.json`, `protection: guarded`), `bin/smolvm-app`,
+`install.json` (prod R2 URLs + shas), `install.sh`.
+
+## 3. Host the bundles + metadata, build the catalogue entry
+
+```bash
+# bundles
+for PLAT in darwin-arm64 darwin-amd64 linux-arm64 linux-amd64; do
+  aws s3 cp io.pilot.smolmachines-1.2.0-$PLAT.tar.gz \
+    s3://pilot-artifacts-prod/bundles/io.pilot.smolmachines/1.2.0/ --endpoint-url=$EP
+done
+# rich metadata.json (carries the LONG description_md)
+aws s3 cp metadata.json s3://pilot-artifacts-prod/catalogue/apps/io.pilot.smolmachines/metadata.json --endpoint-url=$EP
+```
+
+Catalogue v2 entry (the line that lands in the platform catalogue):
+```json
+{ "id":"io.pilot.smolmachines", "version":"1.2.0",
+  "description":"<SHORT one-liner>",
+  "display_name":"Smol Machines", "vendor":"smol machines", "license":"Apache-2.0",
+  "source_url":"https://github.com/smol-machines/smolvm",
+  "bundle_url":"<prod R2>/bundles/.../io.pilot.smolmachines-1.2.0-linux-amd64.tar.gz",
+  "bundle_sha256":"<linux/amd64 tarball sha>",
+  "bundles": { "darwin/arm64":{...}, "darwin/amd64":{...}, "linux/arm64":{...}, "linux/amd64":{...} },
+  "metadata_url":"<prod R2>/catalogue/apps/io.pilot.smolmachines/metadata.json",
+  "metadata_sha256":"<metadata.json sha>" }
+```
+- `description` (short) → `appstore list`.  `metadata.description_md` (long) → `appstore view`.
+
+## 4. Sign + land the catalogue entry
+
+The catalogue is signature-gated; pilotctl verifies `<catalogue>.sig` against the
+**embedded release catalogue key**. In production this is done by the publish
+automation (app-template#28 auto-signs with the `CATALOG_SIGN_KEY` CI secret) when
+you **Approve** the case — it opens the one-line catalogue PR on the platform repo
+(`TeoSlayer/pilotprotocol` → `catalogue/catalogue.json`). Merge that PR and hosts
+pick it up on next `pilotctl appstore catalogue`.
+
+Manual/local signing (testing only) requires a pilotctl built with your key:
+`pilotctl appstore sign-catalogue --key <key> catalogue.json`.
+
+## 5. Install + verify on a host
+
+```bash
+pilotctl appstore catalogue | grep smolmachines           # short description shows here
+pilotctl appstore view io.pilot.smolmachines              # long description_md shows here
+pilotctl appstore install io.pilot.smolmachines           # fetch+verify+stage from R2
+pilotctl appstore call io.pilot.smolmachines smolmachines.exec \
+  '{"args":["machine","run","--net","--image","alpine","--","echo","hi"]}'
+```
+
+## Prerequisites (must be deployed first — see R2-PREDEPLOY-REPORT.md)
+
+1. Daemon on the **proc.exec** app-store version (pilotprotocol#317 → app-store#24).
+2. pilotctl carries `install.json`/`install.sh` on install + daemon wires
+   `TrustedPublishers` (pilotprotocol#318). Without #2 the trust anchor rejects every app.
+3. R2 bucket **CORS** for browser uploads; publish-server R2 env set.

docs/R2-ARTIFACT-REGISTRY.md

@@ -0,0 +1,115 @@
+# R2 Artifact Registry — native binary delivery for cli apps
+
+> Status: IMPLEMENTED (RC). Lets the Pilot app store **host** publisher-supplied,
+> platform-specific, versioned, signed binaries in Cloudflare R2 and install them
+> — in a declared order, with optional install args — via the generated cli
+> adapter. Builds on the cli-app support (proc.exec + CLI adapter) from
+> app-store#24 / app-template#31. **Supersedes** the "deliver by reference, never
+> store the bytes" stance in `NATIVE-APPS.md`: we now store the bytes in R2.
+
+## Why
+
+`NATIVE-APPS.md` / `CLI-ADAPTER.md` shipped the *translation* half (a cli adapter
+that execs a local command under `proc.exec`) but assumed the binary was already
+on the host. Delivering it is the point of a store. This adds the *delivery*
+half: the publisher uploads per-OS/arch binaries to a Pilot-run R2 registry, and
+the adapter fetches + verifies + stages them at install.
+
+## The flow
+
+```
+PUBLISH FORM (Artifacts step)            BUILD (publish-api)                 INSTALL (host)
+upload binaries → R2          ─────▶  generate adapter + install.json  ─▶  adapter staging (stage.go)
+set install order + args              fold into the bundle tarball          fetch R2 → verify sha → stage
+                                      (sha-pinned in the catalogue)         → run install args (order)
+                                                                            → exec the staged command
+```
+
+1. **Artifacts step** (publish form, website). The publisher uploads each
+   platform binary (or `.tar.gz`) to the R2 registry and sets, per artifact:
+   target `os`/`arch`, `exec_path`, install `order`, optional install `args`, and
+   `unpack` for archives. The form submits a JSON `Submission` carrying
+   `artifacts[]` (R2 url + sha256 + order + args — never the bytes).
+2. **Submit** (`POST /api/submit`). `Submission.Validate()` checks the artifacts
+   (cli-only, known os/arch, https URL, 64-hex sha, relative `exec_path` under
+   `$APP`, per-platform-unique order). The sha is the integrity anchor.
+3. **Build** (`/admin/build` → `BuildBundle`). In addition to the signed adapter,
+   the build emits **`install.json`** (the staging spec, from `cfg.Assets`) into
+   every platform tarball, and the manifest gains the delivery grants
+   (`proc.exec`, `fs.write $APP`, `net.dial <r2-host>`). The whole tarball is
+   sha-pinned in the catalogue, so `install.json` (and the expected asset shas)
+   can't be altered undetected.
+4. **Install + call** (host). The generated cli adapter calls `StageAssets($APP)`
+   on first spawn (`internal/backend/stage.go`): read `install.json` → select the
+   asset(s) for `runtime.GOOS/GOARCH` → in ascending `order`, fetch from R2,
+   verify sha256, stage under `$APP` (single file, or `tar.gz` extracted via the
+   host `tar`), run any install `args` — then exec the staged `exec_path` per call.
+
+## R2 layout
+
+```
+s3://pilot-artifacts-{dev,prod}/<app-id>/<app-version>/<os>-<arch>/<filename>
+  io.pilot.smolvm/1.2.0/darwin-arm64/smolvm-1.2.0-darwin-arm64.tar.gz
+```
+Write-once (a new app version = a new prefix). Buckets `pilot-artifacts-dev` and
+`pilot-artifacts-prod` exist on the Pilot R2 account. **Public read** is served by
+an r2.dev managed URL (dev: `https://pub-2328865fa11041b8a5efba00b940ec14.r2.dev`);
+production should attach a custom domain (e.g. `artifacts.pilotprotocol.network`).
+Generated install scripts reference the public base URL.
+
+## Schema
+
+`pilot.app.yaml` / `scaffold.Config` gains `assets[]` (see `example.pilot.app.yaml`);
+the publish `Submission` gains `artifacts[]`. Both map to:
+
+| field | meaning |
+|---|---|
+| `role` | `binary` (chmod +x, default) \| `data` |
+| `os` / `arch` | host match: `linux`/`darwin`, `amd64`/`arm64` |
+| `url` | https R2 public URL of the artifact |
+| `sha256` | 64-hex of the uploaded object; verified after download |
+| `unpack` | `""` (single file) \| `tar.gz` (extract under `$APP`) |
+| `exec_path` | dest under `$APP`, or the path inside the extracted tree |
+| `order` | ascending install sequence (unique per platform) |
+| `args` | optional post-stage invocation (e.g. a one-time setup) |
+
+## Integrity & security
+
+- **sha256** on every asset, checked after download; mismatch refuses to install.
+- The **bundle tarball is sha-pinned** in the catalogue, so `install.json` is
+  tamper-evident transitively (no app-store manifest-schema change needed).
+- **`proc.exec`** (app-store#24) authorizes the exec; **`fs.write $APP`** and
+  **`net.dial <r2-host>`** authorize staging. cli apps ship `protection: guarded`.
+- Archive extraction uses the host `tar` (handles GNU/sparse artifacts Go's
+  `archive/tar` rejects) **after** a name-scan that rejects absolute paths and
+  `..` traversal (zip-slip defence).
+
+## E2E
+
+`scripts/e2e-smolvm.sh` + `internal/scaffold/r2_e2e_test.go` (`TestR2AssetDeliveryE2E`):
+download smolvm (`smol-machines/smolvm`, a real multi-file microVM CLI: wrapper +
+binary + libs + sparse disk images) for the host, upload it to `pilot-artifacts-dev`,
+then build the generated adapter and let it fetch+verify+extract from R2 and exec
+it — asserting `smolvm --version → "smolvm 1.2.0"`. The Go test is env-gated
+(`PILOT_E2E_ASSET_URL/_SHA256/_EXECPATH/...`) so CI needs no live bucket; the
+script wires it up against the real registry.
+
+## Build / repo coordination
+
+| Repo | Role |
+|---|---|
+| **app-template** (this) | schema, build-time `install.json` gen, staging runtime, manifest grants, e2e — the bulk |
+| **app-store** #24 | `proc.exec` capability (reused as the exec permission) |
+| **pilotprotocol** #317 | daemon dep bump so it accepts `proc.exec` |
+| website #44 | publish wizard cli path; **TODO**: add the Artifacts step (uploads + order/args) as a thin client over a presign endpoint |
+
+## Follow-ups
+
+- **Presign upload endpoint** (`POST /api/artifact/presign`) + a signing-proxy
+  `GET /artifact/...` so the form uploads straight to R2 and installs can run off
+  a stable proxy URL where a public domain isn't configured. (The e2e uploads via
+  the S3 API directly.)
+- **Server-side re-verify** of each artifact sha against the stored R2 object at
+  submit time.
+- Production **custom domain** for `pilot-artifacts-prod` (needs a Cloudflare API
+  token with R2 + DNS scope; the S3 keys can't enable public access).