Merge pull request #3779 from OpenNeuroOrg/cli/git-annex-avoid-checkout

refactor(cli): Build git-annex branch without a checkout

Author: nellh

.github/workflows/deno.yml

@@ -34,4 +34,5 @@ jobs:
       - uses: codecov/codecov-action@v5
         if: ${{ always() }}
         with:
-          files: coverage.lcov
+          files: cli/coverage.lcov
+          token: ${{ secrets.CODECOV_TOKEN }}

cli/src/worker/commitBuilder.test.ts

@@ -0,0 +1,113 @@
+import { assertEquals, assertExists } from "@std/assert"
+import { CommitBuilder } from "./commitBuilder.ts"
+import { GitWorkerContext } from "./types/git-context.ts"
+import { default as git } from "isomorphic-git"
+
+Deno.test("CommitBuilder", async (t) => {
+  const testDir = await Deno.makeTempDir()
+  const context = new GitWorkerContext(
+    "ds000000",
+    testDir,
+    testDir,
+    "http://localhost",
+    "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJmNjZhNzRjNS05ZDhmLTQ2M2MtOGE2ZS1lYTE3ODljYTNiOTIiLCJlbWFpbCI6Im5lbGxAZGV2LW5lbGwuY29tIiwicHJvdmlkZXIiOiJnb29nbGUiLCJuYW1lIjoiTmVsbCBIYXJkY2FzdGxlIiwiYWRtaW4iOnRydWUsImlhdCI6MTcwMDUyNDIzNCwiZXhwIjoxNzMyMDYwMjM0fQ.5glc_uoxqcRJ4KWn2EvRR0hH-ono2MPJH0wqvcXBIOg",
+  )
+
+  // Initialize a repo for testing
+  await git.init({ ...context.config(), defaultBranch: "main" })
+
+  await t.step("creates a new commit on a new branch", async () => {
+    const builder = new CommitBuilder(context)
+    builder.add("file1.txt", "content1")
+    builder.add("dir/file2.txt", "content2")
+
+    const commitOid = await builder.commit("main", "initial commit")
+    assertExists(commitOid)
+
+    const log = await git.log({ ...context.config(), ref: "main" })
+    assertEquals(log[0].commit.message, "initial commit\n")
+
+    const { blob: b1 } = await git.readBlob({
+      ...context.config(),
+      oid: commitOid!,
+      filepath: "file1.txt",
+    })
+    assertEquals(new TextDecoder().decode(b1), "content1")
+
+    const { blob: b2 } = await git.readBlob({
+      ...context.config(),
+      oid: commitOid!,
+      filepath: "dir/file2.txt",
+    })
+    assertEquals(new TextDecoder().decode(b2), "content2")
+  })
+
+  await t.step("updates existing tree with new commit", async () => {
+    const builder = new CommitBuilder(context)
+    // Update existing file and add a new one
+    builder.add("file1.txt", "updated content")
+    builder.add("new-file.txt", "new content")
+
+    const commitOid = await builder.commit("main", "second commit")
+    assertExists(commitOid)
+
+    const log = await git.log({ ...context.config(), ref: "main" })
+    assertEquals(log[0].commit.message, "second commit\n")
+    assertEquals(log.length, 2)
+
+    const { blob: b1 } = await git.readBlob({
+      ...context.config(),
+      oid: commitOid!,
+      filepath: "file1.txt",
+    })
+    assertEquals(new TextDecoder().decode(b1), "updated content")
+
+    const { blob: b2 } = await git.readBlob({
+      ...context.config(),
+      oid: commitOid!,
+      filepath: "dir/file2.txt",
+    })
+    assertEquals(new TextDecoder().decode(b2), "content2")
+
+    const { blob: b3 } = await git.readBlob({
+      ...context.config(),
+      oid: commitOid!,
+      filepath: "new-file.txt",
+    })
+    assertEquals(new TextDecoder().decode(b3), "new content")
+  })
+
+  await t.step("handles binary data", async () => {
+    const builder = new CommitBuilder(context)
+    const binaryData = new Uint8Array([0, 1, 2, 3])
+    builder.add("binary.bin", binaryData)
+
+    const commitOid = await builder.commit("main", "binary commit")
+    assertExists(commitOid)
+
+    const { blob } = await git.readBlob({
+      ...context.config(),
+      oid: commitOid!,
+      filepath: "binary.bin",
+    })
+    assertEquals(blob, binaryData)
+  })
+
+  await t.step("creates deep directory structures", async () => {
+    const builder = new CommitBuilder(context)
+    builder.add("a/b/c/d/file.txt", "deep")
+
+    const commitOid = await builder.commit("main", "deep commit")
+    assertExists(commitOid)
+
+    const { blob } = await git.readBlob({
+      ...context.config(),
+      oid: commitOid!,
+      filepath: "a/b/c/d/file.txt",
+    })
+    assertEquals(new TextDecoder().decode(blob), "deep")
+  })
+
+  // Clean up
+  await Deno.remove(testDir, { recursive: true })
+})

cli/src/worker/commitBuilder.ts

@@ -0,0 +1,146 @@
+import { default as git } from "isomorphic-git"
+import type { GitWorkerContext } from "./types/git-context.ts"
+
+export class CommitBuilder {
+  private updates: Map<string, Uint8Array> = new Map()
+
+  constructor(private context: GitWorkerContext) {}
+
+  add(path: string, content: string | Uint8Array) {
+    const data = typeof content === "string"
+      ? new TextEncoder().encode(content)
+      : content
+    this.updates.set(path, data)
+  }
+
+  async commit(branch: string, message: string) {
+    const options = this.context.config()
+    let parentCommit
+    let parentTree
+    try {
+      parentCommit = await git.resolveRef({ ...options, ref: branch })
+      const commitObj = await git.readCommit({ ...options, oid: parentCommit })
+      parentTree = commitObj.commit.tree
+    } catch {
+      // Branch might not exist or is empty
+      parentCommit = undefined
+      parentTree = undefined
+    }
+
+    const newTree = await this.buildTree(parentTree, this.updates)
+
+    if (!newTree) {
+      return undefined
+    }
+
+    const now = new Date()
+    const timestamp = Math.floor(now.getTime() / 1000)
+    const timezoneOffset = now.getTimezoneOffset()
+    const author = {
+      ...this.context.author,
+      timestamp,
+      timezoneOffset,
+    }
+
+    const commitOid = await git.writeCommit({
+      ...options,
+      commit: {
+        message,
+        tree: newTree,
+        parent: parentCommit ? [parentCommit] : [],
+        author,
+        committer: author,
+      },
+    })
+
+    await git.writeRef({
+      ...options,
+      ref: `refs/heads/${branch}`,
+      value: commitOid,
+      force: true,
+    })
+
+    return commitOid
+  }
+
+  private async buildTree(
+    baseOid: string | undefined,
+    updates: Map<string, Uint8Array>,
+  ): Promise<string> {
+    const options = this.context.config()
+    let entries: {
+      mode: string
+      path: string
+      oid: string
+      type: "blob" | "tree" | "commit"
+    }[] = []
+    if (baseOid) {
+      const treeResult = await git.readTree({ ...options, oid: baseOid })
+      entries = treeResult.tree
+    }
+
+    // Group updates by current path segment
+    const bySegment = new Map<string, Map<string, Uint8Array>>()
+    const fileUpdates = new Map<string, Uint8Array>()
+
+    for (const [path, content] of updates) {
+      const parts = path.split("/")
+      const segment = parts[0]
+      if (parts.length === 1) {
+        fileUpdates.set(segment, content)
+      } else {
+        if (!bySegment.has(segment)) {
+          bySegment.set(segment, new Map())
+        }
+        bySegment.get(segment)!.set(parts.slice(1).join("/"), content)
+      }
+    }
+
+    // Process subdirectories
+    for (const [segment, childUpdates] of bySegment) {
+      const existingEntryIndex = entries.findIndex((e) => e.path === segment)
+      const existingEntry = existingEntryIndex >= 0
+        ? entries[existingEntryIndex]
+        : undefined
+
+      const childBaseOid = existingEntry && existingEntry.type === "tree"
+        ? existingEntry.oid
+        : undefined
+
+      const newChildOid = await this.buildTree(childBaseOid, childUpdates)
+
+      const newEntry = {
+        mode: "040000",
+        path: segment,
+        oid: newChildOid,
+        type: "tree" as const,
+      }
+
+      if (existingEntryIndex >= 0) {
+        entries[existingEntryIndex] = newEntry
+      } else {
+        entries.push(newEntry)
+      }
+    }
+
+    // Process files
+    for (const [filename, content] of fileUpdates) {
+      const blobOid = await git.writeBlob({ ...options, blob: content })
+      const newEntry = {
+        mode: "100644",
+        path: filename,
+        oid: blobOid,
+        type: "blob" as const,
+      }
+
+      const existingEntryIndex = entries.findIndex((e) => e.path === filename)
+      if (existingEntryIndex >= 0) {
+        entries[existingEntryIndex] = newEntry
+      } else {
+        entries.push(newEntry)
+      }
+    }
+
+    return await git.writeTree({ ...options, tree: entries })
+  }
+}

cli/src/worker/git.test.ts

@@ -128,5 +128,5 @@ LICENSE annex.largefiles=nothing`),
       assertArrayIncludes(expectedFiles, [relativePath])
     }
   }
-  assertEquals(gitObjects, 10)
+  assertEquals(gitObjects, 15)
 })