main merge

.github/workflows/build.yaml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [ main, master, develop ]
   pull_request:
-    branches: [ main, master, develop ]
   workflow_dispatch:
 
 concurrency:

.github/workflows/pr-checks.yaml

@@ -2,7 +2,6 @@ name: PR Checks
 
 on:
   pull_request:
-    branches: [ main, master, develop ]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/syfon-backend-e2e.yaml

@@ -2,7 +2,6 @@ name: Syfon Backend E2E
 
 on:
   pull_request:
-    branches: [ main, master, develop ]
   workflow_dispatch:
 
 concurrency:
@@ -34,28 +33,25 @@ jobs:
         with:
           go-version-file: git-drs/go.mod
 
-      - name: Resolve Syfon ref from go.mod
+      - name: Resolve Syfon commit from pinned module version
         id: syfon-ref
         working-directory: git-drs
         shell: bash
         run: |
-          version="$(go list -m -f '{{ .Version }}' github.com/calypr/syfon 2>/dev/null || true)"
+          version="$(go list -m -f '{{ .Version }}' github.com/calypr/syfon)"
           if [[ -z "$version" || "$version" == "<nil>" ]]; then
-            version="$(go list -m -f '{{ .Version }}' github.com/calypr/syfon/client)"
+            echo "error: could not resolve pinned syfon version" >&2
+            exit 1
           fi
-          ref="$version"
-          if [[ "$version" =~ -([0-9a-f]{12,})$ ]]; then
-            short_ref="${BASH_REMATCH[1]}"
-            ref="$(git ls-remote https://github.com/calypr/syfon.git "refs/heads/*" "refs/tags/*" |
-              awk -v short="$short_ref" 'index($1, short) == 1 && ref == "" { ref = $1 } END { print ref }')"
-            if [[ -z "$ref" ]]; then
-              echo "Could not resolve Syfon pseudo-version commit $short_ref to a full Git SHA" >&2
-              exit 1
-            fi
+          short_ref="${version##*-}"
+          full_ref="$(git ls-remote https://github.com/calypr/syfon.git | awk -v short="$short_ref" '$1 ~ "^" short && first == "" { first = $1 } END { if (first != "") print first; else exit 1 }')"
+          if [[ -z "$full_ref" ]]; then
+            echo "error: could not resolve full syfon commit for $short_ref" >&2
+            exit 1
           fi
           echo "version=$version" >> "$GITHUB_OUTPUT"
-          echo "ref=$ref" >> "$GITHUB_OUTPUT"
-          echo "Using Syfon module version $version; checking out calypr/syfon ref $ref"
+          echo "ref=$full_ref" >> "$GITHUB_OUTPUT"
+          echo "Using Syfon module version $version; checking out calypr/syfon ref $full_ref"
 
       - name: Check out Syfon
         uses: actions/checkout@v4
@@ -65,13 +61,6 @@ jobs:
           path: syfon
           fetch-depth: 1
 
-      - name: Check out data-client
-        uses: actions/checkout@v4
-        with:
-          repository: calypr/data-client
-          path: data-client
-          fetch-depth: 1
-
       - name: Install test prerequisites
         working-directory: git-drs
         run: |

.github/workflows/test.yaml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [ main, master, develop ]
   pull_request:
-    branches: [ main, master, develop ]
   workflow_dispatch:
 
 concurrency:

Makefile

@@ -48,19 +48,29 @@ lint:
 	@echo "Running go vet..."
 	@go vet ./...
 	@echo "Running gofmt..."
-	@test -z "$$(gofmt -s -l . | tee /dev/stderr)" || (echo "Please run: gofmt -s -w ." && exit 1)
+	@files="$$(gofmt -s -l cmd/ internal/ tests/ git-drs.go)"; \
+	if [ -n "$$files" ]; then \
+		printf "%s\n" "$$files"; \
+		echo "Please run: gofmt -s -w cmd/ internal/ tests/ git-drs.go"; \
+		exit 1; \
+	fi
 	@echo "Running goimports..."
-	@test -z "$$(goimports -l . | tee /dev/stderr)" || (echo "Please run: goimports -w ." && exit 1)
+	@files="$$(goimports -l cmd/ internal/ tests/ git-drs.go)"; \
+	if [ -n "$$files" ]; then \
+		printf "%s\n" "$$files"; \
+		echo "Please run: goimports -w cmd/ internal/ tests/ git-drs.go"; \
+		exit 1; \
+	fi
 	@echo "Running misspell..."
-	@misspell -error .
+	@misspell -error cmd/ internal/ tests/ docs/ *.go *.md Makefile
 	@echo "✅ All lint checks passed!"
 
 # Auto-fix formatting issues
 fmt:
 	@echo "Formatting with gofmt..."
-	@gofmt -s -w .
+	@gofmt -s -w cmd/ internal/ tests/ git-drs.go
 	@echo "Formatting with goimports..."
-	@goimports -w .
+	@goimports -w cmd/ internal/ tests/ git-drs.go
 	@echo "✅ Formatting complete!"
 
 # Run all tests
@@ -117,4 +127,4 @@ full: proto install tidy lint test website webdash
 clean:
 	@rm -rf ./bin ./pkg ./test_tmp ./build ./buildtools
 
-.PHONY: proto proto-lint website docker webdash build debug coverage coverage-clients coverage-html-full
+.PHONY: proto proto-lint website docker webdash build debug coverage coverage-clients coverage-html-full test install

README.md

@@ -3,143 +3,124 @@
 ---
 # NOTICE
 
-git-drs is not yet fully compliant with DRS. It currently works against Gen3 DRS server. Full GA4GH DRS support is expected once v1.6 of the specification has been published.
+`git-drs` is not a pure GA4GH DRS client. It targets Syfon/Gen3-style DRS workflows and uses extensions where repo-scale behavior requires them.
 
 ---
 
 [![Tests](https://github.com/calypr/git-drs/actions/workflows/test.yaml/badge.svg)](https://github.com/calypr/git-drs/actions/workflows/test.yaml)
 
-**Git/DRS orchestration with optional Git LFS compatibility**
+**Git/DRS orchestration with Git-compatible pointer workflows**
 
-Git DRS manages Git-facing DRS workflows: local metadata, Git hooks, filter behavior, lookup/register/push/pull orchestration, and optional Git LFS compatibility. Provider-specific transfer, signed URL behavior, and direct cloud inspection live in client code outside this repo.
+`git-drs` manages:
+
+- remote Gen3/Syfon configuration
+- local DRS metadata
+- pointer-aware push/pull orchestration
+- bucket-scoped object reference workflows
 
 ## Key Features
 
-- **Unified Workflow**: Manage both code and large data files using standard Git commands
-- **DRS Integration**: Built-in support for Gen3 DRS servers
-- **Multi-Remote Support**: Work with development, staging, and production servers in one repository
-- **Automatic Processing**: Files are processed automatically during commits and pushes
-- **Flexible Tracking**: Track individual files, patterns, or entire directories
+- unified Git/data workflow around DRS-backed pointers
+- Gen3/Syfon integration
+- multiple remotes in one repository
+- explicit file tracking and hydration
+- metadata-only reference support for existing bucket objects
 
 ## How It Works
 
-Git DRS works alongside Git LFS when you want LFS-compatible pointers and storage, while still supporting DRS-centric workflows:
+At a high level:
 
-1. **Initialization**: Set up repository and DRS server configuration
-2. **Automatic Commits**: Create DRS objects during pre-commit hooks
-3. **Automatic Pushes**: Register files with DRS servers and upload to configured storage
-4. **On-Demand Downloads**: Pull specific files or patterns as needed
+1. configure a remote for one `organization/project`
+2. let `remote add` bootstrap repo-local `git-drs` state if needed
+3. track file patterns with `git drs track`
+4. add and commit with normal Git
+5. remove tracked pointers with `git drs rm` when you want repository deletion to reconcile with remote DRS state
+6. run `git drs push` for managed metadata registration/upload plus Git push
+7. hydrate pointer files later with `git drs pull`
 
 ## Quick Start
 
-### Installation
-
 ```bash
-# Install Git LFS first
-brew install git-lfs  # macOS
-git lfs install --skip-smudge
-
-# Install Git DRS
-/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/calypr/git-drs/refs/heads/main/install.sh)" -- $GIT_DRS_VERSION
-
-# Install global Git filter configuration for git-drs
 git drs install
-```
-
-### Basic Usage
-
-```bash
-# Initialize repository (one-time Git repo setup)
-git drs init
-
-# Add DRS remote
-git drs remote add gen3 production \
-    --cred /path/to/credentials.json \
-    --url https://calypr-public.ohsu.edu \
-    --organization my-program \
-    --project my-project \
-    --bucket my-bucket
-
-# Required prerequisite (usually steward/admin setup):
-# create bucket credentials, then map org/project to full storage roots before users run push/pull
-git drs bucket add production \
-    --bucket my-bucket \
-    --region us-east-1 \
-    --access-key "$AWS_ACCESS_KEY_ID" \
-    --secret-key "$AWS_SECRET_ACCESS_KEY" \
-    --s3-endpoint https://s3.amazonaws.com
-git drs bucket add-organization production \
-    --organization my-program \
-    --path s3://my-bucket/my-program
-git drs bucket add-project production \
-    --organization my-program \
-    --project my-project \
-    --path s3://my-bucket/my-program/my-project
-
-# Track files
-git lfs track "*.bam"
+git drs remote add gen3 production HTAN_INT/BForePC --cred /path/to/credentials.json
+git drs track "*.bam"
 git add .gitattributes
-
-# Add and commit files
-git add my-file.bam
-git commit -m "Add data file"
-git push
-
-# Download files
-git lfs pull -I "*.bam"
+git add sample.bam
+git commit -m "Add sample"
+git drs push
+git drs ls-files
+git drs pull -I "*.bam"
 ```
 
-## Documentation
-
-For detailed setup and usage information:
+## Current CLI Shape
 
-- **[Getting Started](docs/getting-started.md)** - Repository setup and basic workflows
-- **[Commands Reference](docs/commands.md)** - Complete command documentation
-- **[Installation Guide](docs/installation.md)** - Platform-specific installation
-- **[Troubleshooting](docs/troubleshooting.md)** - Common issues and solutions
-- **[E2E Modes + Local Setup](docs/e2e-modes-and-local-setup.md)** - Local vs remote mode, server config, and end-to-end runbooks
-- **[Cloud/Object Integration](docs/adding-s3-files.md)** - Adding files from provider URLs or configured bucket object keys
-- **[Developer Guide](docs/developer-guide.md)** - Internals and development
+The cleaned CLI intentionally removed legacy commands:
 
-## Supported Servers
+- removed:
+  - `git drs fetch`
+  - `git drs list`
+  - `git drs upload`
+  - `git drs download`
+- `git drs pull` is hydration-only
+- `git drs ls-files` is the local file inventory command
+- `git drs remote add gen3` takes scope as `organization/project`
 
-- **Gen3 Data Commons** (e.g., CALYPR)
+Example:
 
-## Supported Environments
-
-- **Local Development** environments
-- **HPC Systems** (e.g., ARC)
+```bash
+git drs remote add gen3 production HTAN_INT/BForePC --cred /path/to/credentials.json
+```
 
-## Commands Overview
+Current command split:
+
+- `git drs push` is the managed data push path
+- plain `git push` is plain Git only
+- `git drs pull` hydrates tracked pointer files already present in the checkout
+- `git drs ls-files` is the local tracked-file inventory command
+- `git drs add-url` prepares pointer plus local metadata for existing provider objects
+
+## Bucket Mapping Model
+
+End users should not need to know the bucket name.
+
+Push and pull depend on server-side bucket mapping for the requested scope. That mapping is normally provisioned once by a steward/admin using the bucket commands.
+
+## Common Commands
+
+| Command | Description |
+| --- | --- |
+| `git drs install` | Install global `git-drs` filter config |
+| `git drs init` | Explicitly initialize or repair repository-local `git-drs` state |
+| `git drs remote add gen3 [remote] <org/project>` | Add or refresh a Gen3/Syfon remote |
+| `git drs remote list` | List configured remotes |
+| `git drs remote remove <name>` | Remove a configured DRS remote |
+| `git drs remote set <name>` | Set the default remote |
+| `git drs track <pattern>` | Track files or globs |
+| `git drs untrack <pattern>` | Stop tracking files or globs |
+| `git drs rm <path>...` | Remove tracked DRS/LFS files from Git |
+| `git drs ls-files` | List tracked files and localization state |
+| `git drs pull` | Hydrate pointer files in the current checkout |
+| `git drs push` | Register/upload objects, reconcile committed deletes, and push refs |
+| `git drs add-url` | Add an existing provider object by URL or scoped key |
+| `git drs add-ref` | Add a local reference to an existing DRS object |
+| `git drs query` | Query a DRS object by ID |
+| `git drs copy-records` | Copy Syfon records between remotes for one scope |
 
-| Command                | Description                           |
-| ---------------------- | ------------------------------------- |
-| `git drs install`      | Install global git-drs filter config  |
-| `git drs init`         | Initialize repository                 |
-| `git drs remote add`   | Add a DRS remote server               |
-| `git drs remote list`  | List configured remotes               |
-| `git drs remote set`   | Set default remote                    |
-| `git drs add-url`      | Add files via provider URLs or configured bucket object keys |
-| `git lfs track`        | Track file patterns with LFS          |
-| `git lfs ls-files`     | List tracked files                    |
-| `git lfs pull`         | Download tracked files                |
-| `git drs fetch`        | Fetch metadata from DRS server        |
-| `git drs push`         | Push objects to DRS server            |
+## Documentation
 
-Use `--help` with any command for details. See [Commands Reference](docs/commands.md) for complete documentation.
+- [Getting Started](docs/getting-started.md)
+- [Commands Reference](docs/commands.md)
+- [Troubleshooting](docs/troubleshooting.md)
+- [Developer Guide](docs/developer-guide.md)
+- [GA4GH DRS Scalability Gaps](docs/ga4gh-drs-scalability-gaps.md)
 
 ## Requirements
 
-- Git LFS installed and configured
-- Access credentials for your DRS server
-- Go 1.24+ (for building from source)
+- Git
+- access credentials for the target Gen3/Syfon deployment
+- Go 1.26.2+ for local builds
 
 ## Support
 
-- **Issues**: [GitHub Issues](https://github.com/calypr/git-drs/issues)
-- **Releases**: [GitHub Releases](https://github.com/calypr/git-drs/releases)
-- **Documentation**: See `docs/` folder for detailed guides
-
-## License
-
-This project is part of the CALYPR data commons ecosystem.
+- [GitHub Issues](https://github.com/calypr/git-drs/issues)
+- [GitHub Releases](https://github.com/calypr/git-drs/releases)