add mapping docs

README.md

@@ -111,6 +111,7 @@ Push and pull depend on server-side bucket mapping for the requested scope. That
 - [Getting Started](docs/getting-started.md)
 - [Commands Reference](docs/commands.md)
 - [Troubleshooting](docs/troubleshooting.md)
+- [Creating a Syfon Bucket Scope Mapping](docs/syfon-bucket-scope-mapping.md)
 - [Developer Guide](docs/developer-guide.md)
 - [GA4GH DRS Scalability Gaps](docs/ga4gh-drs-scalability-gaps.md)
 

docs/syfon-bucket-scope-mapping.md

@@ -0,0 +1,76 @@
+# Creating a Syfon Bucket Scope Mapping
+
+This fixes upload failures where Syfon can authenticate the request, but cannot
+choose a storage bucket for the requested organization/project.
+
+Example failure:
+
+```text
+failed batch register/upload workflow: upload error: GET https://calypr-dev.ohsu.edu/data/upload/... status 400 body=invalid input: no bucket scope configured for organization "cbds" project "monorepos"
+```
+
+That error means the server is missing a bucket scope mapping for
+`cbds/monorepos`. It is usually steward/admin setup, not a normal end-user file
+problem. `git-drs` can create the mapping if the caller has an admin token for
+the target Syfon server.
+
+## 1. Make Sure the Bucket Credential Exists
+
+If the bucket credential is not already configured on Syfon, add it first:
+
+```bash
+git drs bucket add calypr-dev \
+  --bucket cbds \
+  --region us-east-1 \
+  --access-key "$AWS_ACCESS_KEY_ID" \
+  --secret-key "$AWS_SECRET_ACCESS_KEY" \
+  --s3-endpoint https://s3.amazonaws.com
+```
+
+Use the real bucket name, region, endpoint, and credentials for the environment.
+The remote name, `calypr-dev` above, must resolve to the Syfon endpoint and
+token, or you can pass `--url`, `--token`, or `--cred`.
+
+## 2. Add the Project Mapping
+
+For the error above, create a project-specific mapping:
+
+```bash
+git drs bucket add-project calypr-dev \
+  --organization cbds \
+  --project monorepos \
+  --path s3://cbds/monorepos
+```
+
+`--path` is the storage root Syfon should use for that scope. The bucket is read
+from the URL host (`cbds`), and the remaining path (`monorepos`) is stored as
+the prefix.
+
+If a mapping already exists and needs to be replaced, add `--force`.
+
+## 3. Retry the User Workflow
+
+After the mapping exists, the user can retry:
+
+```bash
+git drs push
+```
+
+No change to the tracked file is required. The upload URL request will include
+`organization=cbds&project=monorepos`, and Syfon should now resolve that scope to
+the configured bucket path.
+
+## When To Use an Organization Mapping
+
+Use an organization mapping when every project in an organization should share a
+common root:
+
+```bash
+git drs bucket add-organization calypr-dev \
+  --organization cbds \
+  --path s3://cbds
+```
+
+Then add project mappings only when a project needs a specific subpath. For most
+one-off upload failures, `add-project` is the least surprising fix.
+

tests/monorepos/Makefile

@@ -4,7 +4,7 @@ GEN := generate-fixtures
 GF_SRC := ./generate-fixtures.go
 OUTDIR := fixtures
 
-.PHONY: test-monorepos build-generate-fixtures prepare-repo generate-fixtures-run finalize clean
+.PHONY: test-monorepos build-generate-fixtures prepare-repo generate-fixtures-run finalize clean generate-large-fixtures-run
 
 test-monorepos: build-generate-fixtures prepare-repo generate-fixtures-run finalize
 	@echo "test-monorepos complete: $(OUTDIR)"
@@ -24,12 +24,17 @@ prepare-repo:
 #	@echo "  OK"
 
 generate-fixtures-run:
-	@echo "Generating fixtures into $(OUTDIR)"
+	@echo "Generating small set of fixtures into $(OUTDIR)"
 	# Provide TCGA list; change or pipe your own list to the binary as needed.
-	# @cd $(OUTDIR);  printf "TARGET-ALL-P2\nTCGA-GBM\nTCGA-HNSC\nTCGA-KICH\nTCGA-KIRC\nTCGA-KIRP\nTCGA-LAML\nTCGA-LGG\nTCGA-LIHC\nTCGA-LUAD\nTCGA-LUSC\nTCGA-MESO\nTCGA-OV\nTCGA-PAAD\nTCGA-PCPG\nTCGA-PRAD\nTCGA-READ\nTCGA-SARC\nTCGA-SKCM\nTCGA-STAD\nTCGA-TGCT\nTCGA-THCA\nTCGA-THYM\nTCGA-UVM\nWCDT-MCRPC\n" | ../$(GEN) --number-of-subdirectories=3 --number-of-files=100 -- > /dev/null
 	@cd $(OUTDIR);  printf "TARGET-ALL-P2\n" | ../$(GEN) --number-of-subdirectories=1 --number-of-files=2 -- > /dev/null
 	@echo "  OK"
 
+generate-large-fixtures-run:
+	@echo "Generating large set of fixtures into $(OUTDIR)"
+	# Provide TCGA list; change or pipe your own list to the binary as needed.
+	@cd $(OUTDIR);  printf "TARGET-ALL-P2\nTCGA-GBM\nTCGA-HNSC\nTCGA-KICH\nTCGA-KIRC\nTCGA-KIRP\nTCGA-LAML\nTCGA-LGG\nTCGA-LIHC\nTCGA-LUAD\nTCGA-LUSC\nTCGA-MESO\nTCGA-OV\nTCGA-PAAD\nTCGA-PCPG\nTCGA-PRAD\nTCGA-READ\nTCGA-SARC\nTCGA-SKCM\nTCGA-STAD\nTCGA-TGCT\nTCGA-THCA\nTCGA-THYM\nTCGA-UVM\nWCDT-MCRPC\n" | ../$(GEN) --number-of-subdirectories=3 --number-of-files=100 -- > /dev/null
+	@echo "  OK"
+
 finalize:
 	@echo "  OK Fixtures created at $(OUTDIR)"
 

tests/monorepos/run-test.sh

@@ -135,31 +135,6 @@ if [ ! -d "fixtures" ]; then
   exit 1
 fi
 
-# ensure git-lfs is installed
-if ! command -v git-lfs >/dev/null 2>&1; then
-  echo "error: git-lfs is not installed; please install it to proceed" >&2
-  # Example install command:
-  if [ "$(uname -s)" = "Darwin" ]; then
-    if ! command -v brew >/dev/null 2>&1; then
-      echo "error: Homebrew is not installed. Please install Homebrew first:" >&2
-      echo "  https://brew.sh" >&2
-      exit 1
-    fi
-    echo "installing git-lfs via Homebrew on macOS" >&2
-    if ! brew install git-lfs; then
-      echo "error: failed to install git-lfs via Homebrew" >&2
-      exit 1
-    fi
-    if ! git lfs install --skip-smudge; then
-      echo "error: failed to initialize git-lfs" >&2
-      exit 1
-    fi
-  else
-    echo "See installation instructions for your platform:" >&2
-    echo "  https://github.com/git-lfs/git-lfs/wiki/Installation" >&2
-    exit 1
-  fi
-fi
 
 # ensure git-drs is running from this project's build
 # run `which git-drs` and check if it's in the build directory
@@ -177,7 +152,7 @@ echo "Using git-drs from: $(which git-drs)" >&2
 
 
 # ensure a gen3 project exists
-calypr_admin projects ls --profile "$PROFILE" | grep "/programs/$PROGRAM/projects/$PROJECT" >/dev/null 2>&1 || {
+calypr-cli auth --profile=$PROFILE --all | grep "/programs/$PROGRAM/projects/$PROJECT" >/dev/null 2>&1 || {
   echo "error: /programs/$PROGRAM/projects/$PROJECT does not exist; please create it first" >&2
   exit 1
 }
@@ -199,7 +174,7 @@ if [ "$CLONE" = "true" ]; then
   cd ../clone
   echo "Cloning remote repository into ../clone" >&2
   # clone into current directory
-  if ! git clone "$GIT_REMOTE" .; then
+  if ! GIT_LFS_SKIP_SMUDGE=1 git clone --branch main "$GIT_REMOTE" .; then
     echo "error: git clone failed" >&2
     exit 1
   fi
@@ -211,8 +186,8 @@ if [ "$CLONE" = "true" ]; then
   fi
   echo "Pulling LFS objects from remote" >&2
   git drs init
-  git drs remote add gen3 "$PROFILE" "$PROGRAM/$PROJECT" --cred "$CREDENTIALS_PATH"
-  git lfs pull origin main
+  git drs remote add gen3 origin "$PROGRAM/$PROJECT" --cred "$CREDENTIALS_PATH"
+  git drs pull origin main
   if grep -q 'https://git-lfs.github.com/spec/v1' ./TARGET-ALL-P2/sub-directory-1/*file-0001.dat; then
     echo "error: LFS pointer resolved and data in `TARGET-ALL-P2/sub-directory-1/file-0001.dat`" >&2
     exit 1
@@ -234,8 +209,8 @@ else
   git remote add origin "$GIT_REMOTE"
 
   # Initialize drs configuration for this repo
-  git drs init -t 16
-  git drs remote add gen3 "$PROFILE" "$PROGRAM/$PROJECT" --cred "$CREDENTIALS_PATH"
+  git drs init -t 16 --enable-data-client-logs
+  git drs remote add gen3 origin "$PROGRAM/$PROJECT" --cred "$CREDENTIALS_PATH"
   # Set multipart-threshold to 10 (MB) for testing purposes
   # Using a smaller threshold to force a multipart upload for testing
   # default is 500 (MB)
@@ -256,8 +231,8 @@ else
   fi
 
   # Ensure enable-data-client-logs is present in git config
-  if ! git config --list | grep -q -- 'enable-data-client-logs'; then
-    echo "error: git config key 'enable-data-client-logs' not found; please set it before running tests" >&2
+  if ! git config --local --get drs.enable-data-client-logs >/dev/null; then
+    echo "error: git config key 'drs.enable-data-client-logs' not found after git drs init" >&2
     exit 1
   fi
 
@@ -303,7 +278,7 @@ for dir in */ ; do
       continue
     fi
     # $dir has trailing slash; don't need trailing slash in track
-    git lfs track "$dir**"
+    git drs track "$dir**"
     git add "$dir"
     git commit -am "Add $dir" 2>&1 | tee commit.log
     cat commit.log >> commit-aggregate.log
@@ -322,10 +297,6 @@ for dir in */ ; do
       echo "# Last 3 lines of .drs/lfs/objects tree:" >> lfs-console.log
       tree .drs/lfs/objects | tail -3 >> lfs-console.log
     fi
-    echo "# git lfs status:" >> lfs-console.log
-    git lfs status >> lfs-console.log
-    echo "# Number of LFS files to be pushed in dry-run:" >> lfs-console.log
-    git lfs push --dry-run origin main | wc -l >> lfs-console.log
     echo "##########################################" >> lfs-console.log
     cat lfs-console.log >> lfs-console-aggregate.log
 
@@ -334,15 +305,15 @@ for dir in */ ; do
     #
 
     # use the first file found in the directory for testing, that will be a single part file
-    target_file=$(find "$dir" -type f -name '*file-0001.dat')
+    target_file=$(find "$dir" -type f -name '*file-0001.dat' | head -n 1)
     # strip double slashes from path if any
     target_file=${target_file//\/\//\/}
     if [ -z "$target_file" ]; then
       echo "error: no files found in $dir to test content/path changes" >&2
       exit 1
     fi
 
-    original_oid=$(git lfs ls-files -l | awk -v path="$target_file" '$0 ~ (" " path "$") {print $1; exit}')
+    original_oid=$(git drs ls-files -l | awk -v path="$target_file" '$0 ~ (" " path "$") {print $1; exit}')
     if [ -z "$original_oid" ]; then
       echo "error: unable to find LFS OID for $target_file" >&2
       exit 1
@@ -353,7 +324,7 @@ for dir in */ ; do
     git add "$target_file"
     git commit -m "Update content for $target_file"
 
-    updated_oid=$(git lfs ls-files -l | awk -v path="$target_file" '$0 ~ (" " path "$") {print $1; exit}')
+    updated_oid=$(git drs ls-files -l | awk -v path="$target_file" '$0 ~ (" " path "$") {print $1; exit}')
     if [ -z "$updated_oid" ]; then
       echo "error: unable to find updated LFS OID for $target_file" >&2
       exit 1
@@ -371,7 +342,7 @@ for dir in */ ; do
     git mv "$target_file" "$renamed_path"
     git commit -m "Rename $target_file to $renamed_path"
 
-    renamed_oid=$(git lfs ls-files -l | awk -v path="$renamed_path" '$0 ~ (" " path "$") {print $1; exit}')
+    renamed_oid=$(git drs ls-files -l | awk -v path="$renamed_path" '$0 ~ (" " path "$") {print $1; exit}')
     if [ -z "$renamed_oid" ]; then
       echo "error: unable to find LFS OID for renamed path $renamed_path" >&2
       exit 1
@@ -380,12 +351,12 @@ for dir in */ ; do
       echo "error: expected same OID after rename for $renamed_path" >&2
       exit 1
     fi
-    if git lfs ls-files -l | grep -Fq " $target_file"; then
+    if git drs ls-files -l | grep -Fq " $target_file"; then
       echo "error: expected old path $target_file to be absent after rename" >&2
       exit 1
     fi
 
     git push origin main 2>&1 | tee -a lfs-console.log
-    break  # uncomment for one directory at a time testing
+    # break  # comment for one directory at a time testing
   fi
 done