Support vendoring plain files (#47)

Signed-off-by: Juan Cruz Viotti <jv@jviotti.com>

Author: jviotti

.github/workflows/test.yml

@@ -11,4 +11,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: shellcheck pull bootstrap
+      - run: shellcheck pull bootstrap upgrade

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 # For self-testing purposes
 vendor
 DEPENDENCIES

README.markdown

@@ -120,6 +120,23 @@ from the `DEPENDENCIES` file and running the following command:
 ./vendor/vendorpull/pull vendorpull
 ```
 
+File dependencies
+-----------------
+
+Apart from `git` repositories, you can vendor plain files. In this case, the
+first column defines the file path inside the `vendor` directory, the second
+column defines the URL to download the file from, and the third column defines
+the SHA-256 checksum of the file contents. For example:
+
+```
+oauth-parameters.csv https://www.iana.org/assignments/oauth-parameters/parameters.csv 6ae9ec171be0d232ee4e267a90d3adb301a75ceac74c2eb6478294139ec55d34
+```
+
+Masking and patches do not apply to file dependencies.
+
+Running the `upgrade` command on a file dependency downloads the current
+contents of the URL and updates the checksum accordingly.
+
 Masking
 -------
 

bootstrap

@@ -7,7 +7,7 @@ ROOT="$(git rev-parse --show-toplevel)"
 DEPENDENCIES="$ROOT/DEPENDENCIES"
 URL="https://github.com/sourcemeta/vendorpull"
 
-TMP="$(mktemp -d -t vendorpull-install-XXXXX)"
+TMP="$(mktemp -d -t vendorpull-install-XXXXXX)"
 clean() { rm -rf "$TMP"; }
 trap clean EXIT
 

pull

@@ -17,6 +17,27 @@ log() {
   echo "-- $1" 1>&2
 }
 
+# $1 = version
+is_file_dependency() {
+  echo "$1" | grep -Eq '^[a-f0-9]{64}$'
+}
+
+# $1 = path
+checksum() {
+  if command -v sha256sum > /dev/null
+  then
+    sha256sum "$1" | cut -d ' ' -f 1
+  elif command -v shasum > /dev/null
+  then
+    shasum --algorithm 256 "$1" | cut -d ' ' -f 1
+  elif command -v openssl > /dev/null
+  then
+    openssl dgst -sha256 "$1" | awk '{ print $NF }'
+  else
+    fail "Cannot find sha256sum, shasum, or openssl to compute checksums"
+  fi
+}
+
 # $1 = name
 # $2 = url
 # $3 = version
@@ -77,6 +98,32 @@ vendor() {
   mv "$4/$1" "$OUTPUT"
 }
 
+# $1 = name
+# $2 = url
+# $3 = checksum
+# $4 = tmp
+vendor_file() {
+  # Downloading
+  log "Downloading $2 into $4/$1"
+  mkdir -p "$(dirname "$4/$1")"
+  curl --fail --silent --show-error --location --output "$4/$1" "$2"
+
+  # Verification
+  ACTUAL_CHECKSUM="$(checksum "$4/$1")"
+  if [ "$ACTUAL_CHECKSUM" != "$3" ]
+  then
+    fail "Checksum mismatch for $1: expected $3 but got $ACTUAL_CHECKSUM"
+  fi
+
+  OUTPUT="$VENDOR/$1"
+
+  # Swap
+  log "Moving $4/$1 to $OUTPUT"
+  rm -rf "$OUTPUT"
+  mkdir -p "$(dirname "$OUTPUT")"
+  mv "$4/$1" "$OUTPUT"
+}
+
 if [ ! -f "$DEPENDENCIES" ]
 then
   fail "File not found: $DEPENDENCIES"
@@ -106,14 +153,19 @@ then
     fail "Invalid dependency definition: $DEPENDENCY"
   fi
 
-  TMP="$(mktemp -d -t vendorpull-clone-XXXXX)"
+  TMP="$(mktemp -d -t vendorpull-clone-XXXXXX)"
   log "Setting up temporary directory at $TMP..."
   clean() { rm -rf "$TMP"; }
   trap clean EXIT
 
-  vendor "$NAME" "$URL" "$VERSION" "$TMP"
+  if is_file_dependency "$VERSION"
+  then
+    vendor_file "$NAME" "$URL" "$VERSION" "$TMP"
+  else
+    vendor "$NAME" "$URL" "$VERSION" "$TMP"
+  fi
 else
-  TMP="$(mktemp -d -t vendorpull-clone-XXXXX)"
+  TMP="$(mktemp -d -t vendorpull-clone-XXXXXX)"
   log "Setting up temporary directory at $TMP..."
   clean() { rm -rf "$TMP"; }
   trap clean EXIT
@@ -128,6 +180,11 @@ else
       fail "Invalid dependency definition"
     fi
 
-    vendor "$NAME" "$URL" "$VERSION" "$TMP"
+    if is_file_dependency "$VERSION"
+    then
+      vendor_file "$NAME" "$URL" "$VERSION" "$TMP"
+    else
+      vendor "$NAME" "$URL" "$VERSION" "$TMP"
+    fi
   done < "$DEPENDENCIES"
 fi

upgrade

@@ -15,6 +15,27 @@ log() {
   echo "-- $1" 1>&2
 }
 
+# $1 = version
+is_file_dependency() {
+  echo "$1" | grep -Eq '^[a-f0-9]{64}$'
+}
+
+# $1 = path
+checksum() {
+  if command -v sha256sum > /dev/null
+  then
+    sha256sum "$1" | cut -d ' ' -f 1
+  elif command -v shasum > /dev/null
+  then
+    shasum --algorithm 256 "$1" | cut -d ' ' -f 1
+  elif command -v openssl > /dev/null
+  then
+    openssl dgst -sha256 "$1" | awk '{ print $NF }'
+  else
+    fail "Cannot find sha256sum, shasum, or openssl to compute checksums"
+  fi
+}
+
 if [ ! -f "$DEPENDENCIES" ]
 then
   fail "File not found: $DEPENDENCIES"
@@ -40,22 +61,30 @@ then
   fail "Invalid dependency definition: $DEPENDENCY"
 fi
 
-TMP="$(mktemp -d -t vendorpull-clone-XXXXX)"
+TMP="$(mktemp -d -t vendorpull-clone-XXXXXX)"
 log "Setting up temporary directory at $TMP..."
 clean() { rm -rf "$TMP"; }
 trap clean EXIT
 
-log "Fetching the tip of $URL into $TMP/$NAME"
-git clone --depth 1 --jobs 8 "$URL" "$TMP/$NAME"
+if is_file_dependency "$VERSION"
+then
+  log "Downloading $URL into $TMP/$NAME"
+  mkdir -p "$(dirname "$TMP/$NAME")"
+  curl --fail --silent --show-error --location --output "$TMP/$NAME" "$URL"
+  NEW_VERSION="$(checksum "$TMP/$NAME")"
+else
+  log "Fetching the tip of $URL into $TMP/$NAME"
+  git clone --depth 1 --jobs 8 "$URL" "$TMP/$NAME"
 
-# Try to determine the tag, otherwise the commit hash
-NEW_VERSION="$(git -C "$TMP/$NAME" describe --tags --exact-match HEAD 2>/dev/null \
-  || git -C "$TMP/$NAME" rev-parse HEAD)"
+  # Try to determine the tag, otherwise the commit hash
+  NEW_VERSION="$(git -C "$TMP/$NAME" describe --tags --exact-match HEAD 2>/dev/null \
+    || git -C "$TMP/$NAME" rev-parse HEAD)"
+fi
 
 log "Upgrading $NAME to $NEW_VERSION"
 awk -v name="$NAME" -v version="$NEW_VERSION" \
   '$1 == name {$3 = version} {print}' \
-  DEPENDENCIES > "$TMP/DEPENDENCIES"
+  "$DEPENDENCIES" > "$TMP/DEPENDENCIES"
 
 mv "$TMP/DEPENDENCIES" "$DEPENDENCIES"
 git diff "$DEPENDENCIES"