Moving BQ and GCS update workflow to Cloud Build

CONTRIBUTING.md

@@ -18,7 +18,9 @@ To get started, see the [README](https://github.com/HTTPArchive/wappalyzer/blob/
 Wappalyzer uses [regular expressions](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) to fingerprint technologies. Refer to the [specification](https://github.com/HTTPArchive/wappalyzer/blob/main/README.md#specification) for detail.
 
 - Add a new block to [`src/technologies/*.json`](https://github.com/HTTPArchive/wappalyzer/blob/main/src/technologies). The filename should match the first letter of the technology name (a-z). Use `_.json` if the first character is a number or symbol.
-- Add an icon to [`src/images/icons`](https://github.com/HTTPArchive/wappalyzer/tree/master/src/images/icons). The image must be square, either SVG or PNG (32 x 32 pixels).
+- Add an icon to [`src/images/icons`](https://github.com/HTTPArchive/wappalyzer/tree/master/src/images/icons). The image must be square, either SVG or PNG (32 x 32 pixels). Note that local execution of the icon sync script (`npm run sync_icons`) requires the `rsvg-convert` utility to convert SVG files to PNG:
+  - **macOS**: Install via Homebrew: `brew install librsvg`
+  - **Linux**: Install via apt: `sudo apt-get install librsvg2-bin`
 
 Only widely used technologies are accepted. When creating a pull request, include ten or more links to websites that use the application, a GitHub page with at least 1,000 stars or anything that will help establish the size of the user base.
 

cloudbuild.yaml

@@ -0,0 +1,66 @@
+steps:
+  # Step 1: Detect changed technology files using Git
+  - name: 'gcr.io/cloud-builders/git'
+    id: 'check-changes'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        # Fetch parent history to diff against
+        git fetch origin main --depth=2 || true
+
+        if git rev-parse --verify HEAD^ >/dev/null 2>&1; then
+          files_changed=$(git diff --name-only HEAD^ HEAD -- 'src/technologies/*.json' 'src/categories.json' 'src/groups.json' | wc -l)
+          echo "Detected $files_changed changed technology files"
+          echo "$files_changed" > /workspace/tech_files_changed.txt
+        else
+          echo "Parent commit (HEAD^) not found (shallow clone or first build). Processing all files."
+          echo "1" > /workspace/tech_files_changed.txt
+        fi
+
+  # Step 2: Install dependencies
+  - name: 'node:24'
+    id: 'install-deps'
+    entrypoint: 'npm'
+    args: ['ci']
+
+  # Step 3: Lint and validate JSON/JS files
+  - name: 'node:24'
+    id: 'lint-and-validate'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        npm run lint
+        npm run validate
+
+  # Step 4: Upload technology rules to BigQuery (if changed)
+  - name: 'node:24'
+    id: 'upload-to-bq'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        if [ "$(cat /workspace/tech_files_changed.txt)" != "0" ]; then
+          echo "Technology files changed. Uploading to BigQuery..."
+          npm run tech_upload
+        else
+          echo "No technology files changed. Skipping BigQuery upload."
+        fi
+
+  # Step 5: Incrementally sync icons to Cloud Storage.
+  # Each SVG is converted to PNG via rsvg-convert (128x128) and uploaded with
+  # its source file SHA-256 stored as GCS object metadata. On subsequent runs,
+  # files whose hash hasn't changed are skipped automatically.
+  # Run `npm run sync_icons -- --all` locally to force re-sync everything.
+  - name: 'node:24'
+    id: 'sync-icons'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        apt-get update -qq && apt-get install -y --no-install-recommends librsvg2-bin
+        npm run sync_icons
+
+options:
+  logging: CLOUD_LOGGING_ONLY

package.json

@@ -9,8 +9,6 @@
     "globals": "^17.6.0",
     "jest": "30.4.2",
     "prettier": "3.8.3",
-    "puppeteer": "25.1.0",
-    "terminal-overwrite": "2.0.1",
     "webpagetest": "github:HTTPArchive/WebPageTest.api-nodejs"
   },
   "engines": {
@@ -22,8 +20,9 @@
     "validate": "node ./scripts/validate.js",
     "test": "jest",
     "tech_upload": "node ./scripts/tech_upload.js",
-    "convert": "node ./scripts/convert.js",
-    "build": "npm run lint && npm run validate && npm run convert"
+    "sync_icons": "node ./scripts/sync_icons.js",
+    "sync_icons:all": "node ./scripts/sync_icons.js --all",
+    "build": "npm run lint && npm run validate"
   },
   "overrides": {
     "glob": "^13.0.0",