Moving BQ and GCS update workflow to Cloud Build#319
Merged
Conversation
…tion for GCS and BigQuery Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
WPT test run for https://almanac.httparchive.org/en/2022/WPT test run results: http://webpagetest.httparchive.org/results.php?test=260610_DQ_1 {
"detected": {
"IaaS": "Google Cloud",
"JavaScript libraries": "web-vitals",
"RUM": "web-vitals",
"Performance": "Priority Hints,Google Cloud Trace",
"Security": "HSTS",
"Webmail": "Google Workspace",
"Email": "Google Workspace",
"Analytics": "Google Analytics",
"CDN": "Cloudflare",
"Miscellaneous": "RSS,Open Graph"
},
"detected_apps": {
"Google Cloud": "",
"web-vitals": "",
"Priority Hints": "",
"HSTS": "",
"Google Workspace": "",
"Google Cloud Trace": "",
"Google Analytics": "",
"Cloudflare": "",
"RSS": "",
"Open Graph": ""
},
"detected_technologies": {
"Google Cloud": {
"name": "Google Cloud",
"description": "Google Cloud is a suite of cloud computing services.",
"slug": "google-cloud",
"categories": [
{
"id": 63,
"slug": "iaas",
"description": "Provides computing resources",
"groups": [
7
],
"name": "IaaS",
"priority": 8
}
],
"confidence": 100,
"version": "",
"icon": "Google Cloud.svg",
"website": "https://cloud.google.com",
"pricing": [],
"cpe": "cpe:2.3:a:google:cloud_platform:*:*:*:*:*:*:*:*"
},
"web-vitals": {
"name": "web-vitals",
"description": "The web-vitals JavaScript is a tiny, modular library for measuring all the web vitals metrics on real users.",
"slug": "web-vitals",
"categories": [
{
"id": 59,
"slug": "javascript-libraries",
"description": "Collections of pre-written JavaScript code",
"groups": [
9
],
"name": "JavaScript libraries",
"priority": 9
},
{
"id": 78,
"slug": "rum",
"description": "Tools that track performance as experienced by users",
"groups": [
2
],
"name": "RUM",
"priority": 9
}
],
"confidence": 100,
"version": "",
"icon": "web-vitals.svg",
"website": "https://github.com/GoogleChrome/web-vitals",
"pricing": [],
"cpe": null
},
"Priority Hints": {
"name": "Priority Hints",
"description": "Priority Hints exposes a mechanism for developers to signal a relative priority for browsers to consider when fetching resources.",
"slug": "priority-hints",
"categories": [
{
"id": 92,
"slug": "performance",
"description": "Tools that measure and optimize site speed",
"groups": [
7
],
"name": "Performance",
"priority": 9
}
],
"confidence": 100,
"version": "",
"icon": "Priority Hints.svg",
"website": "https://wicg.github.io/priority-hints/",
"pricing": [],
"cpe": null
},
"HSTS": {
"name": "HSTS",
"description": "HTTP Strict Transport Security (HSTS) informs browsers that the site should only be accessed using HTTPS.",
"slug": "hsts",
"categories": [
{
"id": 16,
"slug": "security",
"description": "Technologies that protect websites from vulnerabilities and attacks",
"groups": [
11
],
"name": "Security",
"priority": 9
}
],
"confidence": 100,
"version": "",
"icon": "default.svg",
"website": "https://www.rfc-editor.org/rfc/rfc6797#section-6.1",
"pricing": [],
"cpe": null
},
"Google Workspace": {
"name": "Google Workspace",
"description": "Google Workspace, formerly G Suite, is a collection of cloud computing, productivity and collaboration tools.",
"slug": "google-workspace",
"categories": [
{
"id": 30,
"slug": "webmail",
"description": "Systems that allow users to send and receive emails through a browser",
"groups": [
4
],
"name": "Webmail",
"priority": 2
},
{
"id": 75,
"slug": "email",
"description": "Systems that manage email communication",
"groups": [
4,
2
],
"name": "Email",
"priority": 9
}
],
"confidence": 100,
"version": "",
"icon": "Google.svg",
"website": "https://workspace.google.com/",
"pricing": [],
"cpe": null
},
"Google Cloud Trace": {
"name": "Google Cloud Trace",
"description": "Google Cloud Trace is a distributed tracing system that collects latency data from applications and displays it in the Google Cloud Console.",
"slug": "google-cloud-trace",
"categories": [
{
"id": 92,
"slug": "performance",
"description": "Tools that measure and optimize site speed",
"groups": [
7
],
"name": "Performance",
"priority": 9
}
],
"confidence": 100,
"version": "",
"icon": "google-cloud-trace.svg",
"website": "https://cloud.google.com/trace",
"pricing": [],
"cpe": null
},
"Google Analytics": {
"name": "Google Analytics",
"description": "Google Analytics is a free web analytics service that tracks and reports website traffic.",
"slug": "google-analytics",
"categories": [
{
"id": 10,
"slug": "analytics",
"description": "Tools that track user behavior and provide insights into website performance",
"groups": [
8
],
"name": "Analytics",
"priority": 9
}
],
"confidence": 100,
"version": "",
"icon": "Google Analytics.svg",
"website": "https://google.com/analytics",
"pricing": [],
"cpe": null
},
"Cloudflare": {
"name": "Cloudflare",
"description": "Cloudflare is a web-infrastructure and website-security company, providing content-delivery-network services, DDoS mitigation, Internet security, and distributed domain-name-server services.",
"slug": "cloudflare",
"categories": [
{
"id": 31,
"slug": "cdn",
"description": "(Content Delivery Network) Distribute website content globally to improve load times for users",
"groups": [
7
],
"name": "CDN",
"priority": 9
}
],
"confidence": 100,
"version": "",
"icon": "CloudFlare.svg",
"website": "https://www.cloudflare.com",
"pricing": [],
"cpe": null
},
"RSS": {
"name": "RSS",
"description": "RSS is a family of web feed formats used to publish frequently updated works—such as blog entries, news headlines, audio, and video—in a standardized format.",
"slug": "rss",
"categories": [
{
"id": 19,
"slug": "miscellaneous",
"description": "Tools and technologies that don't fit into other categories",
"groups": [
6
],
"name": "Miscellaneous",
"priority": 10
}
],
"confidence": 100,
"version": "",
"icon": "RSS.svg",
"website": "https://www.rssboard.org/rss-specification",
"pricing": [],
"cpe": null
},
"Open Graph": {
"name": "Open Graph",
"description": "Open Graph is a protocol that is used to integrate any web page into the social graph.",
"slug": "open-graph",
"categories": [
{
"id": 19,
"slug": "miscellaneous",
"description": "Tools and technologies that don't fit into other categories",
"groups": [
6
],
"name": "Miscellaneous",
"priority": 10
}
],
"confidence": 100,
"version": "",
"icon": "Open Graph.png",
"website": "https://ogp.me",
"pricing": [],
"cpe": null
}
}
} |
…nd add progress reporting and safety aborts to icon sync script Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
tunetheweb
approved these changes
Jun 12, 2026
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Initiated this migration as can't limit the secrets to environments in GitHub anymore.
Added a
cloudbuild.yamlpipeline that detects changes in technology files, installs dependencies, lints and validates code, conditionally uploads technology rules to BigQuery, and incrementally syncs icons to GCS, converting SVGs to PNGs as needed. The pipeline skips unnecessary uploads by checking file hashes.Introduced
scripts/sync_icons.js, a script that computes SHA-256 hashes for each icon (SVG/PNG), converts SVGs to 128x128 PNGs, uploads them to GCS with hash-based metadata, and skips unchanged files unless forced. This ensures efficient, incremental icon syncing.Removed the unused
convertscript and its invocation from the build process, and replaced it with the newsync_iconsandsync_icons:allscripts inpackage.json.Removed unused dependencies (
puppeteer,terminal-overwrite) frompackage.json.Simplified BigQuery client initialization in
scripts/tech_upload.jsby removing the explicit credentials parameter.