From 0fe4cc11cbdf393760d72ec03288086990055477 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:50:04 +0200 Subject: [PATCH 1/5] cwv_by_shopify_theme --- .../reports/report_cwv_by_shopify_theme.js | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 definitions/output/reports/report_cwv_by_shopify_theme.js diff --git a/definitions/output/reports/report_cwv_by_shopify_theme.js b/definitions/output/reports/report_cwv_by_shopify_theme.js new file mode 100644 index 00000000..ddefbb91 --- /dev/null +++ b/definitions/output/reports/report_cwv_by_shopify_theme.js @@ -0,0 +1,202 @@ +const pastMonth = constants.fnPastMonth(constants.currentMonth) + +publish('cwv_by_shopify_theme', { + schema: 'reports', + type: 'table', + tags: ['crux_ready'] +}).preOps(` +CREATE TEMP FUNCTION IS_GOOD(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good / (good + needs_improvement + poor) >= 0.75 +); + +CREATE TEMP FUNCTION IS_POOR(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + poor / (good + needs_improvement + poor) > 0.25 +); + +CREATE TEMP FUNCTION IS_NON_ZERO(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); +`).query(ctx => ` +WITH archive_pages AS ( + -- All Shopify shops in HTTPArchive + SELECT + client, + page AS url, + JSON_VALUE(custom_metrics.ecommerce.Shopify.theme.name) AS theme_name, + JSON_VALUE(custom_metrics.ecommerce.Shopify.theme.theme_store_id) AS theme_store_id + FROM ${ctx.ref('crawl', 'pages')} + WHERE + date = '${pastMonth}' ${constants.devRankFilter} AND + is_root_page AND + JSON_VALUE(custom_metrics.ecommerce.Shopify.theme.name) IS NOT NULL --first grab all shops for market share +) + +SELECT + client, + archive_pages.theme_store_id AS id, + theme_names.theme_name AS top_theme_name, + COUNT(DISTINCT origin) AS origins, + -- Origins with good LCP divided by origins with any LCP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL)) + ) AS pct_good_lcp, + -- Origins with needs improvement are anything not good, nor poor. + 1 - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL)) + ) + - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) + AS pct_ni_lcp, + -- Origins with poor LCP divided by origins with any LCP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL)) + ) AS pct_poor_lcp, + + -- Origins with good TTFB divided by origins with any TTFB. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)) + ) AS pct_good_ttfb, + -- Origins with needs improvement are anything not good, nor poor. + 1 - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)) + ) + - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) + AS pct_ni_ttfb, + -- Origins with poor TTFB divided by origins with any TTFB. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)) + ) AS pct_poor_ttfb, + + -- Origins with good FCP divided by origins with any FCP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL)) + ) AS pct_good_fcp, + -- Origins with needs improvement are anything not good, nor poor. + 1 - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL)) + ) + - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) + AS pct_ni_fcp, + -- Origins with poor FCP divided by origins with any FCP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL)) + ) AS pct_poor_fcp, + + -- Origins with good INP divided by origins with any INP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL)) + ) AS pct_good_inp, + -- Origins with needs improvement are anything not good, nor poor. + 1 - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL)) + ) + - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) + AS pct_ni_inp, + -- Origins with poor INP divided by origins with any INP. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL)) + ) AS pct_poor_inp, + + -- Origins with good CLS divided by origins with any CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL)) + ) AS pct_good_cls, + -- Origins with needs improvement are anything not good, nor poor. + 1 - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL)) + ) + - + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) + AS pct_ni_cls, + -- Origins with poor CLS divided by origins with any CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF(IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF(IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL)) + ) AS pct_poor_cls, + + -- Origins with good LCP, INP (optional), and CLS divided by origins with any LCP and CLS. + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL + )), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL + )) + ) AS pct_good_cwv +FROM ${ctx.ref('chrome-ux-report', 'materialized', 'device_summary')} +JOIN archive_pages +ON + CONCAT(origin, '/') = url AND + IF(device = 'desktop', 'desktop', 'mobile') = client +JOIN ( + -- Add in top theme name for a theme store id AS this should usually be the real theme name + SELECT + COUNT(DISTINCT url) AS pages_count, + theme_store_id, + theme_name, + row_number() OVER (PARTITION BY theme_store_id ORDER BY COUNT(DISTINCT url) DESC) AS rank + FROM archive_pages + GROUP BY + theme_store_id, + theme_name + ORDER BY COUNT(DISTINCT url) DESC +) theme_names +-- Include null theme store ids so that we can get full market share within CrUX +ON IFNULL(theme_names.theme_store_id, 'N/A') = IFNULL(archive_pages.theme_store_id, 'N/A') +WHERE + date = '${pastMonth}' AND + theme_names.rank = 1 +GROUP BY + client, + id, + top_theme_name +ORDER BY + origins DESC +`).postOps(ctx => ` +SELECT + reports.run_export_job( + JSON '''{ + "destination": "storage", + "config": { + "bucket": "${constants.bucket}", + "name": "httparchive/reports/${pastMonth.replaceAll('-', '_')}/cruxShopifyThemes.json" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); +`) From 4c19008f1980dc13ad4e322f0e7f4ee2d1aca32b Mon Sep 17 00:00:00 2001 From: Max Ostapenko Date: Wed, 30 Jul 2025 10:43:57 +0000 Subject: [PATCH 2/5] fix configs --- definitions/output/reports/report_cwv_by_shopify_theme.js | 4 ++-- includes/constants.js | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/definitions/output/reports/report_cwv_by_shopify_theme.js b/definitions/output/reports/report_cwv_by_shopify_theme.js index ddefbb91..7bf29512 100644 --- a/definitions/output/reports/report_cwv_by_shopify_theme.js +++ b/definitions/output/reports/report_cwv_by_shopify_theme.js @@ -191,10 +191,10 @@ ORDER BY SELECT reports.run_export_job( JSON '''{ - "destination": "storage", + "destination": "cloud_storage", "config": { "bucket": "${constants.bucket}", - "name": "httparchive/reports/${pastMonth.replaceAll('-', '_')}/cruxShopifyThemes.json" + "name": "${constants.storagePath}${pastMonth.replaceAll('-', '_')}/cruxShopifyThemes_test.json" }, "query": "SELECT * FROM ${ctx.self()}" }''' diff --git a/includes/constants.js b/includes/constants.js index 7e87c0c1..07417bce 100644 --- a/includes/constants.js +++ b/includes/constants.js @@ -20,6 +20,8 @@ const [ 'AND rank <= 10000' ] : ['', ''] +const bucket = 'httparchive' +const storagePath = 'reports/' class DataformTemplateBuilder { /** @@ -72,5 +74,7 @@ module.exports = { environment, devTABLESAMPLE, devRankFilter, - DataformTemplateBuilder + DataformTemplateBuilder, + bucket, + storagePath } From 6430c189653b2cc8a2944bc930fc86c7c64618cb Mon Sep 17 00:00:00 2001 From: Max Ostapenko Date: Wed, 30 Jul 2025 10:50:23 +0000 Subject: [PATCH 3/5] cleanup --- definitions/output/reports/report_cwv_by_shopify_theme.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/definitions/output/reports/report_cwv_by_shopify_theme.js b/definitions/output/reports/report_cwv_by_shopify_theme.js index 7bf29512..769ca779 100644 --- a/definitions/output/reports/report_cwv_by_shopify_theme.js +++ b/definitions/output/reports/report_cwv_by_shopify_theme.js @@ -194,7 +194,7 @@ SELECT "destination": "cloud_storage", "config": { "bucket": "${constants.bucket}", - "name": "${constants.storagePath}${pastMonth.replaceAll('-', '_')}/cruxShopifyThemes_test.json" + "name": "${constants.storagePath}${pastMonth.replaceAll('-', '_')}/cruxShopifyThemes.json" }, "query": "SELECT * FROM ${ctx.self()}" }''' From 422e04df26bd007c2fff4e6f1528d22769e7f5e8 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 30 Jul 2025 12:51:18 +0200 Subject: [PATCH 4/5] lint --- infra/tf/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/tf/main.tf b/infra/tf/main.tf index 6853f940..c43b03db 100644 --- a/infra/tf/main.tf +++ b/infra/tf/main.tf @@ -63,7 +63,7 @@ module "masthead_agent" { project_id = local.project enable_privatelogviewer_role = false - enable_apis = false + enable_apis = false # Enable only specific modules enable_modules = { From f0bf0d08573ab6271ed5cf663059578cde058fa2 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 30 Jul 2025 20:35:49 +0200 Subject: [PATCH 5/5] add contact --- definitions/output/reports/report_cwv_by_shopify_theme.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/definitions/output/reports/report_cwv_by_shopify_theme.js b/definitions/output/reports/report_cwv_by_shopify_theme.js index 769ca779..5586717a 100644 --- a/definitions/output/reports/report_cwv_by_shopify_theme.js +++ b/definitions/output/reports/report_cwv_by_shopify_theme.js @@ -3,7 +3,9 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth) publish('cwv_by_shopify_theme', { schema: 'reports', type: 'table', - tags: ['crux_ready'] + tags: ['crux_ready'], + description: `Contact: https://github.com/siakaramalegos +Website: https://themevitals.com/themes/` }).preOps(` CREATE TEMP FUNCTION IS_GOOD(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( good / (good + needs_improvement + poor) >= 0.75