Skip to content

Commit 41a5d19

Browse files
themaroltthemarolt
authored
fix: bq dataset ingest worker didnt start because of temporal lib issue (CM-1227) (#4208)
Signed-off-by: Uroš Marolt <uros@marolt.me>
1 parent 5f2a65b commit 41a5d19

8 files changed

Lines changed: 149 additions & 38 deletions

File tree

‎pnpm-lock.yaml‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎services/apps/packages_worker/package.json‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
"name": "@crowd/packages-worker",
33
"private": true,
44
"scripts": {
5-
"start:criticality-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker tsx src/bin/criticality-worker.ts",
6-
"dev:criticality-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9237 src/bin/criticality-worker.ts",
7-
"dev:criticality-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9237 src/bin/criticality-worker.ts",
5+
"start:criticality-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker SERVICE=criticality-worker tsx src/bin/criticality-worker.ts",
6+
"dev:criticality-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker SERVICE=criticality-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9237 src/bin/criticality-worker.ts",
7+
"dev:criticality-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker SERVICE=criticality-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9237 src/bin/criticality-worker.ts",
88
"run:pagerank": "tsx src/criticality/run-pagerank.ts",
99
"run:impact": "tsx src/criticality/run-impact.ts",
1010
"dev:pagerank": "tsx --expose-gc src/criticality/run-pagerank.ts",
@@ -14,25 +14,28 @@
1414
"start:dockerhub-sync": "SERVICE=dockerhub-sync tsx src/bin/dockerhub-sync.ts",
1515
"dev:dockerhub-sync": "SERVICE=dockerhub-sync LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9239 src/bin/dockerhub-sync.ts",
1616
"dev:dockerhub-sync:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=dockerhub-sync LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9239 src/bin/dockerhub-sync.ts",
17-
"start:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest tsx src/bin/bq-dataset-ingest.ts",
18-
"dev:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
19-
"dev:bq-dataset-ingest:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
17+
"start:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest SERVICE=bq-dataset-ingest tsx src/bin/bq-dataset-ingest.ts",
18+
"dev:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
19+
"dev:bq-dataset-ingest:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
2020
"export-to-bucket": "SERVICE=bq-dataset-ingest tsx src/scripts/exportToBucket.ts",
2121
"export-to-bucket:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=bq-dataset-ingest tsx src/scripts/exportToBucket.ts",
22-
"start:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker tsx src/bin/npm-worker.ts",
23-
"dev:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
24-
"dev:npm-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
25-
"start:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker tsx src/bin/osv-worker.ts",
26-
"dev:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
27-
"dev:osv-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
28-
"start:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker tsx src/bin/maven-worker.ts",
29-
"dev:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
30-
"dev:maven-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
22+
"trigger-bootstrap": "SERVICE=bq-dataset-ingest tsx src/scripts/triggerBootstrap.ts",
23+
"trigger-bootstrap:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=bq-dataset-ingest tsx src/scripts/triggerBootstrap.ts",
24+
"start:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker tsx src/bin/npm-worker.ts",
25+
"dev:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
26+
"dev:npm-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
27+
"start:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker SERVICE=osv-worker tsx src/bin/osv-worker.ts",
28+
"dev:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
29+
"dev:osv-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=osv-worker SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
30+
"start:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker SERVICE=maven-worker tsx src/bin/maven-worker.ts",
31+
"dev:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
32+
"dev:maven-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
3133
"backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts",
3234
"backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts",
3335
"backfill:stewardship": "SERVICE=stewardship-backfill tsx src/bin/stewardship-backfill.ts",
3436
"backfill:stewardship:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=stewardship-backfill LOG_LEVEL=info tsx src/bin/stewardship-backfill.ts",
35-
"monitor:osspckgs:local": "bash -c 'set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && node ../../../scripts/monitor-osspckgs.mjs'",
37+
"monitor:osspckgs": "SERVICE=bq-dataset-ingest tsx src/scripts/monitorOsspckgs.ts",
38+
"monitor:osspckgs:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=bq-dataset-ingest tsx src/scripts/monitorOsspckgs.ts",
3639
"lint": "npx eslint --ext .ts src --max-warnings=0",
3740
"format": "npx prettier --write \"src/**/*.ts\"",
3841
"format-check": "npx prettier --check .",
@@ -42,6 +45,7 @@
4245
"dependencies": {
4346
"@crowd/archetype-standard": "workspace:*",
4447
"@crowd/archetype-worker": "workspace:*",
48+
"@crowd/temporal": "workspace:*",
4549
"@crowd/common": "workspace:*",
4650
"@crowd/data-access-layer": "workspace:*",
4751
"@crowd/database": "workspace:*",

‎services/apps/packages_worker/src/deps-dev/README.md‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# bq-dataset-ingest — BQ → GCS → Postgres pipeline
2+
3+
Ingests public BigQuery datasets (deps.dev, OpenSSF Scorecard) into the
4+
packages database via a BQ export → GCS Parquet → staging → merge pipeline.
5+
Runs on the `bq-dataset-ingest` Temporal task queue.
6+
7+
## How to run
8+
9+
```bash
10+
# from services/apps/packages_worker
11+
12+
# local dev (hot-reload, loads backend/.env.*.local)
13+
pnpm dev:bq-dataset-ingest:local
14+
15+
# prod-style start
16+
pnpm start:bq-dataset-ingest
17+
18+
# trigger a bootstrap run manually
19+
pnpm trigger-bootstrap:local [full|incremental] [ECOSYSTEMS] [options]
20+
pnpm trigger-bootstrap [full|incremental] [ECOSYSTEMS] [options]
21+
22+
# monitor active and recent jobs
23+
pnpm monitor:osspckgs:local
24+
pnpm monitor:osspckgs
25+
```
26+
27+
See `src/scripts/triggerBootstrap.ts --help` for full options.
28+
29+
## BQ byte-ceiling overrides
30+
31+
Each job kind has a hardcoded `maxBytesGb` ceiling checked via a BQ dry-run
32+
before the real export fires. If the dry-run exceeds the ceiling the activity
33+
fails immediately (no BQ cost incurred).
34+
35+
Every ceiling can be overridden at runtime via an env variable — useful when a
36+
table grows past the default ceiling without requiring a code deploy:
37+
38+
```
39+
BQ_DATASET_INGEST_<KIND>_MAX_BQ_GB=<number> # applies to all sync modes
40+
BQ_DATASET_INGEST_<KIND>_<SYNCMODE>_MAX_BQ_GB=<number> # mode-specific, takes precedence
41+
```
42+
43+
where `<KIND>` is the job kind in `UPPER_SNAKE_CASE` and `<SYNCMODE>` is `FULL` or `INCREMENTAL`.
44+
The mode-specific key takes precedence over the generic key. Value must be a positive finite number.
45+
46+
**When adding a new job kind, add a row to this table.**
47+
48+
| Env var override | Default (GB) | Job kind | Notes |
49+
| -------------------------------------------------- | -----------: | ---------------------- | --------------------------------------------------------- |
50+
| `BQ_DATASET_INGEST_PACKAGES_FULL_MAX_BQ_GB` | 6000 | `packages` | Full only (set in `ingestPackages.ts`) |
51+
| `BQ_DATASET_INGEST_PACKAGES_INCREMENTAL_MAX_BQ_GB` | 400 | `packages` | Incremental only (set in `ingestPackages.ts`) |
52+
| `BQ_DATASET_INGEST_VERSIONS_MAX_BQ_GB` | 400 | `versions` | |
53+
| `BQ_DATASET_INGEST_PACKAGE_DEPENDENCIES_MAX_BQ_GB` | 10000 | `package_dependencies` | Incremental always scans ~2 full-day partitions (~3.85TB) |
54+
| `BQ_DATASET_INGEST_REPOS_MAX_BQ_GB` | 2000 | `repos` | |
55+
| `BQ_DATASET_INGEST_PACKAGE_REPOS_MAX_BQ_GB` | 2000 | `package_repos` | |
56+
| `BQ_DATASET_INGEST_ADVISORIES_MAX_BQ_GB` | 10 | `advisories` | |
57+
| `BQ_DATASET_INGEST_ADVISORY_PACKAGES_MAX_BQ_GB` | 1500 | `advisory_packages` | |
58+
| `BQ_DATASET_INGEST_DEPENDENT_COUNTS_MAX_BQ_GB` | 2000 | `dependent_counts` | |
59+
| `BQ_DATASET_INGEST_SCORECARD_REPOS_MAX_BQ_GB` | 50 | `scorecard_repos` | |
60+
| `BQ_DATASET_INGEST_SCORECARD_CHECKS_MAX_BQ_GB` | 500 | `scorecard_checks` | |
61+
62+
The override logic lives in `src/deps-dev/activities/bqExportToGcs.ts`.
63+
64+
## Environment variables
65+
66+
| Variable | Required | Purpose |
67+
| ------------------------------ | --------- | -------------------------------------------------------------------------------------------------- |
68+
| `OSSPCKGS_GCP_PROJECT` | yes | GCP project ID for BQ and GCS |
69+
| `OSSPCKGS_GCS_BUCKET` | yes | GCS bucket for Parquet exports |
70+
| `OSSPCKGS_GCP_CREDENTIALS_B64` | yes | Base64-encoded GCP service account JSON |
71+
| `OSSPCKGS_DEPS_TABLE` | no | Set to `B` to use `DependenciesLatest` (ADR-0003 Option B) instead of `DependencyGraphEdgesLatest` |
72+
| `CROWD_TEMPORAL_SERVER_URL` | yes | Temporal server address |
73+
| `CROWD_TEMPORAL_NAMESPACE` | yes | Temporal namespace (overrides `backend-config` default) |
74+
| `CROWD_TEMPORAL_CERTIFICATE` | prod only | Base64-encoded mTLS client certificate |
75+
| `CROWD_TEMPORAL_PRIVATE_KEY` | prod only | Base64-encoded mTLS private key |

‎services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,24 @@ export async function bqExportToGcs(input: BqExportToGcsInput): Promise<BqExport
150150
const dryRunBytes = Number(dryRunJob.metadata.statistics.totalBytesProcessed ?? 0)
151151
log.info({ jobKind, dryRunBytes, maxBytesGb }, 'BQ dry-run complete')
152152

153-
const ceiling = maxBytesGb * 1e9
153+
// Override table is in src/deps-dev/README.md — update it when adding new job kinds.
154+
// Mode-specific key takes precedence over the generic key (needed for kinds like "packages"
155+
// that have separate full/incremental ceilings: BQ_DATASET_INGEST_PACKAGES_FULL_MAX_BQ_GB).
156+
const baseKey = `BQ_DATASET_INGEST_${jobKind.toUpperCase().replace(/-/g, '_')}`
157+
const modeKey = `${baseKey}_${syncMode.toUpperCase()}_MAX_BQ_GB`
158+
const genericKey = `${baseKey}_MAX_BQ_GB`
159+
const activeKey = process.env[modeKey] !== undefined ? modeKey : genericKey
160+
const envOverride = process.env[activeKey]
161+
if (envOverride !== undefined) {
162+
const parsed = Number(envOverride)
163+
if (!isFinite(parsed) || parsed <= 0) {
164+
throw new Error(
165+
`Invalid env ${activeKey}="${envOverride}" — must be a positive finite number`,
166+
)
167+
}
168+
}
169+
const effectiveMaxBytesGb = envOverride !== undefined ? Number(envOverride) : maxBytesGb
170+
const ceiling = effectiveMaxBytesGb * 1e9
154171
if (dryRunBytes > ceiling) {
155172
throw new Error(
156173
`BQ dry-run for ${jobKind} reports ${dryRunBytes} bytes > ceiling ${ceiling} — aborting`,

‎services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ export async function ingestDependencies(opts: {
180180
runId: opts.runId,
181181
syncMode: opts.syncMode,
182182
snapshotAt: opts.today,
183-
maxBytesGb: 3000,
183+
maxBytesGb: 10000,
184184
reuseExports: opts.reuseExports,
185185
exportName: opts.exportName,
186186
})

‎services/apps/packages_worker/src/scorecard/queries/scorecardSql.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
import { SCORECARD_DATASET } from '../../deps-dev/config'
2-
31
export const SCORECARD_REPOS_SQL = `
42
SELECT
53
CASE
@@ -8,7 +6,7 @@ SELECT
86
END AS repo_url,
97
score,
108
date AS scanned_at
11-
FROM \`${SCORECARD_DATASET}.scorecard-v2_latest\`
9+
FROM \`openssf.scorecardcron.scorecard-v2_latest\`
1210
WHERE repo.name IS NOT NULL
1311
`
1412

@@ -21,7 +19,7 @@ SELECT
2119
c.name AS check_name,
2220
c.score AS check_score,
2321
c.reason AS check_reason
24-
FROM \`${SCORECARD_DATASET}.scorecard-v2_latest\` r,
22+
FROM \`openssf.scorecardcron.scorecard-v2_latest\` r,
2523
UNNEST(r.checks) AS c
2624
WHERE r.repo.name IS NOT NULL
2725
`

‎services/apps/packages_worker/src/scorecard/workflows/ingestScorecard.ts‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ const { gcsParquetToStaging } = proxyActivities<typeof depsDevActivities>({
2121

2222
const { mergeStagingToTable } = proxyActivities<typeof depsDevActivities>({
2323
startToCloseTimeout: '30 minutes',
24-
retry: { maximumAttempts: 1 },
24+
retry: { maximumAttempts: 3, initialInterval: '30 seconds', backoffCoefficient: 2 },
2525
})
2626

2727
const SCORECARD_REPOS_STAGING_TABLE = 'staging.osspckgs_scorecard_repos_raw'
@@ -50,8 +50,19 @@ SET scorecard_score = CASE
5050
END,
5151
scorecard_last_run_at = s.scanned_at::timestamptz,
5252
updated_at = NOW()
53-
FROM staging.osspckgs_scorecard_repos_raw s
53+
FROM (SELECT * FROM staging.osspckgs_scorecard_repos_raw ORDER BY repo_url) s
5454
WHERE r.url = s.repo_url
55+
AND (
56+
r.scorecard_score IS DISTINCT FROM CASE
57+
WHEN s.score IS NULL
58+
OR s.score = 'NaN'::float8
59+
OR s.score = 'Infinity'::float8
60+
OR s.score = '-Infinity'::float8
61+
THEN NULL
62+
ELSE s.score::numeric(3,1)
63+
END
64+
OR r.scorecard_last_run_at IS DISTINCT FROM s.scanned_at::timestamptz
65+
)
5566
`
5667

5768
const SCORECARD_CHECKS_STAGING_TABLE = 'staging.osspckgs_scorecard_checks_raw'
@@ -73,7 +84,11 @@ SELECT r.id,
7384
s.check_name,
7485
NULLIF(s.check_score, -1)::numeric(3,1),
7586
s.check_reason
76-
FROM staging.osspckgs_scorecard_checks_raw s
87+
FROM (
88+
SELECT DISTINCT ON (repo_url, check_name) repo_url, check_name, check_score, check_reason
89+
FROM staging.osspckgs_scorecard_checks_raw
90+
ORDER BY repo_url, check_name, check_score DESC NULLS LAST
91+
) s
7792
JOIN repos r ON r.url = s.repo_url
7893
ON CONFLICT (repo_id, check_name) DO UPDATE SET
7994
score = EXCLUDED.score,
@@ -95,7 +110,7 @@ export async function ingestScorecard(opts: {
95110
runId: opts.runId,
96111
syncMode: 'full',
97112
snapshotAt: null,
98-
maxBytesGb: 10,
113+
maxBytesGb: 50,
99114
reuseExports: opts.reuseExports,
100115
exportName: opts.exportName,
101116
})
@@ -166,7 +181,7 @@ export async function ingestScorecard(opts: {
166181
runId: opts.runId,
167182
syncMode: 'full',
168183
snapshotAt: null,
169-
maxBytesGb: 200,
184+
maxBytesGb: 500,
170185
reuseExports: opts.reuseExports,
171186
exportName: opts.exportName,
172187
})

0 commit comments

Comments
 (0)