diff --git a/api/src/enums/chart-summary-scope.js b/api/src/enums/chart-summary-scope.js new file mode 100644 index 000000000..cac09dbd9 --- /dev/null +++ b/api/src/enums/chart-summary-scope.js @@ -0,0 +1,24 @@ +import { GraphQLEnumType } from 'graphql' + +export const ChartSummaryScopeEnums = new GraphQLEnumType({ + name: 'ChartSummaryScopeEnums', + values: { + ALL: { + value: 'all', + description: 'Summary covering all organizations holding an approved claim.', + }, + VERIFIED: { + value: 'verified', + description: 'Summary covering verified organizations.', + }, + PSD: { + value: 'psd', + description: 'Summary covering organizations subject to the Policy on Service and Digital.', + }, + PGS: { + value: 'pgs', + description: 'Summary covering organizations subject to the Policy on Government Security.', + }, + }, + description: 'An enum used to select which set of organizations a chart summary covers.', +}) \ No newline at end of file diff --git a/api/src/enums/index.js b/api/src/enums/index.js index 3cf345b67..5086e6b0d 100644 --- a/api/src/enums/index.js +++ b/api/src/enums/index.js @@ -1,6 +1,7 @@ export * from './affiliation-org-order-field' export * from './affiliation-user-order-field' export * from './asset-state' +export * from './chart-summary-scope' export * from './user-order-field' export * from './dkim-order-field' export * from './dkim-result-order-field' diff --git a/api/src/summaries/loaders/load-chart-summaries-by-period.js b/api/src/summaries/loaders/load-chart-summaries-by-period.js index 7ffe98667..3d6e2382b 100644 --- a/api/src/summaries/loaders/load-chart-summaries-by-period.js +++ b/api/src/summaries/loaders/load-chart-summaries-by-period.js @@ -3,7 +3,7 @@ import { aql } from 'arangojs' export const loadChartSummariesByPeriod = ({ query, userKey, cleanseInput, i18n }) => - async ({ startDate, endDate, sortDirection = 'ASC', limit }) => { + async ({ startDate, endDate, sortDirection = 'ASC', limit, scope = 'verified' }) => { const cleansedStartDate = startDate ? cleanseInput(startDate) : null const cleansedEndDate = endDate ? cleanseInput(endDate) : new Date().toISOString() @@ -33,10 +33,16 @@ export const loadChartSummariesByPeriod = limitString = aql`LIMIT ${limit}` } + let scopeFilter = aql`FILTER summary.scope == ${scope}` + if (scope === 'verified') { + scopeFilter = aql`FILTER summary.scope == ${scope} OR summary.scope == null` + } + let requestedSummaryInfo try { requestedSummaryInfo = await query` FOR summary IN chartSummaries + ${scopeFilter} ${startDateFilter} ${endDateFilter} ${sortString} diff --git a/api/src/summaries/queries/find-chart-summaries.js b/api/src/summaries/queries/find-chart-summaries.js index 0da7b349f..42755f36c 100644 --- a/api/src/summaries/queries/find-chart-summaries.js +++ b/api/src/summaries/queries/find-chart-summaries.js @@ -1,7 +1,7 @@ import { GraphQLList, GraphQLString, GraphQLInt } from 'graphql' import { chartSummaryType } from '../objects' -import { OrderDirection } from '../../enums' +import { ChartSummaryScopeEnums, OrderDirection } from '../../enums' export const findChartSummaries = { type: new GraphQLList(chartSummaryType), @@ -23,6 +23,10 @@ export const findChartSummaries = { type: GraphQLInt, description: 'The maximum amount of summaries to be returned.', }, + scope: { + type: ChartSummaryScopeEnums, + description: 'The set of organizations the returned summaries should cover. Defaults to verified.', + }, }, resolve: async ( _, diff --git a/services/summaries/summaries.py b/services/summaries/summaries.py index 7fef17387..5aa6dd850 100644 --- a/services/summaries/summaries.py +++ b/services/summaries/summaries.py @@ -26,22 +26,34 @@ "web": ["https", "hsts", "ssl"], } +SCOPES = ["all", "verified", "psd", "pgs"] + logging.basicConfig(stream=sys.stdout, level=logging.INFO) -def domain_has_verified_claim(domain, db): +def domain_scopes(domain, db): cursor = db.aql.execute( """ - FOR v, e IN 1..1 INBOUND @domain_id claims - FILTER v.verified == true - FILTER e.assetState == "approved" - RETURN v + FOR org, claim IN 1..1 INBOUND @domain_id claims + FILTER claim.assetState == "approved" + RETURN { verified: org.verified, policies: org.policies } """, bind_vars={"domain_id": domain["_id"]}, ) - if cursor.empty(): - return False - return True + orgs = [org for org in cursor] + if len(orgs) == 0: + return set() + + scopes = {"all"} + for org in orgs: + if org.get("verified") is True: + scopes.add("verified") + policies = org.get("policies") or {} + if policies.get("psd") is True: + scopes.add("psd") + if policies.get("pgs") is True: + scopes.add("pgs") + return scopes def ignore_domain(domain): @@ -59,27 +71,28 @@ def ignore_domain(domain): ) -def get_domain_negative_findings(db, domain_id): +def get_domain_negative_findings(db, domain): cursor = db.aql.execute( """ LET emailTags = ( - FOR dnsScan, dnsE IN 1 OUTBOUND @domain_id domainsDNS - SORT dnsScan.timestamp DESC - LIMIT 1 - RETURN FLATTEN([dnsScan.dmarc.negativeTags, dnsScan.dkim.negativeTags, dnsScan.spf.negativeTags]) + LET dnsScan = DOCUMENT(@latest_dns_scan) + FILTER dnsScan != null + RETURN FLATTEN([dnsScan.dmarc.negativeTags, dnsScan.dkim.negativeTags, dnsScan.spf.negativeTags]) )[0] LET webTags = ( - FOR web, webE IN 1 OUTBOUND @domain_id domainsWeb - SORT web.timestamp DESC - LIMIT 1 - FOR webScan, webScanE IN 1 OUTBOUND web webToWebScans - RETURN FLATTEN([webScan.results.tlsResult.negativeTags, webScan.results.connectionResults.negativeTags]) + LET web = DOCUMENT(@latest_web_scan) + FILTER web != null + FOR webScan, webScanE IN 1 OUTBOUND web webToWebScans + RETURN FLATTEN([webScan.results.tlsResult.negativeTags, webScan.results.connectionResults.negativeTags]) ) FOR tag IN FLATTEN([emailTags, webTags], 2) FILTER tag != null RETURN tag """, - bind_vars={"domain_id": domain_id}, + bind_vars={ + "latest_dns_scan": domain.get("latestDnsScan"), + "latest_web_scan": domain.get("latestWebScan"), + }, ) if cursor.empty(): return [] @@ -94,97 +107,105 @@ def update_chart_summaries(host=DB_URL, name=DB_NAME, user=DB_USER, password=DB_ db = client.db(name, username=user, password=password) chartSummariesCol = db.collection("chartSummaries") - # Gather summaries from domain statuses + # TODO: remove this once all summaries have been migrated to the new format + db.aql.execute( + """ + FOR summary IN chartSummaries + FILTER summary.scope == null + UPDATE summary WITH { scope: "verified" } IN chartSummaries + """ + ) + + # Gather summaries from domain statuses, one accumulator per scope. chartSummaries = {} - for chart_type, scan_types in CHARTS.items(): - chartSummaries[chart_type] = { - "scan_types": scan_types, - "pass": 0, - "fail": 0, - "total": 0, + dmarc_phases = {} + for scope in SCOPES: + chartSummaries[scope] = { + chart_type: { + "scan_types": scan_types, + "pass": 0, + "fail": 0, + "total": 0, + } + for chart_type, scan_types in CHARTS.items() + } + # DMARC phases: assess, deploy, enforce, maintain + dmarc_phases[scope] = { + "assess": 0, + "deploy": 0, + "enforce": 0, + "maintain": 0, } - - # DMARC phases: - # 1. Assess - assess_count = 0 - # 2. Deploy - deploy_count = 0 - # 3. Enforce - enforce_count = 0 - # 4. Maintain - maintain_count = 0 for domain in db.collection("domains"): - if ( - ignore_domain(domain) is False - and domain_has_verified_claim(domain, db) is True - ): - # Update chart summaries - for chart_type in chartSummaries: - chart = chartSummaries[chart_type] - category_status = [] - for scan_type in chart["scan_types"]: - category_status.append(domain.get("status", {}).get(scan_type)) - if "fail" in category_status: - chart["fail"] += 1 - chart["total"] += 1 - elif ( - chart_type == "mail" - and domain.get("status", {}).get("dkim") == "info" - and "pass" in category_status - ) or "info" not in category_status: - chart["pass"] += 1 - chart["total"] += 1 - - # Update DMARC phase summaries - phase = domain.get("phase") - if phase is None or domain.get("status", {}).get("dmarc") == "info": - logging.info( - f"No DMARC scan data available for domain \"{domain['domain']}\"." - ) + if ignore_domain(domain) is True: + continue + + scopes = domain_scopes(domain, db) + if len(scopes) == 0: + continue + + # Update chart summaries + for chart_type, scan_types in CHARTS.items(): + category_status = [] + for scan_type in scan_types: + category_status.append(domain.get("status", {}).get(scan_type)) + if "fail" in category_status: + result = "fail" + elif ( + chart_type == "mail" + and domain.get("status", {}).get("dkim") == "info" + and "pass" in category_status + ) or "info" not in category_status: + result = "pass" + else: continue + for scope in scopes: + chart = chartSummaries[scope][chart_type] + chart[result] += 1 + chart["total"] += 1 + + # Update DMARC phase summaries + phase = domain.get("phase") + if phase is None or domain.get("status", {}).get("dmarc") == "info": + logging.info( + f"No DMARC scan data available for domain \"{domain['domain']}\"." + ) + continue - if phase == "assess": - assess_count = assess_count + 1 - elif phase == "deploy": - deploy_count = deploy_count + 1 - elif phase == "enforce": - enforce_count = enforce_count + 1 - elif phase == "maintain": - maintain_count = maintain_count + 1 - - # Update DMARC phase summaries in DB - dmarc_phase_summary = { - "assess": assess_count, - "deploy": deploy_count, - "enforce": enforce_count, - "maintain": maintain_count, - "total": assess_count - + deploy_count - + enforce_count - + maintain_count, - } - - # Update chart summaries in DB + if phase in ("assess", "deploy", "enforce", "maintain"): + for scope in scopes: + dmarc_phases[scope][phase] += 1 + + # Write one document per date, scope today_iso = date.today().isoformat() - cursor = chartSummariesCol.find({"date": today_iso}) - if cursor.empty(): - chartSummariesCol.insert( - { - "date": today_iso, - **chartSummaries, - "dmarc_phase": dmarc_phase_summary, - } - ) - else: - logging.info("Chart summary from today already present. Updating summary...") - chartSummariesCol.update_match( - {"date": today_iso}, - { - **chartSummaries, - "dmarc_phase": dmarc_phase_summary, - }, - ) + for scope in SCOPES: + dmarc_phase_summary = { + **dmarc_phases[scope], + "total": sum(dmarc_phases[scope].values()), + } + + cursor = chartSummariesCol.find({"date": today_iso, "scope": scope}) + if cursor.empty(): + chartSummariesCol.insert( + { + "date": today_iso, + "scope": scope, + **chartSummaries[scope], + "dmarc_phase": dmarc_phase_summary, + } + ) + else: + logging.info( + f'Chart summary for scope "{scope}" from today already present. Updating summary...' + ) + chartSummariesCol.update_match( + {"date": today_iso, "scope": scope}, + { + **chartSummaries[scope], + "dmarc_phase": dmarc_phase_summary, + }, + ) logging.info(f"Chart summary update completed.") @@ -343,7 +364,7 @@ def update_org_summaries(host=DB_URL, name=DB_NAME, user=DB_USER, password=DB_PA # Negative tags domain_negative_tags = get_domain_negative_findings( - db, domain["_id"] + db, domain ) for tag in domain_negative_tags: if tag in negative_tags: diff --git a/services/summaries/tests/test_summaries.py b/services/summaries/tests/test_summaries.py index 4d9ea6cb7..59968f499 100644 --- a/services/summaries/tests/test_summaries.py +++ b/services/summaries/tests/test_summaries.py @@ -61,6 +61,7 @@ def arango_db(self): { "_key": "testorg", "verified": True, + "policies": {"psd": True, "pgs": False}, "summaries": { "organization": "organizations/testorg", "dmarc": {"pass": 0, "fail": 0, "total": 0}, @@ -170,34 +171,84 @@ def test_update_chart_summaries(self, arango_db): host=DB_URL, name=db_name, user=DB_USER, password=DB_PASS ) - summary = arango_db.collection("chartSummaries").all().next() - assert summary["https"] == { - "scan_types": ["https"], - "pass": 2, - "fail": 1, - "total": 3, - } + chart_summaries = arango_db.collection("chartSummaries") - assert summary["web"] == { - "scan_types": ["https", "hsts", "ssl"], - "pass": 2, - "fail": 1, - "total": 3, - } + def get_scope(scope): + summary = chart_summaries.find({"scope": scope}).next() + for k in ("_id", "_key", "_rev"): + summary.pop(k, None) + return summary - assert summary["mail"] == { - "scan_types": ["dmarc", "spf", "dkim"], - "pass": 1, - "fail": 2, - "total": 3, + populated = { + "date": date.today().isoformat(), + "https": {"scan_types": ["https"], "pass": 2, "fail": 1, "total": 3}, + "dmarc": {"scan_types": ["dmarc"], "pass": 2, "fail": 1, "total": 3}, + "web_connections": { + "scan_types": ["https", "hsts"], + "pass": 2, + "fail": 1, + "total": 3, + }, + "ssl": {"scan_types": ["ssl"], "pass": 2, "fail": 1, "total": 3}, + "spf": {"scan_types": ["spf"], "pass": 2, "fail": 1, "total": 3}, + "dkim": {"scan_types": ["dkim"], "pass": 1, "fail": 2, "total": 3}, + "mail": { + "scan_types": ["dmarc", "spf", "dkim"], + "pass": 1, + "fail": 2, + "total": 3, + }, + "web": { + "scan_types": ["https", "hsts", "ssl"], + "pass": 2, + "fail": 1, + "total": 3, + }, + "dmarc_phase": { + "assess": 0, + "deploy": 0, + "enforce": 0, + "maintain": 2, + "total": 2, + }, } - assert summary["dmarc_phase"] == { - "assess": 0, - "deploy": 0, - "enforce": 0, - "maintain": 2, - "total": 2, + for scope in ("all", "verified", "psd"): + assert get_scope(scope) == {**populated, "scope": scope} + + assert get_scope("pgs") == { + "date": date.today().isoformat(), + "scope": "pgs", + "https": {"scan_types": ["https"], "pass": 0, "fail": 0, "total": 0}, + "dmarc": {"scan_types": ["dmarc"], "pass": 0, "fail": 0, "total": 0}, + "web_connections": { + "scan_types": ["https", "hsts"], + "pass": 0, + "fail": 0, + "total": 0, + }, + "ssl": {"scan_types": ["ssl"], "pass": 0, "fail": 0, "total": 0}, + "spf": {"scan_types": ["spf"], "pass": 0, "fail": 0, "total": 0}, + "dkim": {"scan_types": ["dkim"], "pass": 0, "fail": 0, "total": 0}, + "mail": { + "scan_types": ["dmarc", "spf", "dkim"], + "pass": 0, + "fail": 0, + "total": 0, + }, + "web": { + "scan_types": ["https", "hsts", "ssl"], + "pass": 0, + "fail": 0, + "total": 0, + }, + "dmarc_phase": { + "assess": 0, + "deploy": 0, + "enforce": 0, + "maintain": 0, + "total": 0, + }, } def test_update_org_summaries(self, arango_db):