diff --git a/known_bad_urls.txt b/known_bad_urls.txt index 3b0ecdf..38e57a8 100644 --- a/known_bad_urls.txt +++ b/known_bad_urls.txt @@ -10,3 +10,4 @@ https://journalprivacyconfidentiality.org/index.php/jpc/article/view/722/684 https://doi.org/10.1145/3177732.3177733 https://medium.com/uber-security-privacy/uber-open-source-differential-privacy-57f31e85c57a https://analytics.wikimedia.org/published/datasets/country_project_page_historical/00_README.html +https://example.com/ diff --git a/schemas/deployments-schema.yaml b/schemas/deployments-schema.yaml index d7dc366..cb24f29 100644 --- a/schemas/deployments-schema.yaml +++ b/schemas/deployments-schema.yaml @@ -6,6 +6,99 @@ description: | This schema is based on ongoing research by Elena Ghazi, Priyanka Nanayakkara, and Salil Vadhan. type: object +$defs: + privacy_parameters: + type: object + additionalProperties: False + properties: + epsilon: + type: number + delta: + type: + - number + - string + rho: + type: number + + evidence_source: + oneOf: + - type: object + additionalProperties: False + properties: + peer_reviewed_url_ref: { type: string } + peer_reviewed_note: { type: string } + - type: object + additionalProperties: False + properties: + formal_publication_url_ref: { type: string } + formal_publication_note: { type: string } + - type: object + additionalProperties: False + properties: + authorized_attestation_url_ref: { type: string } + authorized_attestation_note: { type: string } + - type: object + additionalProperties: False + properties: + other_source_url_ref: { type: string } + other_source_note: { type: string } + + sourced_string: + # TODO: oneOf should only be retained temporarily during translation; + # Once the translation is complete, bare strings should not be allowed. + # We should not ask the UI to handle both forms at the same time. + oneOf: + - + type: string + - + type: object + additionalProperties: False + required: + - value + - evidence_sources + properties: + value: { type: string } + evidence_sources: + type: array + items: { $ref: "#/$defs/evidence_source" } + + sourced_date: + # TODO: oneOf should only be retained temporarily during translation. + oneOf: + - + type: string + format: date + - + type: object + additionalProperties: False + required: + - value + - evidence_sources + properties: + value: + type: string + format: date + evidence_sources: + type: array + items: { $ref: "#/$defs/evidence_source" } + + sourced_privacy_parameters: + # TODO: oneOf should only be retained temporarily during translation. + oneOf: + - + $ref: "#/$defs/privacy_parameters" + - + type: object + additionalProperties: False + required: + - value + - evidence_sources + properties: + value: + $ref: "#/$defs/privacy_parameters" + evidence_sources: + type: array + items: { $ref: "#/$defs/evidence_source" } additionalProperties: False required: - url_slug @@ -76,14 +169,14 @@ properties: intended_use: tier: 1 description: Intended use(s) of the data product. - type: string + $ref: "#/$defs/sourced_string" data_product_type: tier: 1 description: Type of data product. This enumeration will grow over time with more examples. description_long: | The **data product** is the output of the differentially-private pipeline. Examples of data products include summary statistics and machine learning models. type: string - enum: + enum: # TODO: pull into defs - Summary statistics - Machine learning model - Dataset @@ -94,23 +187,22 @@ properties: data_product_region: tier: 1 description: Free text. What region does the data describe, and/or, what region's laws apply to this data product. - type: string + $ref: "#/$defs/sourced_string" description: tier: 1 description: Brief description of the data product. - type: string + $ref: "#/$defs/sourced_string" publication_date: tier: 1 description: When the data product was published, in YYYY-MM-DD format. Day and month can be set to "01" if unknown. In cases of many releases, the publication date can be expressed as the date of first publication. description_long: | The publication **date** refers to when the data product was published or otherwise created. In cases of data products that are created under continual release, the date listed is the date of first publication. - type: string - format: date + $ref: "#/$defs/sourced_date" data_product_sector: tier: 1 description: The industry or domain described by the data product. type: string - enum: + enum: # TODO: pull into defs - Technology - Healthcare - Education @@ -139,7 +231,7 @@ properties: If a custom flavor of DP is used for a deployment, its entry must specify the privacy relation. Otherwise, the common name of the flavor used is provided. type: string - enum: + enum: # TODO: pull into defs - Pure DP - Approximate DP - Zero-concentrated DP @@ -149,30 +241,30 @@ properties: description: Function that computes the distance between between any two datasets in the data domain (based partly on HoDP, pg 55). description_long: | A function that computes the distance between between any two datasets in the data domain (pg 55, Cowan, Shoemate, Pereira 2024) - type: string + $ref: "#/$defs/sourced_string" bound_on_input_distance: description: Maximum distance, computed by the input metric, between any two datasets in the data domain; pairs of datasets whose distance is within this bound are called “adjacent datasets”. description_long: | The differential privacy guarantee holds for any two datasets in the data domain whose distance (computed by the input metric) is less than or equal to the bound on the input distance. Pairs of datasets whose distance is within the bound are called “adjacent datasets” – the entity which changes under adjacent datasets (e.g., an individual) is the privacy unit. - type: string + $ref: "#/$defs/sourced_string" output_measure: description: Function that computes the distance between probability distributions of the differentially private mechanism applied to datasets in the data domain. description_long: | A function that computes the distance between probability distributions of the differentially private mechanism applied to datasets in the data domain. - type: string + $ref: "#/$defs/sourced_string" bound_on_output_distance: description: Maximum distance, computed by the output metric, between probability distributions of the differentially private mechanism applied to adjacent datasets. description_long: | The maximum distance, computed by the output measure, between probability distributions of the differentially private mechanism applied to adjacent datasets. - type: string + $ref: "#/$defs/sourced_string" data_domain: tier: 3 description: Actual, potential, or counterfactual datasets eligible for protection. - type: string + $ref: "#/$defs/sourced_string" unprotected_quantities: tier: 3 description: Any quantities in the data product that are unprotected by DP (e.g., statistics computed over a dataset that are released in the clear, without DP noise, and sometimes called “invariants”). - type: string + $ref: "#/$defs/sourced_string" privacy_loss: description_long: | Privacy loss under differential privacy is characterized by a privacy unit and privacy parameters, both described below. @@ -189,11 +281,11 @@ properties: In other words, the privacy unit “characterizes what [is being] protected” (pg 27, Cowan, Shoemate, Pereira 2024). More formally, the privacy unit is the entity whose data changes under adjacent datasets (Bailie 2025). Tier 3 entries go beyond stating a high-level description of the privacy unit and include a precise specification of what constitutes adjacent datasets. - type: string + $ref: "#/$defs/sourced_string" privacy_unit_description: tier: 3 description: A precise specification of what constitutes adjacent datasets (e.g., in terms of your dataset schema. Function that computes the distance between between any two datasets in the data domain (based partly on HoDP, pg 55) Maximum distance, computed by the input metric, between any two datasets in the data domain; pairs of datasets whose distance is within this bound are called “adjacent datasets”. - type: string + $ref: "#/$defs/sourced_string" privacy_parameters: tier: 2 description: Intensity of protection, as characterized by values set for parameters like epsilon, delta, or rho. Which parameters are specified will vary according to the DP flavor. @@ -251,18 +343,18 @@ properties: A deployment can also vary in terms of whether (1) it is one-shot or many-shot, (2) its underlying data are dynamic or static, and (3) it is interactive or non-interactive: type: string - enum: + enum: # TODO: pull into defs - Local - Central - Shuffle - Federated - Varies # Present in converted records model_name_description: - type: string + $ref: "#/$defs/sourced_string" actors: tier: 3 description: Who are the relevant actors in the deployment? This includes anyone who may see the data product, even partially, and including adversaries. What are their trust assumptions and what is the rationale for these trust assumptions? - type: string + $ref: "#/$defs/sourced_string" release_type: tier: 2 description: | @@ -272,7 +364,7 @@ properties: description_long: | **One-shot vs. many-shot**: If the data product is one-shot, it is published once. In cases of one-shot deployments, entries should state if there are plans for future uses or publications of the data used to create the data product. If the data product is many-shot, there are many publications of it over time. Entries should include a description of the refreshment timeframe (the amount of time after which the privacy loss budget resets), how privacy loss is managed over time, and whether a fixed amount of privacy loss is allowed before the data used to create the data product is no longer queried. type: string - enum: + enum: # TODO: pull into defs - One release - Many releases release_type_description: @@ -280,14 +372,14 @@ properties: For one-release deployments, entries should state if there are plans for future uses or publications of the data used to create the data product. For many-release deployments, entries should include a description of the refreshment timeframe (the amount of time after which the privacy loss budget resets), how privacy loss is managed over time, and whether a fixed amount of privacy loss is allowed before the data used to create the data product is no longer queried. - type: string + $ref: "#/$defs/sourced_string" data_source_type: tier: 2 description: If the underlying data are dynamic, it means that new underlying data come in over time. On the other hand, if the underlying data are static, new data do not come in over time. description_long: | **Dynamic vs. static**: If the underlying data are dynamic, it means that new underlying data come in over time. On the other hand, if the underlying data are static, new data do not come in over time. type: string - enum: + enum: # TODO: pull into defs - Static - Dynamic data_source_type_description: @@ -300,13 +392,13 @@ properties: description_long: | **Interactive vs. non-interactive**: Under interactive deployments, people with permission, like data analysts, can interactively query the underlying data under differential privacy. They will be returned privacy-protected query estimates. Tier 3 entries that are interactive should also describe how the privacy loss budget is apportioned to and across analysts, and whether non-collusion between analysts is assumed. Under non-interactive deployments, people cannot interactively query the underlying data. Instead, they must interact with the published data product as is. type: string - enum: + enum: # TODO: pull into defs - Interactive - Non-interactive access_type_description: tier: 3 description: Tier 3 entries that are interactive should also describe how the privacy loss budget is apportioned to and across analysts, and whether non-collusion between analysts is assumed. Under non-interactive deployments, people cannot interactively query the underlying data. Instead, they must interact with the published data product as is. - type: string + $ref: "#/$defs/sourced_string" accounting: description_long: | **Privacy-loss accounting**, i.e., how privacy loss is accounted for over multiple queries to the underlying data, is described by composition and post-processing: @@ -321,7 +413,7 @@ properties: The impact of these functions may be important to know about for downstream applications using the data product, so they are reported for Tier 3 entries. For example, imagine a data curator releasing a data product that includes counts of people in different geographic regions. A privacy-protected count may be negative, but the curator may not want to publish a seemingly nonsensical count and may turn these negative counts into zeros. - type: string + $ref: "#/$defs/sourced_string" composition: tier: 3 description: How privacy loss is accounted across multiple differentially private queries, like sequential or parallel composition @@ -332,7 +424,7 @@ properties: Examples of composition include sequential composition and parallel composition. Under sequential composition, assuming pure DP, the sum of the privacy parameter value (epsilon) per query is the total privacy parameter value. Parallel composition, again assuming pure DP, says that the total privacy parameter value for a set of queries applied on disjoint subsets of the data is the maximum privacy parameter value used for one of the queries. - type: string + $ref: "#/$defs/sourced_string" implementation: description_long: | **Implementation** details are provided for Tier 3 entries. They include the following information: @@ -345,7 +437,7 @@ properties: description_long: | How privacy loss was accounted for, if at all, during steps taken prior to the creation of the data product and application of differential privacy. These steps include data pre-processing, EDA, and hyperparameter tuning (e.g., on privacy-loss parameters). - type: string + $ref: "#/$defs/sourced_string" mechanisms: tier: 3 description: | @@ -359,7 +451,7 @@ properties: description_long: | Differential privacy can be achieved using several mechanisms, i.e., algorithms that take the data as input and output a privacy-protected data product. Tier 3 entries describe these mechanisms and how they were implemented (e.g., using a library). These entries also describe other security-related implementation details, like what measures were taken to protect against known vulnerabilities, like floating point and timing channel attacks (e.g., see (Jin, McMurtry, Rubinstein, Ohrimenko 2022). If available, code for implementation should be linked in this section. - type: string + $ref: "#/$defs/sourced_string" justification: tier: 3 description: | @@ -368,7 +460,7 @@ properties: “What were the assumptions, modelling decisions, thresholds, and subjective decisions made in determining the implementation choices above? Why is the approach a thorough test of the stated assumptions? Was the process validated and verified? If so, how?” (Dwork Kohli Mulligan 2019) description_long: | Rationale for how decisions around implementation were made by the curator or others. In short, this information should answer the following questions laid out by Dwork Mulligan Kohli 2019: “What were the assumptions, modelling decisions, thresholds, and subjective decisions made in determining the implementation choices above? Why is the approach a thorough test of the stated assumptions? Was the process validated and verified? If so, how? - type: string + $ref: "#/$defs/sourced_string" resources: type: object additionalProperties: False @@ -380,7 +472,9 @@ properties: This section includes links to white papers, blogposts, or other sources that describe the deployment. This section may also include links to the data product itself, if publicly available. tier: 1 - type: string + oneOf: + - type: string # TODO: drop when translation is complete. + - type: object # Keys should match *_url_ref values. notes: tier: 1 type: string diff --git a/tests/good_deployments/google_tier_3.yaml b/tests/good_deployments/google_tier_3.yaml index 6914cac..0e0ad6b 100644 --- a/tests/good_deployments/google_tier_3.yaml +++ b/tests/good_deployments/google_tier_3.yaml @@ -9,7 +9,10 @@ deployment: description: Placeholder for tests intended_use: To help researchers, public health experts, and data analysts better understand the impact of COVID-19 via population-level symptom search trends data_product_type: Summary statistics - data_product_region: Global + data_product_region: + value: Global + source_url: https://example.com/ + page_number: 42 publication_date: "2020-09-01" dp_flavor: name: Pure DP