Support evidence source

schemas/deployments-schema.yaml

@@ -60,10 +60,12 @@ properties:
           - data_product_region
           - publication_date
         properties:
+          name_source: { type: string }
           name:
             tier: 1
             description: The name of the data product
             type: string
+          data_curators_source: { type: string }
           data_curators:
             tier: 1
             type: array
@@ -73,10 +75,12 @@ properties:
             description: The name of the entity publishing the data product.
             description_long: |
               The **curator** refers to the entity who published the data product. Examples of curators include specific government agencies and companies.
+          intended_use_source: { type: string }
           intended_use:
             tier: 1
             description: Intended use(s) of the data product.
             type: string
+          data_product_type_source: { type: string }
           data_product_type:
             tier: 1
             description: Type of data product. This enumeration will grow over time with more examples.
@@ -91,21 +95,25 @@ properties:
               - Interactive API
               - SQL queries
               # Add to this list as needed.
+          data_product_region_source: { type: string }
           data_product_region:
             tier: 1
             description: Free text. What region does the data describe, and/or, what region's laws apply to this data product.
             type: string
+          description_source: { type: string }
           description:
             tier: 1
             description: Brief description of the data product.
             type: string
+          publication_date_source: { type: string }
           publication_date:
             tier: 1
             description: When the data product was published, in YYYY-MM-DD format. Day and month can be set to "01" if unknown. In cases of many releases, the publication date can be expressed as the date of first publication.
             description_long: |
               The publication **date** refers to when the data product was published or otherwise created. In cases of data products that are created under continual release, the date listed is the date of first publication.
             type: string
             format: date
+          data_product_sector_source: { type: string }
           data_product_sector:
             tier: 1
             description: The industry or domain described by the data product.
@@ -123,6 +131,7 @@ properties:
         type: object
         additionalProperties: False
         properties:
+          variant_name_source: { type: string }
           variant_name:
             tier: 2
             description: |
@@ -143,10 +152,12 @@ properties:
               - Zero-concentrated DP
               - Renyi DP
               - Custom
+          data_domain_source: { type: string }
           data_domain:
             tier: 3
             description: Actual, potential, or counterfactual datasets eligible for protection.
             type: string
+          unprotected_quantities_source: { type: string }
           unprotected_quantities:
             tier: 3
             description: Any quantities in the data product that are unprotected by DP (e.g., statistics computed over a dataset that are released in the clear, without DP noise, and sometimes called “invariants”).
@@ -157,6 +168,7 @@ properties:
         type: object
         additionalProperties: False
         properties:
+          privacy_unit_source: { type: string }
           privacy_unit:
             tier: 2
             description: The entity whose data changes under adjacent datasets (see above), like a user or a user’s contribution (among multiple) (based partly on Bailie et al. 2025). High-level description of the granularity of protection (e.g., user level; user-day).
@@ -168,10 +180,12 @@ properties:
               More formally, the privacy unit is the entity whose data changes under adjacent datasets <a href="https://www.proquest.com/docview/3217403500" target="_blank">(Bailie 2025)</a>.
               Tier 3 entries go beyond stating a high-level description of the privacy unit and include a precise specification of what constitutes adjacent datasets.
             type: string
+          privacy_unit_details_source: { type: string }
           privacy_unit_details:
             tier: 3
             description: A precise specification of what constitutes adjacent datasets (e.g., in terms of your dataset schema. Function that computes the distance between between any two datasets in the data domain (based partly on HoDP, pg 55) Maximum distance, computed by the input metric, between any two datasets in the data domain; pairs of datasets whose distance is within this bound are called “adjacent datasets”.
             type: string
+          privacy_parameters_source: { type: string }
           privacy_parameters:
             tier: 2
             description: Intensity of protection, as characterized by values set for parameters like epsilon, delta, or rho. Which parameters are specified will vary according to the DP variant.
@@ -192,13 +206,15 @@ properties:
                   - string
               rho:
                 type: number
+          privacy_parameters_details_source: { type: string }
           privacy_parameters_details:
             tier: 3
             description: More detail on the parameters, if necessary.
       deployment_model:
         type: object
         additionalProperties: False
         properties:
+          model_name_source: { type: string }
           model_name:
             tier: 2
             description: Name of the deployment model. “The models differ based on how much trust individuals in the data have in a central authority data system.” (HoDP, pg 34)
@@ -235,12 +251,15 @@ properties:
               - Shuffle
               - Federated
               - Varies # Present in converted records
+          model_name_details_source: { type: string }
           model_name_details:
             type: string
+          actors_source: { type: string }
           actors:
             tier: 3
             description: Who are the relevant actors in the deployment? This includes anyone who may see the data product, even partially, and including adversaries. What are their trust assumptions and what is the rationale for these trust assumptions?
             type: string
+          release_type_source: { type: string }
           release_type:
             tier: 2
             description: |
@@ -253,12 +272,14 @@ properties:
             enum:
               - One release
               - Many releases
+          release_type_details_source: { type: string }
           release_type_details:
             description: |
               For one-release deployments, entries should state if there are plans for future uses or publications of the data used to create the data product.
 
               For many-release deployments, entries should include a description of the refreshment timeframe (the amount of time after which the privacy loss budget resets), how privacy loss is managed over time, and whether a fixed amount of privacy loss is allowed before the data used to create the data product is no longer queried.
             type: string
+          data_source_type_source: { type: string }
           data_source_type:
             tier: 2
             description: If the underlying data are dynamic, it means that new underlying data come in over time. On the other hand, if the underlying data are static, new data do not come in over time.
@@ -268,9 +289,11 @@ properties:
             enum:
               - Static
               - Dynamic
+          data_source_type_details_source: { type: string }
           data_source_type_details:
             tier: 3
             description: If more description is useful.
+          access_type_source: { type: string }
           access_type:
             tier: 2
             description: |
@@ -281,6 +304,7 @@ properties:
             enum:
               - Interactive
               - Non-interactive
+          access_type_details_source: { type: string }
           access_type_details:
             tier: 3
             description: Tier 3 entries that are interactive should also describe how the privacy loss budget is apportioned to and across analysts, and whether non-collusion between analysts is assumed. Under non-interactive deployments, people cannot interactively query the underlying data. Instead, they must interact with the published data product as is.
@@ -291,6 +315,7 @@ properties:
         type: object
         additionalProperties: False
         properties:
+          post_processing_source: { type: string }
           post_processing:
             tier: 3
             description: Functions applied to the data product after being protected under DP
@@ -300,6 +325,7 @@ properties:
               For example, imagine a data curator releasing a data product that includes counts of people in different geographic regions.
               A privacy-protected count may be negative, but the curator may not want to publish a seemingly nonsensical count and may turn these negative counts into zeros.
             type: string
+          composition_source: { type: string }
           composition:
             tier: 3
             description: How privacy loss is accounted across multiple differentially private queries, like sequential or parallel composition
@@ -317,13 +343,15 @@ properties:
         type: object
         additionalProperties: False
         properties:
+          preprocessing_and_hyperparameter_tuning_source: { type: string }
           preprocessing_and_hyperparameter_tuning:
             tier: 3
             description: Description of any pre-processing of the data before DP protections and any exploratory data analysis conducted before DP protections, and whether privacy loss was accounted for. How were hyperparameters, like privacy loss parameters, tuned? Was privacy loss associated with this tuning accounted for?
             description_long: |
               How privacy loss was accounted for, if at all, during steps taken prior to the creation of the data product and application of differential privacy.
               These steps include data pre-processing, EDA, and hyperparameter tuning (e.g., on privacy-loss parameters).
             type: string
+          mechanisms_source: { type: string }
           mechanisms:
             tier: 3
             description: |
@@ -338,6 +366,7 @@ properties:
               Differential privacy can be achieved using several mechanisms, i.e., algorithms that take the data as input and output a privacy-protected data product.
               Tier 3 entries describe these mechanisms and how they were implemented (e.g., using a library). These entries also describe other security-related implementation details, like what measures were taken to protect against known vulnerabilities, like floating point and timing channel attacks (e.g., see <a href="https://ieeexplore.ieee.org/abstract/document/9833672" target="_blank">(Jin, McMurtry, Rubinstein, Ohrimenko 2022)</a>. If available, code for implementation should be linked in this section.
             type: string
+          justification_source: { type: string }
           justification:
             tier: 3
             description: |
@@ -359,6 +388,29 @@ properties:
               This section may also include links to the data product itself, if publicly available.
             tier: 1
             type: string
+          evidence_sources:
+            description_long: |
+              This section includes links to white papers, blogposts, or other sources that describe the deployment.
+              This section may also include links to the data product itself, if publicly available.
+            tier: 1
+            type: object
+            patternProperties:
+              ^\w+$:
+                type: object
+                additionalProperties: False
+                required:
+                  - base
+                properties:
+                  url: { type: string }
+                  title: { type: string }
+                  note: { type: string }
+                  base:
+                    type: string
+                    enum:
+                      - "Peer-reviewed publication"
+                      - "Formal publication by the deploying organization"
+                      - "Authorized organizational attestation"
+                      - "Other public source"
           notes:
             tier: 1
             type: string

scripts/check.py

@@ -96,6 +96,24 @@ def check_latex_escapes(yaml_path):
     return errors
 
 
+def check_evidence_sources(yaml_path):
+    deployment = load(yaml_path.open(), Loader=Loader)
+    # TODO: When evidence_sources is required, get rid of "get".
+    evidence_sources_keys = (
+        deployment["deployment"]["administrative"].get("evidence_sources", {}).keys()
+    )
+
+    pairs = get_all_values_paths(deployment)
+    source_pairs = [pair for pair in pairs if pair[0].endswith("_source")]
+
+    errors = []
+    for path, text in source_pairs:
+        if text not in evidence_sources_keys:
+            errors.append(f"{path} should be one of {evidence_sources_keys}")
+
+    return errors
+
+
 checks = {name for name in globals().keys() if name.startswith("check_")}
 
 

tests/good_deployments/template.yaml

@@ -4,53 +4,87 @@ tier: 3
 deployment:
   basic:
     # Tier 1:
+    name_source: 'example'
     name: ''
+    data_curators_source: 'example'
     data_curators:
       - 'U.S. Census Bureau' # Fill in correct value.
+    intended_use_source: 'example'
     intended_use: ''
+    data_product_type_source: 'example'
     data_product_type: 'Summary statistics' # Fill in correct value.
+    data_product_region_source: 'example'
     data_product_region: ''
+    description_source: 'example'
     description: ''
+    publication_date_source: 'example'
     publication_date: '1900-01-01' # Fill in correct value.
+    data_product_sector_source: 'example'
     data_product_sector: 'Technology' # Fill in correct value.
 
   # Tiers 2 and 3:
   dp_variant:
+    variant_name_source: 'example'
     variant_name: Pure DP # Fill in correct value.
+    data_domain_source: 'example'
     data_domain: '' # Tier 3
+    unprotected_quantities_source: 'example'
     unprotected_quantities: '' # Tier 3
 
   privacy_loss:
+    privacy_unit_source: 'example'
     privacy_unit: ''
+    privacy_unit_details_source: 'example'
     privacy_unit_details: ''
+    privacy_parameters_source: 'example'
     privacy_parameters: # Fill in correct value, and remove others
       epsilon: 1.0
       delta: 1.0e-10
       rho: 1.0
+    privacy_parameters_details_source: 'example'
     privacy_parameters_details: ''
 
   deployment_model:
+    model_name_source: 'example'
     model_name: Central # Fill in correct value.
+    model_name_details_source: 'example'
     model_name_details: ''
+    actors_source: 'example'
     actors: ''
+    release_type_source: 'example'
     release_type: Many releases # Fill in correct value.
+    release_type_details_source: 'example'
     release_type_details: ''
+    data_source_type_source: 'example'
     data_source_type: Static # Fill in correct value.
+    data_source_type_details_source: 'example'
     data_source_type_details: ''
+    access_type_source: 'example'
     access_type: Non-interactive # Fill in correct value.
+    access_type_details_source: 'example'
     access_type_details: ''
 
   accounting:
+    post_processing_source: 'example'
     post_processing: '' # Tier 3
+    composition_source: 'example'
     composition: '' # Tier 3
 
   implementation:
+    preprocessing_and_hyperparameter_tuning_source: 'example'
     preprocessing_and_hyperparameter_tuning: '' # Tier 3
+    mechanisms_source: 'example'
     mechanisms: '' # Tier 3
+    justification_source: 'example'
     justification: '' # Tier 3
 
   administrative:
     sources: '' # white papers
+    evidence_sources:
+      example:
+        url: 'http://example.com' # Fill in correct value
+        title: 'Example' # Fill in correct value
+        base: 'Peer-reviewed publication'
     notes: '' # other extra information
     registry_authors:
       - '' # Fill in correct value.

tests/test_schema.py

@@ -41,10 +41,13 @@ def test_node_has_details(path, node):
         "/deployment/administrative",
         "/deployment/administrative/sources",
         "/deployment/administrative/notes",
+        "/deployment/administrative/evidence_sources",
     ]:
         assert "description" not in node.keys()
         pytest.skip("TODO: More description would be nice to have")
-    assert "description" in node.keys()
+    if path.endswith("_source"):
+        pytest.skip("description_long not needed for every _source field")
+    assert "description" in node.keys(), f"{path} missing description"
 
 
 @pytest.mark.parametrize(("path", "node"), path_nodes, ids=paths)
@@ -82,7 +85,9 @@ def test_node_has_details_long(path, node):
     if path in skip_list:
         assert "description_long" not in node.keys()
         pytest.skip("TODO: More description_long would be nice to have")
-    assert "description_long" in node.keys()
+    if path.endswith("_source"):
+        pytest.skip("description_long not needed for every _source field")
+    assert "description_long" in node.keys(), f"{path} missing description_long"
 
 
 @pytest.mark.parametrize(("path", "node"), path_nodes, ids=paths)
@@ -100,7 +105,9 @@ def test_node_has_tier(path, node):
     ]:
         assert "tier" not in node.keys()
         pytest.skip("TODO: More tiers would be nice to have")
-    assert "tier" in node.keys()
+    if path.endswith("_source"):
+        pytest.skip("tier not needed for every _source field")
+    assert "tier" in node.keys(), f"{path} missing tier"
 
 
 @pytest.mark.parametrize(("path", "node"), path_nodes, ids=paths)
@@ -111,5 +118,12 @@ def test_object_has_additional_properties_false(path, node):
 
 
 def test_template_is_complete():
-    template_paths = [path for path, node in path_templates]
+    template_paths = [
+        # Unlike other objects in the schema, the keys of evidences_sources
+        # are user-supplied rather than being fixed, so there are keys in the template,
+        # but not in the schema.
+        path
+        for path, node in path_templates
+        if "evidence_sources/" not in path
+    ]
     assert template_paths == paths