AllenNeuralDynamics
diff --git a/‎.github/workflows/publish_dev.yml‎
Lines changed: 2 additions & 2 deletions b/‎.github/workflows/publish_dev.yml‎
Lines changed: 2 additions & 2 deletions
diff --git a/‎.github/workflows/run_dev_tests.yml‎
Lines changed: 1 addition & 1 deletion b/‎.github/workflows/run_dev_tests.yml‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎.github/workflows/run_main_tests.yml‎
Lines changed: 1 addition & 1 deletion b/‎.github/workflows/run_main_tests.yml‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎.github/workflows/tag_and_publish_main.yml‎
Lines changed: 2 additions & 39 deletions b/‎.github/workflows/tag_and_publish_main.yml‎
Lines changed: 2 additions & 39 deletions
diff --git a/‎.github/workflows/update_docs.yml‎
Lines changed: 4 additions & 4 deletions b/‎.github/workflows/update_docs.yml‎
Lines changed: 4 additions & 4 deletions
diff --git a/‎.gitignore‎
Lines changed: 5 additions & 0 deletions b/‎.gitignore‎
Lines changed: 5 additions & 0 deletions
diff --git a/‎.readthedocs.yaml‎
Lines changed: 5 additions & 1 deletion b/‎.readthedocs.yaml‎
Lines changed: 5 additions & 1 deletion
diff --git a/‎CONTRIBUTING.md‎
Lines changed: 77 additions & 9 deletions b/‎CONTRIBUTING.md‎
Lines changed: 77 additions & 9 deletions
@@ -20,10 +20,10 @@ jobs:
       - uses: actions/checkout@master
       - name: Pull latest changes
         run: git pull origin dev
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
       - name: Configure aws credentials
         uses: aws-actions/configure-aws-credentials@v2
         with:
 
@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10', '3.11' ]
+        python-version: [ '3.10', '3.11', '3.12', '3.13' ]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
 
@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10', '3.11' ]
+        python-version: [ '3.10', '3.11', '3.12', '3.13' ]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
 
@@ -53,10 +53,10 @@ jobs:
       - uses: actions/checkout@master
       - name: Pull latest changes
         run: git pull origin main
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
       - name: Configure aws credentials
         uses: aws-actions/configure-aws-credentials@v2
         with:
@@ -69,40 +69,3 @@ jobs:
           python -m aind_data_schema.utils.json_writer --output $TEMP_DIR --attach-version
           python -m pip install awscli
           aws s3 sync $TEMP_DIR s3://${AWS_DATA_SCHEMA_BUCKET}/$S3_PREFIX
-
-  upload_diagrams:
-    needs: tag_and_publish
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: read
-    env:
-      AWS_DATA_SCHEMA_ROLE: ${{ secrets.AWS_DATA_SCHEMA_ROLE_PROD }}
-      AWS_DATA_SCHEMA_BUCKET: ${{ vars.AWS_DATA_SCHEMA_BUCKET_PROD }}
-      AWS_REGION : ${{ vars.AWS_REGION_PROD }}
-      DIAGRAM_DIRECTORY: ./docs/source/_static/diagrams
-    steps:
-      - uses: actions/checkout@master
-      - name: Pull latest changes
-        run: git pull origin main
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v3
-        with:
-          python-version: 3.8
-      - name: Install dependencies
-        run: |
-          sudo apt install graphviz libgraphviz-dev -y
-          python -m pip install -e .[docs] --no-cache-dir
-          python -m pip install awscli
-      - name: Generate erdantic diagrams
-        run: |
-          mkdir $DIAGRAM_DIRECTORY
-          python -m aind_data_schema.utils.diagrams $DIAGRAM_DIRECTORY
-      - name: Configure aws credentials
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          role-to-assume: ${{ env.AWS_DATA_SCHEMA_ROLE }}
-          role-session-name: github-diagram-upload-session
-          aws-region: ${{ env.AWS_REGION }}
-      - name: Create and upload diagrams
-        run: aws s3 sync $DIAGRAM_DIRECTORY s3://${AWS_DATA_SCHEMA_BUCKET}/diagrams
@@ -14,10 +14,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           token: ${{ secrets.SERVICE_TOKEN }}
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
       - name: Install dependencies
         run: |
           python -m pip install -e .[dev] --no-cache-dir
@@ -51,10 +51,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           token: ${{ secrets.SERVICE_TOKEN }}
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: '3.10'
       - name: Install dependencies
         run: |
           sudo apt install graphviz libgraphviz-dev -y
 
@@ -140,3 +140,8 @@ dmypy.json
 
 # Jupyter
 notebook/
+*.vscode
+ephys_instrument.json
+examples/*.json
+*.json
+docs/base/models/*
@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.8"
+    python: "3.13"
 
 python:
   install:
@@ -12,3 +12,7 @@ python:
       extra_requirements:
         - dev
         - docs
+
+sphinx:
+  # Path to your Sphinx configuration file.
+  configuration: docs/source/conf.py
@@ -1,45 +1,92 @@
 # Contributor Guidelines
-This repository defines the schemas needed to validate and document metadata. As a core service, it is used across multiple teams and services. Therefore, any contributions must follow certain rules to ensure stability and organization.
-This document will go through best practices for contributing to this project
+
+Contributions to `aind-data-schema` must follow certain rules to ensure stability and organization. This document will go through best practices for contributing to this project
 
 ## Issues and Feature Requests
+
 Feature requests and bug reports are all welcome as [issues](https://github.com/AllenNeuralDynamics/aind-data-schema/issues). Create a ticket using the provided [templates](https://github.com/AllenNeuralDynamics/aind-metadata-mapper/issues/new/choose) to ensure we have enough information to work with.
 Our team will review, assign, and address the ticket. If the ticket is urgent, you may tag a dedicated engineer in the issue but please refrain from assigning it.
 
 If you have a broader suggestion or a question about how things work, start a new [Discussion](https://github.com/AllenNeuralDynamics/aind-data-schema/discussions)!
 
-NOTE: If your request requires upgrading pydantic, create a separate ticket and a dedicated engineer will handle the upgrade.
-
 ## Installation and Development
+
 To develop the software, *clone* the repository and create a new branch for your changes.
 Please do not fork this repository unless you are an external developer.
+
 ```bash
-git clone git@github.com:AllenNeuralDynamics/aind-metadata-mapper.git
+git clone git@github.com:AllenNeuralDynamics/aind-data-schema.git
+cd aind-data-schema
 git checkout -b my-new-feature-branch
-``` 
-Then run the following command in the checked out directory. 
+```
+
+It's recommended you work in an isolated virtual environment.
+
+```bash
+python -m venv .venv
+source .venv/bin/activate  # (unix)
+```
+
+Then run the following command in the checked out directory.
+
 ```bash
 pip install -e .[dev]
 ```
 
+### Upgrades
+
+Starting with the v2.0 release all changes must be accompanied by an [upgrader](https://github.com/AllenNeuralDynamics/aind-metadata-upgrader/) that converts valid metadata from the latest version tag of `aind-data-schema` to valid metadata in the version tag where your changes are introduced. Breaking changes are exempt from this requirement.
+
+### Documentation
+
+**Note**: The core files (`docs/source/acquisition.md`, etc) are auto-generated from the base files in the folder `docs/base/core` with the model definitions appended. You must modify the **base** file or your changes will be overwritten when you run the documentation generators.
+
+To generate the source files for the documentation and model class links, run:
+
+```python
+python src/aind_data_schema/utils/docs/model_generator.py
+python src/aind_data_schema/utils/docs/registries_generator.py
+python src/aind_data_schema/utils/docs/doc_generator.py
+```
+
+Then to create the documentation html files, run:
+
+```bash
+sphinx-build -b html docs/source/ docs/build/html
+```
+
+More info on sphinx installation can be found here: https://www.sphinx-doc.org/en/master/usage/installation.html
+
 ### Testing
+
 Testing is required to open a PR in this repository to ensure robustness and reliability of our codebase.
 - **1:1 Correspondence:** Structure unit tests in a manner that mirrors the module structure. 
   - For every package in the src directory, there should be a corresponding test package.
   - For every module in a package, there should be a corresponding unit test module.
   - For every method in a module, there should be a corresponding unit test.
+  - For complicated functions, keep unit test functions small and interpretable.
 - **Test Coverage:** Aim for comprehensive test coverage to validate all critical paths and edge cases within the module. To open a PR, you will need at least 80% coverage. 
   - Please test your changes using the **coverage** library, which will run the tests and log a coverage report:
+
     ```bash
     coverage run -m unittest discover && coverage report
     ```
+
     To open the coverage report in a browser, you can run
+
     ```bash
     coverage html
     ```
     and find the report in the htmlcov/index.html.
 
+To run a single unit test file you can use
+
+```bash
+coverage run -m unittest tests/your_test.py
+```
+
 ### Linters
+
 There are several libraries used to run linters and check documentation. We've included these in the development package. You can run them as described [here](https://github.com/AllenNeuralDynamics/aind-metadata-mapper/blob/main/README.md#linters-and-testing).
 
 - To run tests locally, navigate to AIND-DATA-SCHEMA directory in terminal and run (this will not run any on-line only tests):
@@ -80,11 +127,32 @@ There are several libraries used to run linters and check documentation. We've i
   ```
 **NOTE**: Please note that these linters are automatically run in github actions when a PR is opened. These linters must pass for a PR to merge. 
 
-## Documentation and Style Guide
-Documentation is required for contributing to this project. We have settled on using Numpy's conventions as a default: [Numpy docstring standards](https://numpydoc.readthedocs.io/en/latest/format.html)
+### Units
+Unit types (i.e. anything from [aind_data_schema_models.units](https://github.com/AllenNeuralDynamics/aind-data-schema-models/blob/main/src/aind_data_schema_models/units.py)) should always be paired with a variable in one of two patterns.
+
+When you have a single `variable` with a unit, you should add the `_unit` suffix on the name of the unit:
+
+```python
+variable: type = Field(...)
+variable_unit: XUnit = Field(...)
+```
+
+If the variable is `Optional[]` the unit should also be marked as optional.
 
+If you have multiple variables that map onto a single unit type, start each `variable` with the same prefix. The prefix should be unique within the class (watch out for inherited fields). 
+
+```python
+fov_width: type = Field(...)
+fov_height: type = Field(...)
+fov_unit: XUnit = Field(...)
+```
+
+### Docstrings
+
+Docstrings are required for contributing to this project. We have settled on using Numpy's conventions as a default: [Numpy docstring standards](https://numpydoc.readthedocs.io/en/latest/format.html)
 
 ## Pull Requests
+
 For internal members, please create a branch. For external members, please fork the repo and open a pull request from the fork. We'll primarily use [Angular](https://github.com/angular/angular/blob/main/CONTRIBUTING.md#commit) style for commit messages. Roughly, they should follow the pattern:
 ```
 <type>(<scope>): <short summary>