From fc632f7d05d28820720b9b0678379f6273172ac2 Mon Sep 17 00:00:00 2001 From: cdc-ap66 Date: Fri, 24 Oct 2025 15:44:06 +0000 Subject: [PATCH 1/2] manual fixes and auto fixes --- .gitignore | 760 +++++++++--------- .pre-commit-config.yaml | 1 + LICENSE | 402 ++++----- .../repo_files/_cfa_only/LICENSE | 402 ++++----- 4 files changed, 783 insertions(+), 782 deletions(-) diff --git a/.gitignore b/.gitignore index 85aae97..e4321ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,380 +1,380 @@ -##### -# Exclude many data and output file types by default - -# Data -*.csv -*.tsv -*.parquet -*.dat -*.bin -*.xls -*.xlsx - -# Documents -*.doc -*.docx -*.htm -# *.html -*.ppt -*.pptx -*.pdf - -# Images -*.bmp -*.jpg -*.jpeg -*.gif -*.pdf -# *.png - -# Compressed archives -*.gz -*.tar -*.tgz -*.rar -*.zip - -# Allowlist -# !your_data_file.csv -# !your_data_directory/ - - -##### -# Python -# https://github.com/github/gitignore/blob/main/Python.gitignore - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the -# code is intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in -# version control. However, in case of collaboration, if having -# platform-specific dependencies or dependencies having no cross-platform -# support, pipenv may install dependencies that don't work, or not install all -# needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock -# in version control. This is especially recommended for binary packages to -# ensure reproducibility, and is more commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in -# version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended -# to not include it in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore -# that can be found at -# https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. -# For a more nuclear option (not recommended) you can uncomment the following -# to ignore the entire idea folder. -#.idea/ - -##### -# R -# https://github.com/github/gitignore/blob/main/R.gitignore - -# History files -.Rhistory -.Rapp.history - -# Session Data files -.RData -.RDataTmp - -# User-specific files -.Ruserdata - -# Example code in package build process -*-Ex.R - -# Output files from R CMD build -/*.tar.gz - -# Output files from R CMD check -/*.Rcheck/ - -# RStudio files -.Rproj.user/ - -# produced vignettes -vignettes/*.html -vignettes/*.pdf - -# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 -.httr-oauth - -# knitr and R markdown default cache directories -*_cache/ -/cache/ - -# Temporary files created by R markdown -*.utf8.md -*.knit.md - -# R Environment Variables -.Renviron - -# translation temp files -po/*~ - -# RStudio Connect folder -rsconnect/ - -##### -# Java -# https://github.com/github/gitignore/blob/main/Java.gitignore - -# Compiled class file -*.class - -# Log file -*.log - -# BlueJ files -*.ctxt - -# Mobile Tools for Java (J2ME) -.mtj.tmp/ - -# Package Files # -*.jar -*.war -*.nar -*.ear -*.zip -*.tar.gz -*.rar - -# virtual machine crash logs, see -# http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* -replay_pid* - -##### -# C++ -# https://github.com/github/gitignore/blob/main/C%2B%2B.gitignore - -# Prerequisites -*.d - -# Compiled Object files -*.slo -*.lo -*.o -*.obj - -# Precompiled Headers -*.gch -*.pch - -# Compiled Dynamic libraries -*.so -*.dylib -*.dll - -# Fortran module files -*.mod -*.smod - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - - -##### -# Rust -# https://github.com/github/gitignore/blob/main/Rust.gitignore - -# Generated by Cargo -# will have compiled files and executables -debug/ -target/ - -# Remove Cargo.lock from gitignore if creating an executable, leave it for -# libraries. More information here -# https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -#Cargo.lock - -# These are backup files generated by rustfmt -**/*.rs.bk - -# MSVC Windows builds of rustc generate these, which store debugging information -*.pdb - -##### -# Julia -# https://github.com/github/gitignore/blob/main/Julia.gitignore - -# Files generated by invoking Julia with --code-coverage -*.jl.cov -*.jl.*.cov - -# Files generated by invoking Julia with --track-allocation -*.jl.mem - -# System-specific files and directories generated by the BinaryProvider and -# BinDeps packages. They contain absolute paths specific to the host computer, -# and so should not be committed -deps/deps.jl -deps/build.log -deps/downloads/ -deps/usr/ -deps/src/ - -# File generated by Pkg, the package manager, based on a corresponding -# Project.toml It records a fixed state of all packages used by the project. As -# such, it should not be committed for packages, but should be committed for -# applications that require a static environment. -#Manifest.toml - -*.lock -.vscode -tmp/ -*/logs/ -logs/ - -# mkdocs local build -site/ +##### +# Exclude many data and output file types by default + +# Data +*.csv +*.tsv +*.parquet +*.dat +*.bin +*.xls +*.xlsx + +# Documents +*.doc +*.docx +*.htm +# *.html +*.ppt +*.pptx +*.pdf + +# Images +*.bmp +*.jpg +*.jpeg +*.gif +*.pdf +# *.png + +# Compressed archives +*.gz +*.tar +*.tgz +*.rar +*.zip + +# Allowlist +# !your_data_file.csv +# !your_data_directory/ + + +##### +# Python +# https://github.com/github/gitignore/blob/main/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the +# code is intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in +# version control. However, in case of collaboration, if having +# platform-specific dependencies or dependencies having no cross-platform +# support, pipenv may install dependencies that don't work, or not install all +# needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock +# in version control. This is especially recommended for binary packages to +# ensure reproducibility, and is more commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in +# version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended +# to not include it in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore +# that can be found at +# https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. +# For a more nuclear option (not recommended) you can uncomment the following +# to ignore the entire idea folder. +#.idea/ + +##### +# R +# https://github.com/github/gitignore/blob/main/R.gitignore + +# History files +.Rhistory +.Rapp.history + +# Session Data files +.RData +.RDataTmp + +# User-specific files +.Ruserdata + +# Example code in package build process +*-Ex.R + +# Output files from R CMD build +/*.tar.gz + +# Output files from R CMD check +/*.Rcheck/ + +# RStudio files +.Rproj.user/ + +# produced vignettes +vignettes/*.html +vignettes/*.pdf + +# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 +.httr-oauth + +# knitr and R markdown default cache directories +*_cache/ +/cache/ + +# Temporary files created by R markdown +*.utf8.md +*.knit.md + +# R Environment Variables +.Renviron + +# translation temp files +po/*~ + +# RStudio Connect folder +rsconnect/ + +##### +# Java +# https://github.com/github/gitignore/blob/main/Java.gitignore + +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see +# http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +replay_pid* + +##### +# C++ +# https://github.com/github/gitignore/blob/main/C%2B%2B.gitignore + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + + +##### +# Rust +# https://github.com/github/gitignore/blob/main/Rust.gitignore + +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for +# libraries. More information here +# https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +#Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +##### +# Julia +# https://github.com/github/gitignore/blob/main/Julia.gitignore + +# Files generated by invoking Julia with --code-coverage +*.jl.cov +*.jl.*.cov + +# Files generated by invoking Julia with --track-allocation +*.jl.mem + +# System-specific files and directories generated by the BinaryProvider and +# BinDeps packages. They contain absolute paths specific to the host computer, +# and so should not be committed +deps/deps.jl +deps/build.log +deps/downloads/ +deps/usr/ +deps/src/ + +# File generated by Pkg, the package manager, based on a corresponding +# Project.toml It records a fixed state of all packages used by the project. As +# such, it should not be committed for packages, but should be committed for +# applications that require a static environment. +#Manifest.toml + +*.lock +.vscode +tmp/ +*/logs/ +logs/ + +# mkdocs local build +site/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc28c61..1bf396d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: - id: check-toml - id: end-of-file-fixer - id: mixed-line-ending + args: ['--fix', 'lf'] - id: trailing-whitespace ##### # Python diff --git a/LICENSE b/LICENSE index 29f81d8..261eeb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,201 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cfa/dataops/create_catalog/repo_files/_cfa_only/LICENSE b/cfa/dataops/create_catalog/repo_files/_cfa_only/LICENSE index 29f81d8..261eeb9 100644 --- a/cfa/dataops/create_catalog/repo_files/_cfa_only/LICENSE +++ b/cfa/dataops/create_catalog/repo_files/_cfa_only/LICENSE @@ -1,201 +1,201 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 044a2018d82ae41a54e224769eba3bf07708f209 Mon Sep 17 00:00:00 2001 From: cdc-ap66 Date: Fri, 24 Oct 2025 15:45:02 +0000 Subject: [PATCH 2/2] autofixed --- .github/ISSUE_TEMPLATE/bug_report.md | 54 +- .github/ISSUE_TEMPLATE/new_issue.md | 42 +- .github/actions/pre-commit/action.yaml | 44 +- .github/workflows/release.yaml | 238 +++--- .github/workflows/run_qaqc.yaml | 84 +- .github/workflows/tagging.yaml | 94 +-- .secrets.baseline | 224 +++--- README.md | 166 ++-- .../create_catalog/repo_files/.gitignore | 760 +++++++++--------- .../repo_files/_cfa_only/.secrets.baseline | 224 +++--- .../repo_files/_cfa_only/ruff.toml | 6 +- ruff.toml | 6 +- tests/conftest.py | 144 ++-- tests/test_config_validator.py | 150 ++-- 14 files changed, 1118 insertions(+), 1118 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index d960760..5260a4e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,27 +1,27 @@ ---- -name: Bug Fix -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Additional context** -Add any other context about the problem here. +--- +name: Bug Fix +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/new_issue.md b/.github/ISSUE_TEMPLATE/new_issue.md index a94d89a..75ad524 100644 --- a/.github/ISSUE_TEMPLATE/new_issue.md +++ b/.github/ISSUE_TEMPLATE/new_issue.md @@ -1,21 +1,21 @@ ---- -name: New Issue -about: Create a new issue and provide enough information to have a discussion -title: '' -labels: '' -assignees: '' - ---- - -## Description -A clear and concise description of what the problem is. - -## Considerations -Make sure not to forget about these. - -## Sub-tasks -List any sub-tasks that might help create a clear flow or sub-issues. - -## Definition of Done -This might help in cases when what is to be delivered is more than just one -piece of functional code. +--- +name: New Issue +about: Create a new issue and provide enough information to have a discussion +title: '' +labels: '' +assignees: '' + +--- + +## Description +A clear and concise description of what the problem is. + +## Considerations +Make sure not to forget about these. + +## Sub-tasks +List any sub-tasks that might help create a clear flow or sub-issues. + +## Definition of Done +This might help in cases when what is to be delivered is more than just one +piece of functional code. diff --git a/.github/actions/pre-commit/action.yaml b/.github/actions/pre-commit/action.yaml index 1264b59..e753db8 100644 --- a/.github/actions/pre-commit/action.yaml +++ b/.github/actions/pre-commit/action.yaml @@ -1,22 +1,22 @@ -name: pre-commit -description: run pre-commit -inputs: - extra_args: - description: options to pass to pre-commit run - required: false - default: '--all-files' -runs: - using: composite - steps: - - run: python -m pip install pre-commit - shell: bash - - run: python -m pip freeze --local - shell: bash - - uses: actions/cache@v4 - with: - path: | - ~/.cache/pre-commit - ~/.cache/R/renv - key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - run: pre-commit run --show-diff-on-failure --color=always ${{ inputs.extra_args }} - shell: bash +name: pre-commit +description: run pre-commit +inputs: + extra_args: + description: options to pass to pre-commit run + required: false + default: '--all-files' +runs: + using: composite + steps: + - run: python -m pip install pre-commit + shell: bash + - run: python -m pip freeze --local + shell: bash + - uses: actions/cache@v4 + with: + path: | + ~/.cache/pre-commit + ~/.cache/R/renv + key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + - run: pre-commit run --show-diff-on-failure --color=always ${{ inputs.extra_args }} + shell: bash diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 48a1d96..149e07e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,119 +1,119 @@ -name: Build Release Distribution - -on: - push: - tags: - # below is same as ^\d{4}\.\d{2}\.\d{2}$ seems workflows use simpler regex - - '[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9][0-9].[0-9]+' - -jobs: - build-and-release: - name: Build and Release - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Checkout release - run: | - git fetch origin release - git checkout release - - - name: Determine Version - id: get-version - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - VERSION="${GITHUB_REF#refs/tags/}" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: Determine If Tag Is From Release - id: is-release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - TAG_COMMIT=$(git rev-parse ${{ github.ref }}) - BRANCHES=$(git branch -r --contains "$TAG_COMMIT") - - if echo "$BRANCHES" | grep -q 'origin/release'; then - echo "should_continue=true" >> "$GITHUB_OUTPUT" - else - echo "Exiting Workflow: Tag is not from release branch" - echo "IF intending to create a release from tag follow steps under 'Creating A Release' in the README file" - echo "should_continue=false" >> "$GITHUB_OUTPUT" - fi - - - name: Setup Python - if: steps.is-release.outputs.should_continue == 'true' - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Install Poetry - if: steps.is-release.outputs.should_continue == 'true' - run: | - curl -sSL https://install.python-poetry.org | python3 - - poetry --version - - - name: Install Dependencies - if: steps.is-release.outputs.should_continue == 'true' - run: | - poetry install --no-root --with dev - - - name: Build Wheel - if: steps.is-release.outputs.should_continue == 'true' - run: | - poetry build - - - name: Create Release - if: steps.is-release.outputs.should_continue == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release create ${{ github.ref }} dist/* --generate-notes --latest - - - name: Update pyproject.toml Version - if: steps.is-release.outputs.should_continue == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - VERSION=${{ steps.get-version.outputs.version }} - - sed -i.bak -E "s/^version\s*=\s*\"[^\"]+\"/version = \"${VERSION}\"/" pyproject.toml - - - name: Update Change Log - if: steps.is-release.outputs.should_continue == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - VERSION=${{ steps.get-version.outputs.version }} - REPO_URL="https://github.com/${{ github.repository }}" - RELEASE_URL="$REPO_URL/releases/tag/$VERSION" - - awk -v tag="## [$VERSION]" -v url="$RELEASE_URL" ' - !done && /---/ { - print $0 - print "" - print tag - print url - done = 1 - next - } - { print $0 } - ' changelog.md > temp.md && mv temp.md changelog.md - - - name: Commit Change Log and pyproject.toml Updates - if: steps.is-release.outputs.should_continue == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - VERSION=${{ steps.get-version.outputs.version }} - - git config user.name "github-actions" - git config user.email "github-actions@users.noreply.github.com" - git add pyproject.toml changelog.md - git commit -m "Update Version to ${{ steps.get-version.outputs.version }}" - git push origin release +name: Build Release Distribution + +on: + push: + tags: + # below is same as ^\d{4}\.\d{2}\.\d{2}$ seems workflows use simpler regex + - '[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9][0-9].[0-9]+' + +jobs: + build-and-release: + name: Build and Release + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Checkout release + run: | + git fetch origin release + git checkout release + + - name: Determine Version + id: get-version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION="${GITHUB_REF#refs/tags/}" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Determine If Tag Is From Release + id: is-release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG_COMMIT=$(git rev-parse ${{ github.ref }}) + BRANCHES=$(git branch -r --contains "$TAG_COMMIT") + + if echo "$BRANCHES" | grep -q 'origin/release'; then + echo "should_continue=true" >> "$GITHUB_OUTPUT" + else + echo "Exiting Workflow: Tag is not from release branch" + echo "IF intending to create a release from tag follow steps under 'Creating A Release' in the README file" + echo "should_continue=false" >> "$GITHUB_OUTPUT" + fi + + - name: Setup Python + if: steps.is-release.outputs.should_continue == 'true' + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install Poetry + if: steps.is-release.outputs.should_continue == 'true' + run: | + curl -sSL https://install.python-poetry.org | python3 - + poetry --version + + - name: Install Dependencies + if: steps.is-release.outputs.should_continue == 'true' + run: | + poetry install --no-root --with dev + + - name: Build Wheel + if: steps.is-release.outputs.should_continue == 'true' + run: | + poetry build + + - name: Create Release + if: steps.is-release.outputs.should_continue == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create ${{ github.ref }} dist/* --generate-notes --latest + + - name: Update pyproject.toml Version + if: steps.is-release.outputs.should_continue == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION=${{ steps.get-version.outputs.version }} + + sed -i.bak -E "s/^version\s*=\s*\"[^\"]+\"/version = \"${VERSION}\"/" pyproject.toml + + - name: Update Change Log + if: steps.is-release.outputs.should_continue == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION=${{ steps.get-version.outputs.version }} + REPO_URL="https://github.com/${{ github.repository }}" + RELEASE_URL="$REPO_URL/releases/tag/$VERSION" + + awk -v tag="## [$VERSION]" -v url="$RELEASE_URL" ' + !done && /---/ { + print $0 + print "" + print tag + print url + done = 1 + next + } + { print $0 } + ' changelog.md > temp.md && mv temp.md changelog.md + + - name: Commit Change Log and pyproject.toml Updates + if: steps.is-release.outputs.should_continue == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION=${{ steps.get-version.outputs.version }} + + git config user.name "github-actions" + git config user.email "github-actions@users.noreply.github.com" + git add pyproject.toml changelog.md + git commit -m "Update Version to ${{ steps.get-version.outputs.version }}" + git push origin release diff --git a/.github/workflows/run_qaqc.yaml b/.github/workflows/run_qaqc.yaml index bf25c91..cc12766 100644 --- a/.github/workflows/run_qaqc.yaml +++ b/.github/workflows/run_qaqc.yaml @@ -1,42 +1,42 @@ -name: qaqc - -on: - pull_request: - push: - branches: [main] - -jobs: - tests: - name: pytest - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install poetry - run: | - pip install poetry>=2.0 - - - name: Test - run: | - poetry install --with dev - poetry run pytest - - # - name: Build badge - # if: ${{ github.event.pull_request.merged == true }} - # run: | - # poetry run coverage-badge -f -o docs/assets/badges/coverage.svg - # echo '' >> docs/assets/badges/coverage.svg - # git config user.name "${{ github.actor }}" - # git config user.email "${{ github.actor }}@users.noreply.github.com" - # git add docs/assets/badges/coverage.svg - # git commit -m "Updated coverage badge" - # git push origin main +name: qaqc + +on: + pull_request: + push: + branches: [main] + +jobs: + tests: + name: pytest + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install poetry + run: | + pip install poetry>=2.0 + + - name: Test + run: | + poetry install --with dev + poetry run pytest + + # - name: Build badge + # if: ${{ github.event.pull_request.merged == true }} + # run: | + # poetry run coverage-badge -f -o docs/assets/badges/coverage.svg + # echo '' >> docs/assets/badges/coverage.svg + # git config user.name "${{ github.actor }}" + # git config user.email "${{ github.actor }}@users.noreply.github.com" + # git add docs/assets/badges/coverage.svg + # git commit -m "Updated coverage badge" + # git push origin main diff --git a/.github/workflows/tagging.yaml b/.github/workflows/tagging.yaml index 85944a6..15699f2 100644 --- a/.github/workflows/tagging.yaml +++ b/.github/workflows/tagging.yaml @@ -1,47 +1,47 @@ -name: Tagging - -on: - push: - branches: - - main - -jobs: - tag: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Install dependencies - run: pip install toml # Install toml package to read pyproject.toml - - - name: Get version from pyproject.toml - id: get_version - run: | - VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['project']['version'])") - echo "VERSION=${VERSION}" >> $GITHUB_ENV - - - name: Create Tag - run: | - # Fetch all tags to ensure we have the latest tag information - git fetch --tags - - if [[ -z $(git tag -l "$VERSION") ]]; then - echo "Creating new tag: $VERSION" - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - git tag -a "$VERSION" -m "Release version $VERSION" - git push origin "$VERSION" || echo "Failed to push tag, it might already exist" - else - echo "Tag $VERSION already exists. Skipping tag creation." - fi - - # List all tags for verification - echo "Current tags:" - git tag --list +name: Tagging + +on: + push: + branches: + - main + +jobs: + tag: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: pip install toml # Install toml package to read pyproject.toml + + - name: Get version from pyproject.toml + id: get_version + run: | + VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['project']['version'])") + echo "VERSION=${VERSION}" >> $GITHUB_ENV + + - name: Create Tag + run: | + # Fetch all tags to ensure we have the latest tag information + git fetch --tags + + if [[ -z $(git tag -l "$VERSION") ]]; then + echo "Creating new tag: $VERSION" + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + git tag -a "$VERSION" -m "Release version $VERSION" + git push origin "$VERSION" || echo "Failed to push tag, it might already exist" + else + echo "Tag $VERSION already exists. Skipping tag creation." + fi + + # List all tags for verification + echo "Current tags:" + git tag --list diff --git a/.secrets.baseline b/.secrets.baseline index f2b52b1..63f42f9 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,112 +1,112 @@ -{ - "version": "1.4.0", - "plugins_used": [ - { - "name": "ArtifactoryDetector" - }, - { - "name": "AWSKeyDetector" - }, - { - "name": "AzureStorageKeyDetector" - }, - { - "name": "Base64HighEntropyString", - "limit": 4.5 - }, - { - "name": "BasicAuthDetector" - }, - { - "name": "CloudantDetector" - }, - { - "name": "DiscordBotTokenDetector" - }, - { - "name": "GitHubTokenDetector" - }, - { - "name": "HexHighEntropyString", - "limit": 3.0 - }, - { - "name": "IbmCloudIamDetector" - }, - { - "name": "IbmCosHmacDetector" - }, - { - "name": "JwtTokenDetector" - }, - { - "name": "KeywordDetector", - "keyword_exclude": "" - }, - { - "name": "MailchimpDetector" - }, - { - "name": "NpmDetector" - }, - { - "name": "PrivateKeyDetector" - }, - { - "name": "SendGridDetector" - }, - { - "name": "SlackDetector" - }, - { - "name": "SoftlayerDetector" - }, - { - "name": "SquareOAuthDetector" - }, - { - "name": "StripeDetector" - }, - { - "name": "TwilioKeyDetector" - } - ], - "filters_used": [ - { - "path": "detect_secrets.filters.allowlist.is_line_allowlisted" - }, - { - "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", - "min_level": 2 - }, - { - "path": "detect_secrets.filters.heuristic.is_indirect_reference" - }, - { - "path": "detect_secrets.filters.heuristic.is_likely_id_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_lock_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_potential_uuid" - }, - { - "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" - }, - { - "path": "detect_secrets.filters.heuristic.is_sequential_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_swagger_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_templated_secret" - } - ], - "results": {}, - "generated_at": "2023-09-24T19:52:08Z" -} +{ + "version": "1.4.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": {}, + "generated_at": "2023-09-24T19:52:08Z" +} diff --git a/README.md b/README.md index 972db6e..36d4b8c 100644 --- a/README.md +++ b/README.md @@ -1,83 +1,83 @@ -# CFA DataOps - -## Overview - -This project provides data tools and low friction access to versioned datasets within CDC's Center for Forecasting and Outbreak Analytics (CFA) virtual private cloud (VPC). It includes end-to-end ETL pipelines for configured data sources, custom workflows, schema validation patterns, and submodules that expand common data visualization and modeling/interpolation patterns. The project demonstrates best practices for data processing within the CFA environment. - -### Key Features - -- Configuration-driven ETL pipelines using TOML files -- SQL-based transformations using DuckDB, or Polar's built-in SQL context -- Azure Blob Storage integration for raw and transformed data storage -- Utilities for time-stamping and template loading -- Data validation and quality checks -- Custom workflows -- Tooling for easy access to Socrata Open Data API (e.i., data.cdc.gov) (copied and modified from [here](https://github.com/CDCgov/cfasodapy)) -- Ability to initialize new dataset catalog libraries and maintain datasets in more than one repo - -## Getting started - -1. Clone the repository and install dependencies with `poetry install` (**requires `poetry >= 2.0`**) -2. Ensure your have access to the relevant resources (e.g., `az login --identity`) -3. See which datasets are available: - ```python - from cfa.dataops import datacat - print(datacat.__namespace_list__) - ``` -4. See which versions of a dataset are available (blob endpoints for datasets will be namedL `.load`, `.extract`, or `.stage_##`): - ```python - print(datacat.{{dataset}}.load.get_versions()) - ``` -5. Get a specific version of a dataset: - ```python - df = datacat.{{dataset}}.load.get_dataframe() - df.glimpse() - ``` - -Read the [Dataset User Guide](docs/dataset_user.md) for more information about accessing datasets. - -Read the [Dataset Developer Guide](docs/dataset_developer.md) for information about how to run ETL to add new versions of an existing datasets and about how to create new datasets. - -## Project admins - -- Thomas Hladish (CDC/OD/ORR/CFA) -- Phillip Rogers (CDC/OD/ORR/CFA)(CTR) -- Ryan Raasch (CDC/OD/ORR/CFA)(CTR) - ---- - -## Disclaimers - -### General Disclaimer - -This repository was created for use by CDC programs to collaborate on public health related projects in support of the [CDC mission](https://www.cdc.gov/about/organization/mission.htm). GitHub is not hosted by the CDC, but is a third party website used by CDC and its partners to share information and collaborate on software. CDC use of GitHub does not imply an endorsement of any one particular service, product, or enterprise. - -### Public Domain Standard Notice - -This repository constitutes a work of the United States Government and is not subject to domestic copyright protection under 17 USC § 105. This repository is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). All contributions to this repository will be released under the CC0 dedication. By submitting a pull request you are agreeing to comply with this waiver of copyright interest. - -### License Standard Notice - -This repository is licensed under ASL v2 or later. - -This source code in this repository is free: you can redistribute it and/or modify it under the terms of the Apache Software License version 2, or (at your option) any later version. - -This source code in this repository is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache Software License for more details. - -You should have received a copy of the Apache Software License along with this program. If not, see http://www.apache.org/licenses/LICENSE-2.0.html - -The source code forked from other open source projects will inherit its license. - -### Privacy Standard Notice - -This repository contains only non-sensitive, publicly available data and information. All material and community participation is covered by the [Disclaimer](https://github.com/CDCgov/template/blob/master/DISCLAIMER.md) and [Code of Conduct](https://github.com/CDCgov/template/blob/master/code-of-conduct.md). For more information about CDC's privacy policy, please visit [http://www.cdc.gov/other/privacy.html](https://www.cdc.gov/other/privacy.html). - -### Contributing Standard Notice - -Anyone is encouraged to contribute to the repository by [forking](https://help.github.com/articles/fork-a-repo) and submitting a pull request. (If you are new to GitHub, you might start with a [basic tutorial](https://help.github.com/articles/set-up-git).) By contributing to this project, you grant a world-wide, royalty-free, perpetual, irrevocable, non-exclusive, transferable license to all users under the terms of the [Apache Software License v2](http://www.apache.org/licenses/LICENSE-2.0.html) or later. - -All comments, messages, pull requests, and other submissions received through CDC including this GitHub page may be subject to applicable federal law, including but not limited to the Federal Records Act, and may be archived. Learn more at [http://www.cdc.gov/other/privacy.html](http://www.cdc.gov/other/privacy.html). - -### Records Management Standard Notice - -This repository is not a source of government records but is a copy to increase collaboration and collaborative potential. All government records will be published through the [CDC web site](http://www.cdc.gov). +# CFA DataOps + +## Overview + +This project provides data tools and low friction access to versioned datasets within CDC's Center for Forecasting and Outbreak Analytics (CFA) virtual private cloud (VPC). It includes end-to-end ETL pipelines for configured data sources, custom workflows, schema validation patterns, and submodules that expand common data visualization and modeling/interpolation patterns. The project demonstrates best practices for data processing within the CFA environment. + +### Key Features + +- Configuration-driven ETL pipelines using TOML files +- SQL-based transformations using DuckDB, or Polar's built-in SQL context +- Azure Blob Storage integration for raw and transformed data storage +- Utilities for time-stamping and template loading +- Data validation and quality checks +- Custom workflows +- Tooling for easy access to Socrata Open Data API (e.i., data.cdc.gov) (copied and modified from [here](https://github.com/CDCgov/cfasodapy)) +- Ability to initialize new dataset catalog libraries and maintain datasets in more than one repo + +## Getting started + +1. Clone the repository and install dependencies with `poetry install` (**requires `poetry >= 2.0`**) +2. Ensure your have access to the relevant resources (e.g., `az login --identity`) +3. See which datasets are available: + ```python + from cfa.dataops import datacat + print(datacat.__namespace_list__) + ``` +4. See which versions of a dataset are available (blob endpoints for datasets will be namedL `.load`, `.extract`, or `.stage_##`): + ```python + print(datacat.{{dataset}}.load.get_versions()) + ``` +5. Get a specific version of a dataset: + ```python + df = datacat.{{dataset}}.load.get_dataframe() + df.glimpse() + ``` + +Read the [Dataset User Guide](docs/dataset_user.md) for more information about accessing datasets. + +Read the [Dataset Developer Guide](docs/dataset_developer.md) for information about how to run ETL to add new versions of an existing datasets and about how to create new datasets. + +## Project admins + +- Thomas Hladish (CDC/OD/ORR/CFA) +- Phillip Rogers (CDC/OD/ORR/CFA)(CTR) +- Ryan Raasch (CDC/OD/ORR/CFA)(CTR) + +--- + +## Disclaimers + +### General Disclaimer + +This repository was created for use by CDC programs to collaborate on public health related projects in support of the [CDC mission](https://www.cdc.gov/about/organization/mission.htm). GitHub is not hosted by the CDC, but is a third party website used by CDC and its partners to share information and collaborate on software. CDC use of GitHub does not imply an endorsement of any one particular service, product, or enterprise. + +### Public Domain Standard Notice + +This repository constitutes a work of the United States Government and is not subject to domestic copyright protection under 17 USC § 105. This repository is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). All contributions to this repository will be released under the CC0 dedication. By submitting a pull request you are agreeing to comply with this waiver of copyright interest. + +### License Standard Notice + +This repository is licensed under ASL v2 or later. + +This source code in this repository is free: you can redistribute it and/or modify it under the terms of the Apache Software License version 2, or (at your option) any later version. + +This source code in this repository is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache Software License for more details. + +You should have received a copy of the Apache Software License along with this program. If not, see http://www.apache.org/licenses/LICENSE-2.0.html + +The source code forked from other open source projects will inherit its license. + +### Privacy Standard Notice + +This repository contains only non-sensitive, publicly available data and information. All material and community participation is covered by the [Disclaimer](https://github.com/CDCgov/template/blob/master/DISCLAIMER.md) and [Code of Conduct](https://github.com/CDCgov/template/blob/master/code-of-conduct.md). For more information about CDC's privacy policy, please visit [http://www.cdc.gov/other/privacy.html](https://www.cdc.gov/other/privacy.html). + +### Contributing Standard Notice + +Anyone is encouraged to contribute to the repository by [forking](https://help.github.com/articles/fork-a-repo) and submitting a pull request. (If you are new to GitHub, you might start with a [basic tutorial](https://help.github.com/articles/set-up-git).) By contributing to this project, you grant a world-wide, royalty-free, perpetual, irrevocable, non-exclusive, transferable license to all users under the terms of the [Apache Software License v2](http://www.apache.org/licenses/LICENSE-2.0.html) or later. + +All comments, messages, pull requests, and other submissions received through CDC including this GitHub page may be subject to applicable federal law, including but not limited to the Federal Records Act, and may be archived. Learn more at [http://www.cdc.gov/other/privacy.html](http://www.cdc.gov/other/privacy.html). + +### Records Management Standard Notice + +This repository is not a source of government records but is a copy to increase collaboration and collaborative potential. All government records will be published through the [CDC web site](http://www.cdc.gov). diff --git a/cfa/dataops/create_catalog/repo_files/.gitignore b/cfa/dataops/create_catalog/repo_files/.gitignore index 85aae97..e4321ca 100644 --- a/cfa/dataops/create_catalog/repo_files/.gitignore +++ b/cfa/dataops/create_catalog/repo_files/.gitignore @@ -1,380 +1,380 @@ -##### -# Exclude many data and output file types by default - -# Data -*.csv -*.tsv -*.parquet -*.dat -*.bin -*.xls -*.xlsx - -# Documents -*.doc -*.docx -*.htm -# *.html -*.ppt -*.pptx -*.pdf - -# Images -*.bmp -*.jpg -*.jpeg -*.gif -*.pdf -# *.png - -# Compressed archives -*.gz -*.tar -*.tgz -*.rar -*.zip - -# Allowlist -# !your_data_file.csv -# !your_data_directory/ - - -##### -# Python -# https://github.com/github/gitignore/blob/main/Python.gitignore - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the -# code is intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in -# version control. However, in case of collaboration, if having -# platform-specific dependencies or dependencies having no cross-platform -# support, pipenv may install dependencies that don't work, or not install all -# needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock -# in version control. This is especially recommended for binary packages to -# ensure reproducibility, and is more commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in -# version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended -# to not include it in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore -# that can be found at -# https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. -# For a more nuclear option (not recommended) you can uncomment the following -# to ignore the entire idea folder. -#.idea/ - -##### -# R -# https://github.com/github/gitignore/blob/main/R.gitignore - -# History files -.Rhistory -.Rapp.history - -# Session Data files -.RData -.RDataTmp - -# User-specific files -.Ruserdata - -# Example code in package build process -*-Ex.R - -# Output files from R CMD build -/*.tar.gz - -# Output files from R CMD check -/*.Rcheck/ - -# RStudio files -.Rproj.user/ - -# produced vignettes -vignettes/*.html -vignettes/*.pdf - -# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 -.httr-oauth - -# knitr and R markdown default cache directories -*_cache/ -/cache/ - -# Temporary files created by R markdown -*.utf8.md -*.knit.md - -# R Environment Variables -.Renviron - -# translation temp files -po/*~ - -# RStudio Connect folder -rsconnect/ - -##### -# Java -# https://github.com/github/gitignore/blob/main/Java.gitignore - -# Compiled class file -*.class - -# Log file -*.log - -# BlueJ files -*.ctxt - -# Mobile Tools for Java (J2ME) -.mtj.tmp/ - -# Package Files # -*.jar -*.war -*.nar -*.ear -*.zip -*.tar.gz -*.rar - -# virtual machine crash logs, see -# http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* -replay_pid* - -##### -# C++ -# https://github.com/github/gitignore/blob/main/C%2B%2B.gitignore - -# Prerequisites -*.d - -# Compiled Object files -*.slo -*.lo -*.o -*.obj - -# Precompiled Headers -*.gch -*.pch - -# Compiled Dynamic libraries -*.so -*.dylib -*.dll - -# Fortran module files -*.mod -*.smod - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - - -##### -# Rust -# https://github.com/github/gitignore/blob/main/Rust.gitignore - -# Generated by Cargo -# will have compiled files and executables -debug/ -target/ - -# Remove Cargo.lock from gitignore if creating an executable, leave it for -# libraries. More information here -# https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -#Cargo.lock - -# These are backup files generated by rustfmt -**/*.rs.bk - -# MSVC Windows builds of rustc generate these, which store debugging information -*.pdb - -##### -# Julia -# https://github.com/github/gitignore/blob/main/Julia.gitignore - -# Files generated by invoking Julia with --code-coverage -*.jl.cov -*.jl.*.cov - -# Files generated by invoking Julia with --track-allocation -*.jl.mem - -# System-specific files and directories generated by the BinaryProvider and -# BinDeps packages. They contain absolute paths specific to the host computer, -# and so should not be committed -deps/deps.jl -deps/build.log -deps/downloads/ -deps/usr/ -deps/src/ - -# File generated by Pkg, the package manager, based on a corresponding -# Project.toml It records a fixed state of all packages used by the project. As -# such, it should not be committed for packages, but should be committed for -# applications that require a static environment. -#Manifest.toml - -*.lock -.vscode -tmp/ -*/logs/ -logs/ - -# mkdocs local build -site/ +##### +# Exclude many data and output file types by default + +# Data +*.csv +*.tsv +*.parquet +*.dat +*.bin +*.xls +*.xlsx + +# Documents +*.doc +*.docx +*.htm +# *.html +*.ppt +*.pptx +*.pdf + +# Images +*.bmp +*.jpg +*.jpeg +*.gif +*.pdf +# *.png + +# Compressed archives +*.gz +*.tar +*.tgz +*.rar +*.zip + +# Allowlist +# !your_data_file.csv +# !your_data_directory/ + + +##### +# Python +# https://github.com/github/gitignore/blob/main/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the +# code is intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in +# version control. However, in case of collaboration, if having +# platform-specific dependencies or dependencies having no cross-platform +# support, pipenv may install dependencies that don't work, or not install all +# needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock +# in version control. This is especially recommended for binary packages to +# ensure reproducibility, and is more commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in +# version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended +# to not include it in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore +# that can be found at +# https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. +# For a more nuclear option (not recommended) you can uncomment the following +# to ignore the entire idea folder. +#.idea/ + +##### +# R +# https://github.com/github/gitignore/blob/main/R.gitignore + +# History files +.Rhistory +.Rapp.history + +# Session Data files +.RData +.RDataTmp + +# User-specific files +.Ruserdata + +# Example code in package build process +*-Ex.R + +# Output files from R CMD build +/*.tar.gz + +# Output files from R CMD check +/*.Rcheck/ + +# RStudio files +.Rproj.user/ + +# produced vignettes +vignettes/*.html +vignettes/*.pdf + +# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 +.httr-oauth + +# knitr and R markdown default cache directories +*_cache/ +/cache/ + +# Temporary files created by R markdown +*.utf8.md +*.knit.md + +# R Environment Variables +.Renviron + +# translation temp files +po/*~ + +# RStudio Connect folder +rsconnect/ + +##### +# Java +# https://github.com/github/gitignore/blob/main/Java.gitignore + +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see +# http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +replay_pid* + +##### +# C++ +# https://github.com/github/gitignore/blob/main/C%2B%2B.gitignore + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + + +##### +# Rust +# https://github.com/github/gitignore/blob/main/Rust.gitignore + +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for +# libraries. More information here +# https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +#Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +##### +# Julia +# https://github.com/github/gitignore/blob/main/Julia.gitignore + +# Files generated by invoking Julia with --code-coverage +*.jl.cov +*.jl.*.cov + +# Files generated by invoking Julia with --track-allocation +*.jl.mem + +# System-specific files and directories generated by the BinaryProvider and +# BinDeps packages. They contain absolute paths specific to the host computer, +# and so should not be committed +deps/deps.jl +deps/build.log +deps/downloads/ +deps/usr/ +deps/src/ + +# File generated by Pkg, the package manager, based on a corresponding +# Project.toml It records a fixed state of all packages used by the project. As +# such, it should not be committed for packages, but should be committed for +# applications that require a static environment. +#Manifest.toml + +*.lock +.vscode +tmp/ +*/logs/ +logs/ + +# mkdocs local build +site/ diff --git a/cfa/dataops/create_catalog/repo_files/_cfa_only/.secrets.baseline b/cfa/dataops/create_catalog/repo_files/_cfa_only/.secrets.baseline index f2b52b1..63f42f9 100644 --- a/cfa/dataops/create_catalog/repo_files/_cfa_only/.secrets.baseline +++ b/cfa/dataops/create_catalog/repo_files/_cfa_only/.secrets.baseline @@ -1,112 +1,112 @@ -{ - "version": "1.4.0", - "plugins_used": [ - { - "name": "ArtifactoryDetector" - }, - { - "name": "AWSKeyDetector" - }, - { - "name": "AzureStorageKeyDetector" - }, - { - "name": "Base64HighEntropyString", - "limit": 4.5 - }, - { - "name": "BasicAuthDetector" - }, - { - "name": "CloudantDetector" - }, - { - "name": "DiscordBotTokenDetector" - }, - { - "name": "GitHubTokenDetector" - }, - { - "name": "HexHighEntropyString", - "limit": 3.0 - }, - { - "name": "IbmCloudIamDetector" - }, - { - "name": "IbmCosHmacDetector" - }, - { - "name": "JwtTokenDetector" - }, - { - "name": "KeywordDetector", - "keyword_exclude": "" - }, - { - "name": "MailchimpDetector" - }, - { - "name": "NpmDetector" - }, - { - "name": "PrivateKeyDetector" - }, - { - "name": "SendGridDetector" - }, - { - "name": "SlackDetector" - }, - { - "name": "SoftlayerDetector" - }, - { - "name": "SquareOAuthDetector" - }, - { - "name": "StripeDetector" - }, - { - "name": "TwilioKeyDetector" - } - ], - "filters_used": [ - { - "path": "detect_secrets.filters.allowlist.is_line_allowlisted" - }, - { - "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", - "min_level": 2 - }, - { - "path": "detect_secrets.filters.heuristic.is_indirect_reference" - }, - { - "path": "detect_secrets.filters.heuristic.is_likely_id_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_lock_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_potential_uuid" - }, - { - "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" - }, - { - "path": "detect_secrets.filters.heuristic.is_sequential_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_swagger_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_templated_secret" - } - ], - "results": {}, - "generated_at": "2023-09-24T19:52:08Z" -} +{ + "version": "1.4.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": {}, + "generated_at": "2023-09-24T19:52:08Z" +} diff --git a/cfa/dataops/create_catalog/repo_files/_cfa_only/ruff.toml b/cfa/dataops/create_catalog/repo_files/_cfa_only/ruff.toml index 36858bc..62943b7 100644 --- a/cfa/dataops/create_catalog/repo_files/_cfa_only/ruff.toml +++ b/cfa/dataops/create_catalog/repo_files/_cfa_only/ruff.toml @@ -1,3 +1,3 @@ -extend-include = ["*.ipynb"] -line-length = 79 -ignore = [] +extend-include = ["*.ipynb"] +line-length = 79 +ignore = [] diff --git a/ruff.toml b/ruff.toml index 36858bc..62943b7 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,3 +1,3 @@ -extend-include = ["*.ipynb"] -line-length = 79 -ignore = [] +extend-include = ["*.ipynb"] +line-length = 79 +ignore = [] diff --git a/tests/conftest.py b/tests/conftest.py index 0784f58..e5321e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,72 +1,72 @@ -import os - -from pytest import fixture - -_here = os.path.abspath(os.path.dirname(__file__)) -test_datasets_dir = os.path.join(_here, "test_datasets") - - -@fixture(scope="session") -def data_dir(tmpdir_factory): - """Fixture to create a temporary data directory for tests.""" - return tmpdir_factory.mktemp("data") - - -@fixture(scope="session") -def mock_write_blob_stream(): - def mock_write_blob_stream( - data, - blob_url: str, - account_name: str, - container_name: str, - append_blob: bool = False, - overwrite: bool = True, - ) -> None: - return None - - return mock_write_blob_stream - - -@fixture(scope="session") -def dataset_ns_map(): - dataset_ns_map = { - "tests": { - "multistage": { - "multistage_test": os.path.join( - test_datasets_dir, "multistage_test.toml" - ) - }, - "etl_test": os.path.join(test_datasets_dir, "etl_test.toml"), - "experiment_test": os.path.join( - test_datasets_dir, "experiment_test.toml" - ), - "reference_test": os.path.join( - test_datasets_dir, "reference_test.toml" - ), - } - } - return dataset_ns_map - - -@fixture(scope="session") -def dataset_defaults(): - return { - "storage": {"account": "account_test", "container": "container_test"}, - "access_ledger": {"path": "_access/test/ledger/"}, - } - - -@fixture(scope="session") -def simple_dataset_ns_map(): - dataset_ns_map = { - "space": { - "example": os.path.join(test_datasets_dir, "etl_test.toml"), - } - } - return dataset_ns_map - - -@fixture(scope="session") -def catalog_parent(tmpdir_factory): - """Fixture to create a temporary data directory for tests.""" - return tmpdir_factory.mktemp("cat_parent") +import os + +from pytest import fixture + +_here = os.path.abspath(os.path.dirname(__file__)) +test_datasets_dir = os.path.join(_here, "test_datasets") + + +@fixture(scope="session") +def data_dir(tmpdir_factory): + """Fixture to create a temporary data directory for tests.""" + return tmpdir_factory.mktemp("data") + + +@fixture(scope="session") +def mock_write_blob_stream(): + def mock_write_blob_stream( + data, + blob_url: str, + account_name: str, + container_name: str, + append_blob: bool = False, + overwrite: bool = True, + ) -> None: + return None + + return mock_write_blob_stream + + +@fixture(scope="session") +def dataset_ns_map(): + dataset_ns_map = { + "tests": { + "multistage": { + "multistage_test": os.path.join( + test_datasets_dir, "multistage_test.toml" + ) + }, + "etl_test": os.path.join(test_datasets_dir, "etl_test.toml"), + "experiment_test": os.path.join( + test_datasets_dir, "experiment_test.toml" + ), + "reference_test": os.path.join( + test_datasets_dir, "reference_test.toml" + ), + } + } + return dataset_ns_map + + +@fixture(scope="session") +def dataset_defaults(): + return { + "storage": {"account": "account_test", "container": "container_test"}, + "access_ledger": {"path": "_access/test/ledger/"}, + } + + +@fixture(scope="session") +def simple_dataset_ns_map(): + dataset_ns_map = { + "space": { + "example": os.path.join(test_datasets_dir, "etl_test.toml"), + } + } + return dataset_ns_map + + +@fixture(scope="session") +def catalog_parent(tmpdir_factory): + """Fixture to create a temporary data directory for tests.""" + return tmpdir_factory.mktemp("cat_parent") diff --git a/tests/test_config_validator.py b/tests/test_config_validator.py index 79fc65a..33ed291 100644 --- a/tests/test_config_validator.py +++ b/tests/test_config_validator.py @@ -1,75 +1,75 @@ -import unittest - -from pydantic import ValidationError - -from cfa.dataops.config_validator import ConfigValidator - - -class TestConfigValidator(unittest.TestCase): - def setUp(self): - self.valid_storage_endpoint = { - "account": "test_account", - "container": "test_container", - "prefix": "dataops/scenarios/test", - } - - self.good_config = { - "properties": { - "name": "test_dataset_name", - "type": "etl", - "automate": False, - "transform_templates": ["test_tf.sql"], - "schema": "datasets/schemas/tests.py", - }, - "source": {"url": "https://data.csv", "uid": "test_uid"}, - "extract": self.valid_storage_endpoint.copy(), - "load": { - **self.valid_storage_endpoint, - "prefix": "dataops/scenarios/transformed/test", - }, - "stage_00": { - **self.valid_storage_endpoint, - "prefix": "dataops/scenarios/stage_00/test", - }, - "stage_01": { - **self.valid_storage_endpoint, - "prefix": "dataops/scenarios/stage_01/test", - }, - } - - def test_valid_config(self): - """Test that a valid configuration passes validation.""" - config = ConfigValidator(**self.good_config) - self.assertIsInstance(config, ConfigValidator) - - def test_invalid_storage_endpoint(self): - """Test that invalid storage endpoints are caught.""" - bad_config = self.good_config.copy() - bad_config["stage_00"] = { - "account": "test_account" - } # Missing required fields - - with self.assertRaises(ValidationError): - ConfigValidator(**bad_config) - - def test_stage_field_validation(self): - """Test that stage fields are properly validated.""" - # Test valid stage field - config = self.good_config.copy() - config["stage_02"] = self.valid_storage_endpoint.copy() - validated = ConfigValidator(**config) - self.assertIsNotNone(validated.stage_02) - - # Test invalid stage field - config["stage_03"] = {"invalid": "endpoint"} - with self.assertRaises(ValidationError): - ConfigValidator(**config) - - def test_optional_stages(self): - """Test that stages are optional.""" - config = self.good_config.copy() - del config["stage_00"] - del config["stage_01"] - - validated = ConfigValidator(**config) - self.assertIsInstance(validated, ConfigValidator) +import unittest + +from pydantic import ValidationError + +from cfa.dataops.config_validator import ConfigValidator + + +class TestConfigValidator(unittest.TestCase): + def setUp(self): + self.valid_storage_endpoint = { + "account": "test_account", + "container": "test_container", + "prefix": "dataops/scenarios/test", + } + + self.good_config = { + "properties": { + "name": "test_dataset_name", + "type": "etl", + "automate": False, + "transform_templates": ["test_tf.sql"], + "schema": "datasets/schemas/tests.py", + }, + "source": {"url": "https://data.csv", "uid": "test_uid"}, + "extract": self.valid_storage_endpoint.copy(), + "load": { + **self.valid_storage_endpoint, + "prefix": "dataops/scenarios/transformed/test", + }, + "stage_00": { + **self.valid_storage_endpoint, + "prefix": "dataops/scenarios/stage_00/test", + }, + "stage_01": { + **self.valid_storage_endpoint, + "prefix": "dataops/scenarios/stage_01/test", + }, + } + + def test_valid_config(self): + """Test that a valid configuration passes validation.""" + config = ConfigValidator(**self.good_config) + self.assertIsInstance(config, ConfigValidator) + + def test_invalid_storage_endpoint(self): + """Test that invalid storage endpoints are caught.""" + bad_config = self.good_config.copy() + bad_config["stage_00"] = { + "account": "test_account" + } # Missing required fields + + with self.assertRaises(ValidationError): + ConfigValidator(**bad_config) + + def test_stage_field_validation(self): + """Test that stage fields are properly validated.""" + # Test valid stage field + config = self.good_config.copy() + config["stage_02"] = self.valid_storage_endpoint.copy() + validated = ConfigValidator(**config) + self.assertIsNotNone(validated.stage_02) + + # Test invalid stage field + config["stage_03"] = {"invalid": "endpoint"} + with self.assertRaises(ValidationError): + ConfigValidator(**config) + + def test_optional_stages(self): + """Test that stages are optional.""" + config = self.good_config.copy() + del config["stage_00"] + del config["stage_01"] + + validated = ConfigValidator(**config) + self.assertIsInstance(validated, ConfigValidator)