Skip to content

Commit 2852182

Browse files
bcwalrusbcwalrus
committed
Sentinel release
- Library for generating the index - Usage documentation - Example notebook to demo detection of extremism
1 parent 5a44b26 commit 2852182

545 files changed

Lines changed: 13342 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.flake8‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[flake8]
2+
max-line-length = 120
3+
# Whitespace before ':' (conflicts with Black)
4+
extend-ignore = E203
5+
exclude = examples/*
6+
per-file-ignores =
7+
# Relax rules for test files
8+
tests/*:D100,D101,D102,D103,D104,D105,D106,D107,D202,D205,D400,D401,D403,D415

‎.gitattributes‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
**/data/**/* filter=lfs diff=lfs merge=lfs -text
2+
**/*.csv filter=lfs diff=lfs merge=lfs -text
3+
**/*.safetensors filter=lfs diff=lfs merge=lfs -text

‎.github/workflows/docs.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: Documentation
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches: [ main ]
9+
paths:
10+
- 'src/**/*.py'
11+
- 'docs/**'
12+
workflow_dispatch:
13+
14+
permissions:
15+
contents: read
16+
pages: write
17+
id-token: write
18+
pull-requests: write
19+
20+
# Allow only one concurrent deployment
21+
concurrency:
22+
group: "pages"
23+
cancel-in-progress: true
24+
25+
jobs:
26+
check-docs:
27+
runs-on: ubuntu-latest
28+
if: github.event_name == 'pull_request'
29+
# Make this job informational only - don't block PR
30+
continue-on-error: true
31+
steps:
32+
- name: Checkout repository
33+
uses: actions/checkout@v4
34+
35+
- name: Set up Python
36+
uses: actions/setup-python@v5
37+
with:
38+
python-version: '3.11'
39+
cache: 'pip'
40+
41+
- name: Install Poetry
42+
run: |
43+
curl -sSL https://install.python-poetry.org | python3 -
44+
echo "$HOME/.local/bin" >> $GITHUB_PATH
45+
46+
- name: Install dependencies
47+
run: |
48+
poetry install --with docs
49+
50+
- name: Check documentation sync
51+
run: |
52+
cd ${{ github.workspace }}
53+
# Generate the documentation
54+
poetry run python docs/generate_docs.py
55+
56+
# Check if any files were modified (docs are out of sync)
57+
if [ -n "$(git status --porcelain docs/source/)" ]; then
58+
echo "::warning::Documentation is out of sync with the codebase!"
59+
echo "::warning::Please run 'python docs/generate_docs.py' and commit the updated documentation files."
60+
exit 1
61+
fi
62+
echo "Documentation is in sync with the codebase."
63+
64+
- name: Notify about out-of-sync docs
65+
if: failure()
66+
uses: actions/github-script@v6
67+
with:
68+
script: |
69+
github.rest.issues.createComment({
70+
issue_number: context.issue.number,
71+
owner: context.repo.owner,
72+
repo: context.repo.repo,
73+
body: '⚠️ **Documentation is out of sync with the code!**\n\nPlease run `python docs/generate_docs.py` and commit the updated documentation files.'
74+
})
75+
76+
build:
77+
runs-on: ubuntu-latest
78+
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
79+
needs: [check-docs]
80+
# Make this job informational only for PRs - don't block PR
81+
continue-on-error: ${{ github.event_name == 'pull_request' }}
82+
steps:
83+
- name: Checkout repository
84+
uses: actions/checkout@v4
85+
86+
- name: Set up Python
87+
uses: actions/setup-python@v5
88+
with:
89+
python-version: '3.11'
90+
cache: 'pip'
91+
92+
- name: Install Poetry
93+
run: |
94+
curl -sSL https://install.python-poetry.org | python3 -
95+
echo "$HOME/.local/bin" >> $GITHUB_PATH
96+
97+
- name: Install dependencies
98+
run: |
99+
poetry install --with docs
100+
101+
- name: Generate RST files
102+
run: |
103+
cd ${{ github.workspace }}
104+
poetry run python docs/generate_docs.py
105+
106+
- name: Build documentation
107+
run: |
108+
cd ${{ github.workspace }}/docs
109+
poetry run sphinx-build -b html source build/html
110+
111+
- name: Setup Pages
112+
uses: actions/configure-pages@v4
113+
114+
- name: Upload artifact
115+
uses: actions/upload-pages-artifact@v3
116+
with:
117+
path: ${{ github.workspace }}/docs/build/html
118+
119+
deploy:
120+
# Only deploy on push to main or workflow_dispatch, skip on PRs
121+
if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'
122+
needs: build
123+
environment:
124+
name: github-pages
125+
url: ${{ steps.deployment.outputs.page_url }}
126+
runs-on: ubuntu-latest
127+
steps:
128+
- name: Deploy to GitHub Pages
129+
id: deployment
130+
uses: actions/deploy-pages@v4

‎.github/workflows/publish.yml‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
name: Publish Package
2+
3+
on:
4+
push:
5+
tags:
6+
- "v*" # Trigger on tags that start with 'v', such as v0.1.0
7+
8+
jobs:
9+
verify-version:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout repository
13+
uses: actions/checkout@v4
14+
15+
- name: Set up Python
16+
uses: actions/setup-python@v5
17+
with:
18+
python-version: '3.11'
19+
cache: 'pip'
20+
21+
- name: Install Poetry
22+
run: |
23+
curl -sSL https://install.python-poetry.org | python3 -
24+
echo "$HOME/.local/bin" >> $GITHUB_PATH
25+
26+
- name: Get tag version
27+
id: get_version
28+
run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
29+
30+
- name: Extract version from pyproject.toml
31+
id: get_toml_version
32+
run: |
33+
TOML_VERSION=$(grep -m 1 "version" pyproject.toml | cut -d '"' -f2)
34+
echo "TOML_VERSION=${TOML_VERSION}" >> $GITHUB_ENV
35+
echo "Version from pyproject.toml: ${TOML_VERSION}"
36+
echo "Version from tag: ${{ env.VERSION }}"
37+
38+
- name: Verify versions match
39+
run: |
40+
if [ "${{ env.VERSION }}" != "${{ env.TOML_VERSION }}" ]; then
41+
echo "Error: Version in pyproject.toml (${{ env.TOML_VERSION }}) does not match tag version (${{ env.VERSION }})"
42+
exit 1
43+
else
44+
echo "Versions match!"
45+
fi
46+
47+
build:
48+
needs: verify-version
49+
runs-on: ubuntu-latest
50+
steps:
51+
- name: Checkout repository
52+
uses: actions/checkout@v4
53+
54+
- name: Set up Python
55+
uses: actions/setup-python@v5
56+
with:
57+
python-version: '3.11'
58+
cache: 'pip'
59+
60+
- name: Install Poetry
61+
run: |
62+
curl -sSL https://install.python-poetry.org | python3 -
63+
echo "$HOME/.local/bin" >> $GITHUB_PATH
64+
65+
- name: Install dependencies
66+
run: |
67+
poetry install --with dev,docs
68+
69+
- name: Run tests
70+
run: |
71+
poetry run pytest tests/
72+
73+
- name: Build package
74+
run: |
75+
poetry build
76+
77+
- name: Upload distribution artifacts
78+
uses: actions/upload-artifact@v3
79+
with:
80+
name: dist
81+
path: dist/
82+
retention-days: 7
83+
84+
publish-docs:
85+
needs: build
86+
runs-on: ubuntu-latest
87+
permissions:
88+
contents: read
89+
pages: write
90+
id-token: write
91+
92+
environment:
93+
name: github-pages
94+
url: ${{ steps.deployment.outputs.page_url }}
95+
96+
steps:
97+
- name: Checkout repository
98+
uses: actions/checkout@v4
99+
100+
- name: Set up Python
101+
uses: actions/setup-python@v5
102+
with:
103+
python-version: '3.11'
104+
cache: 'pip'
105+
106+
- name: Install Poetry
107+
run: |
108+
curl -sSL https://install.python-poetry.org | python3 -
109+
echo "$HOME/.local/bin" >> $GITHUB_PATH
110+
111+
- name: Install dependencies
112+
run: |
113+
poetry install --with docs
114+
115+
- name: Generate RST files
116+
run: |
117+
cd ${{ github.workspace }}
118+
poetry run python docs/generate_docs.py
119+
120+
- name: Build documentation
121+
run: |
122+
cd ${{ github.workspace }}/docs
123+
poetry run sphinx-build -b html source build/html
124+
125+
- name: Setup Pages
126+
uses: actions/configure-pages@v4
127+
128+
- name: Upload artifact
129+
uses: actions/upload-pages-artifact@v3
130+
with:
131+
path: ${{ github.workspace }}/docs/build/html
132+
133+
- name: Deploy to GitHub Pages
134+
id: deployment
135+
uses: actions/deploy-pages@v4
136+
137+
publish-pypi:
138+
needs: publish-docs
139+
runs-on: ubuntu-latest
140+
environment:
141+
name: pypi
142+
permissions:
143+
id-token: write # For trusted publishing to PyPI
144+
145+
steps:
146+
- name: Download distribution artifacts
147+
uses: actions/download-artifact@v3
148+
with:
149+
name: dist
150+
path: dist/
151+
152+
- name: Publish to PyPI
153+
uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/test.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Run Tests
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
test:
11+
runs-on: ubuntu-latest
12+
strategy:
13+
matrix:
14+
python-version: ['3.9', '3.10', '3.11', '3.12']
15+
16+
steps:
17+
- uses: actions/checkout@v4
18+
- name: Set up Python ${{ matrix.python-version }}
19+
uses: actions/setup-python@v4
20+
with:
21+
python-version: ${{ matrix.python-version }}
22+
23+
- name: Install Poetry and dependencies
24+
run: |
25+
pip install poetry>=1.8.0
26+
poetry config virtualenvs.create true
27+
poetry config virtualenvs.in-project true
28+
poetry install --with dev --extras "sbert"
29+
30+
- name: Run tests with pytest
31+
run: |
32+
poetry run pytest tests/ --cov=sentinel --cov-report=xml
33+
34+
- name: Upload coverage to Codecov
35+
uses: codecov/codecov-action@v3
36+
with:
37+
file: ./coverage.xml
38+
fail_ci_if_error: false

‎.gitignore‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
6+
# Vim and emacs files
7+
*~
8+
*.swp
9+
*.swo
10+
.DS_Store
11+
.ipynb_checkpoints/

‎.pre-commit-config.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
repos:
2+
- repo: https://github.com/pycqa/flake8
3+
rev: 7.2.0
4+
hooks:
5+
- id: flake8
6+
args: [--config=.flake8]
7+
additional_dependencies: [flake8-docstrings]
8+
9+
- repo: https://github.com/psf/black
10+
rev: 25.1.0
11+
hooks:
12+
- id: black
13+
language_version: python3
14+
15+
- repo: https://github.com/pre-commit/pre-commit-hooks
16+
rev: v4.5.0
17+
hooks:
18+
- id: trailing-whitespace
19+
- id: end-of-file-fixer
20+
- id: check-yaml
21+
- id: check-added-large-files
22+
23+
- repo: local
24+
hooks:
25+
- id: update-docs
26+
name: Update Documentation
27+
entry: python docs/update_docs_hook.py
28+
language: system
29+
pass_filenames: false
30+
files: 'src/.*\.py$'

0 commit comments

Comments
 (0)