Skip to content

Commit 3edb971

Browse files
hrechthrecht
committed
web page and full action
1 parent 52457f1 commit 3edb971

38 files changed

+36606
-31512
lines changed

‎.github/workflows/deploy-site.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Deploy to GitHub Pages
2+
3+
on:
4+
push:
5+
branches: 'main'
6+
paths:
7+
- 'src/**'
8+
- 'static/'
9+
workflow_call:
10+
11+
jobs:
12+
build_site:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout
16+
uses: actions/checkout@v4
17+
with:
18+
ref: main
19+
20+
# If you're using pnpm, add this step then change the commands and cache key below to use `pnpm`
21+
# - name: Install pnpm
22+
# uses: pnpm/action-setup@v3
23+
# with:
24+
# version: 8
25+
26+
- name: Install Node.js
27+
uses: actions/setup-node@v4
28+
with:
29+
node-version: latest
30+
cache: npm
31+
32+
- name: Install dependencies
33+
run: npm install
34+
35+
- name: build
36+
env:
37+
BASE_PATH: '/${{ github.event.repository.name }}'
38+
run: |
39+
npm run build
40+
41+
- name: Upload Artifacts
42+
uses: actions/upload-pages-artifact@v3
43+
with:
44+
path: 'build/'
45+
46+
deploy:
47+
needs: build_site
48+
runs-on: ubuntu-latest
49+
50+
permissions:
51+
pages: write
52+
id-token: write
53+
54+
environment:
55+
name: github-pages
56+
url: ${{ steps.deployment.outputs.page_url }}
57+
58+
steps:
59+
- name: Deploy
60+
id: deployment
61+
uses: actions/deploy-pages@v4

‎.github/workflows/update-data-json.yml‎

Lines changed: 71 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,18 @@ on:
22
workflow_dispatch:
33
schedule:
44
# Run once per day
5-
- cron: "0 13 * * *"
5+
- cron: "30 3,12 * * *"
66

77
name: Check for metadata changes
88

99
jobs:
10-
check:
10+
check_data:
1111
runs-on: ubuntu-latest
12+
outputs:
13+
COMMIT_CHANGES: ${{ steps.passvars.outputs.COMMIT_CHANGES}}
14+
POST_BSKY: ${{ steps.passvars.outputs.POST_BSKY}}
15+
POST_TEXT: ${{ steps.passvars.outputs.POST_TEXT}}
1216
steps:
13-
1417
- uses: actions/checkout@v4
1518

1619
# Install R and dependencies
@@ -22,18 +25,22 @@ jobs:
2225
2326
# Run R script to grab metadata and compare against saved .csv
2427
- name: Run update-metadata.R
25-
run: Rscript scripts/update-metadata.R
26-
27-
# Commit or not based on env variable passed from R
28-
- name: Print $UPDATED_DATA
29-
run: echo "endpoints.csv changed:" $UPDATED_DATA
28+
run: |
29+
Rscript scripts/update-metadata.R
3030
31-
- name: If the cached CSV has not changed, finish
32-
if: env.UPDATED_DATA == 'false'
33-
run: echo "no changes, nothing committed"
31+
- name: Pass vars to output for use in following jobs
32+
id: passvars
33+
run: |
34+
echo "endpoints.csv changed: ${{env.UPDATED_DATA}}"
35+
echo "Should changes be committed: ${{env.COMMIT_CHANGES}}"
36+
echo "Should it post on bsky: ${{env.POST_BSKY}}"
37+
echo "COMMIT_CHANGES=${{env.COMMIT_CHANGES}}" >> $GITHUB_OUTPUT
38+
echo "POST_BSKY=${{env.POST_BSKY}}" >> $GITHUB_OUTPUT
39+
echo "POST_TEXT=${{env.POST_TEXT}}" >> $GITHUB_OUTPUT
3440
35-
- name: Set up commit
36-
if: env.UPDATED_DATA == 'true'
41+
# Commit using message text prepared in R
42+
- name: Commit changes
43+
if: ${{ env.COMMIT_CHANGES == 'true' }}
3744
run: |
3845
# Configure git user
3946
git config user.name "hrecht[bot]"
@@ -42,16 +49,57 @@ jobs:
4249
# Commit files
4350
git add .
4451
45-
- name: Push minor changes update
46-
if: env.UPDATED_DATA == 'true' && env.MAJOR_CHANGES == 'false'
47-
run: |
48-
timestamp=$(TZ=:America/New_York date '+%Y-%m-%d %H:%M')
49-
git commit -m "Minor data change ${timestamp}" || exit 0
52+
git commit -m "${{env.COMMIT_MESSAGE}}" || exit 0
5053
git push
5154
52-
- name: Push major changes update
53-
if: env.UPDATED_DATA == 'true' && env.MAJOR_CHANGES == 'true'
55+
# Pause a little before deploying to github pages
56+
# Otherwise it's happening too quickly to work
57+
- name: Build pause
58+
if: ${{ env.COMMIT_CHANGES == 'true' }}
5459
run: |
55-
timestamp=$(TZ=:America/New_York date '+%Y-%m-%d %H:%M')
56-
git commit -m "Major data change ${timestamp}" || exit 0
57-
git push
60+
sleep 45
61+
echo "Pause before building site"
62+
63+
# Build the page only if changes were committed
64+
build_site:
65+
needs: check_data
66+
if: ${{ needs.check_data.outputs.COMMIT_CHANGES == 'true' }}
67+
uses: ./.github/workflows/deploy-site.yml
68+
secrets: inherit
69+
70+
# Post to Bluesky based on R output
71+
post_bluesky:
72+
runs-on: ubuntu-latest
73+
needs: check_data
74+
if: ${{ needs.check_data.outputs.POST_BSKY == 'true' }}
75+
steps:
76+
- name: Echo vars
77+
run: |
78+
echo "Should it post: ${{ needs.check_data.outputs.POST_BSKY }}"
79+
echo "Post text: ${{ needs.check_data.outputs.POST_TEXT }}"
80+
81+
- name: Set up end of post text
82+
run: |
83+
echo
84+
POST_END=$(cat << EOF
85+
86+
See more: https://www.hrecht.com/census-api-datasets/
87+
EOF
88+
)
89+
echo "POST_END<<EOF" >> $GITHUB_ENV
90+
echo "$POST_END" >> $GITHUB_ENV
91+
echo "EOF" >> $GITHUB_ENV
92+
93+
# Pause a little before posting to give time for page to build
94+
- name: Bluesky pause
95+
run: |
96+
sleep 60
97+
echo "Pause before posting to Bluesky"
98+
99+
- name: Send post
100+
uses: myConsciousness/bluesky-post@v5
101+
with:
102+
text: ${{ needs.check_data.outputs.POST_TEXT }}${{ env.POST_END }}
103+
identifier: ${{ secrets.BLUESKY_IDENTIFIER }}
104+
password: ${{ secrets.BLUESKY_PASSWORD }}
105+

‎.gitignore‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
node_modules
2+
3+
# Output
4+
.output
5+
.vercel
6+
.netlify
7+
.wrangler
8+
/.svelte-kit
9+
/build
10+
11+
# OS
12+
.DS_Store
13+
Thumbs.db
14+
15+
# Env
16+
.env
17+
.env.*
18+
!.env.example
19+
!.env.test
20+
21+
# Vite
22+
vite.config.js.timestamp-*
23+
vite.config.ts.timestamp-*

‎.npmrc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
engine-strict=true

‎.prettierignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Package Managers
2+
package-lock.json
3+
pnpm-lock.yaml
4+
yarn.lock
5+
bun.lock
6+
bun.lockb
7+
8+
# Miscellaneous
9+
/static/

‎.prettierrc‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"useTabs": true,
3+
"singleQuote": true,
4+
"trailingComma": "none",
5+
"printWidth": 100,
6+
"bracketSameLine:": true,
7+
"plugins": ["prettier-plugin-svelte"],
8+
"overrides": [
9+
{
10+
"files": "*.svelte",
11+
"options": {
12+
"parser": "svelte"
13+
}
14+
}
15+
]
16+
}

‎README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,3 @@
22

33
This project tracks the dataset endpoints available in the U.S. Census Bureau APIs. There are no publicly available changelogs or regular announcements of added or removed endpoints. This daily scraper aims to fill that gap.
44

5-
The scraper uses the censusapi R package's [listCensusApis()](https://www.hrecht.com/censusapi/reference/listCensusApis.html) function, with output saved in [data/endpoints.csv](data/endpoints.csv). If there are any changes to `endpoints.csv`, the full [dataset metadata json](https://api.census.gov/data.json) is also saved to [data/data.json](data/data.json). Endpoint additions and removals are also saved to [data/endpoint-changes.csv](data/endpoint-changes.csv).

0 commit comments

Comments
 (0)