Skip to content

Commit 95e2789

Browse files
mxclmxcl
committed
.some(mash)
1 parent fda3a3f commit 95e2789

12 files changed

Lines changed: 887 additions & 0 deletions

File tree

‎.github/dependabot.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: github-actions
4+
directory: /
5+
schedule:
6+
interval: weekly

‎.github/scripts/build.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/usr/bin/env -S pkgx deno run --allow-read --allow-write
2+
3+
import * as flags from "https://deno.land/std@0.206.0/flags/mod.ts";
4+
import { Path } from "https://deno.land/x/libpkgx@v0.16.0/mod.ts";
5+
import { Script } from "./index.ts";
6+
7+
const args = flags.parse(Deno.args);
8+
const indir = (s => Path.abs(s) ?? Path.cwd().join(s))(args['input'])
9+
const outdir = (s => Path.abs(s) ?? Path.cwd().join(s))(args['output'])
10+
const index_json_path = args['index-json']
11+
12+
if (!indir || !outdir || !index_json_path) {
13+
console.error(`usage: build.ts --input <path> --output <path> --index-json <path>`);
14+
Deno.exit(64);
15+
}
16+
17+
const scripts = JSON.parse(Deno.readTextFileSync(index_json_path)).scripts as Script[]
18+
19+
const categories: Record<string, Script[]> = {}
20+
const users: Record<string, Script[]> = {}
21+
22+
for (const script of scripts) {
23+
if (script.category) {
24+
categories[script.category] ??= []
25+
categories[script.category].push(script)
26+
}
27+
const user = script.fullname.split('/')[0]
28+
users[user] ??= []
29+
users[user].push(script)
30+
}
31+
32+
// sort each entry in categories and users by the script birthtime
33+
for (const scripts of Object.values(categories)) {
34+
scripts.sort((a, b) => new Date(b.birthtime).getTime() - new Date(a.birthtime).getTime());
35+
}
36+
for (const scripts of Object.values(users)) {
37+
scripts.sort((a, b) => new Date(b.birthtime).getTime() - new Date(a.birthtime).getTime());
38+
}
39+
40+
for (const category in categories) {
41+
const d = outdir.join(category)
42+
const scripts = categories[category].filter(({description}) => description)
43+
d.mkdir('p').join('index.json').write({ json: { scripts }, force: true, space: 2 })
44+
}
45+
46+
for (const user in users) {
47+
const d = outdir.join('u', user)
48+
const scripts = users[user].filter(({description}) => description)
49+
d.mkdir('p').join('index.json').write({ json: { scripts }, force: true, space: 2 })
50+
}
51+
52+
for (const script of scripts) {
53+
console.error(script)
54+
const [user, name] = script.fullname.split('/')
55+
const { category } = script
56+
const gh_slug = new URL(script.url).pathname.split('/').slice(1, 3).join('/')
57+
const infile = indir.join(gh_slug, 'scripts', name)
58+
59+
infile.cp({ into: outdir.join('u', user).mkdir('p') })
60+
61+
const leaf = infile.basename().split('-').slice(1).join('-')
62+
if (category && !outdir.join(category, leaf).exists()) { // not already snagged
63+
infile.cp({ to: outdir.join(category, leaf) })
64+
}
65+
}
66+
67+
outdir.join('u/index.json').write({ json: { users }, force: true, space: 2})

‎.github/scripts/index.ts‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
#!/usr/bin/env -S pkgx deno run --allow-run=bash --allow-read=.
2+
3+
import { join, basename, dirname } from "https://deno.land/std@0.206.0/path/mod.ts";
4+
import { walk, exists } from "https://deno.land/std@0.206.0/fs/mod.ts";
5+
import * as flags from "https://deno.land/std@0.206.0/flags/mod.ts";
6+
7+
if (import.meta.main) {
8+
const args = flags.parse(Deno.args);
9+
const inputdir = args['input']
10+
11+
if (!inputdir) {
12+
console.error(`usage: index.ts --input <path>`);
13+
Deno.exit(1);
14+
}
15+
16+
Deno.chdir(inputdir);
17+
18+
const scripts: Script[] = []
19+
for await (const slug of iterateGitRepos('.')) {
20+
console.error(`iterating: ${slug}`);
21+
scripts.push(...await get_metadata(slug));
22+
}
23+
24+
scripts.sort((a, b) => b.birthtime.getTime() - a.birthtime.getTime());
25+
26+
const categories = (() => {
27+
const categories: Record<string, number> = {}
28+
for (const script of scripts) {
29+
if (script.category && script.description) {
30+
categories[script.category] ??= 0;
31+
categories[script.category]++;
32+
}
33+
}
34+
return Object.keys(categories)
35+
})();
36+
37+
38+
console.log(JSON.stringify({ scripts, categories }, null, 2));
39+
}
40+
41+
////////////////////////////////////////////////////////////////////// lib
42+
async function extractMarkdownSection(filePath: string, sectionTitle: string): Promise<string | undefined> {
43+
const data = await Deno.readTextFile(filePath);
44+
const lines = data.split('\n');
45+
let capturing = false;
46+
let sectionContent = '';
47+
48+
for (const line of lines) {
49+
if (line.startsWith('## ')) {
50+
if (capturing) {
51+
break; // stop if we reach another ## section
52+
} else if (normalize_title(line.slice(3)) == normalize_title(sectionTitle)) {
53+
capturing = true;
54+
} else if (line.slice(3).trim() == mash_title(sectionTitle)) {
55+
capturing = true;
56+
}
57+
} else if (capturing) {
58+
sectionContent += line + '\n';
59+
}
60+
}
61+
62+
return chuzzle(sectionContent);
63+
64+
function normalize_title(input: string) {
65+
return input.toLowerCase().replace(/[^a-z0-9]/g, '').trim();
66+
}
67+
68+
function mash_title(input: string) {
69+
const [category, ...name] = input.trim().split('-')
70+
return `\`mash ${category} ${name.join('-')}\``
71+
}
72+
}
73+
74+
export interface Script {
75+
fullname: string
76+
birthtime: Date
77+
description?: string
78+
avatar: string
79+
url: string
80+
category?: string
81+
README?: string
82+
cmd: string
83+
}
84+
85+
async function* iterateGitRepos(basePath: string): AsyncIterableIterator<string> {
86+
for await (const entry of walk(basePath, { maxDepth: 2 })) {
87+
if (entry.isDirectory && await exists(join(entry.path, '.git'))) {
88+
yield entry.path;
89+
}
90+
}
91+
}
92+
93+
function chuzzle(ln: string): string | undefined {
94+
const out = ln.trim()
95+
return out || undefined;
96+
}
97+
98+
async function get_metadata(slug: string) {
99+
100+
const cmdString = `git -C '${slug}' log --pretty=format:'%H %aI' --name-only --diff-filter=AR -- scripts`;
101+
102+
const process = Deno.run({
103+
cmd: ["bash", "-c", cmdString],
104+
stdout: "piped"
105+
});
106+
107+
const output = new TextDecoder().decode(await process.output());
108+
await process.status();
109+
process.close();
110+
111+
const lines = chuzzle(output)?.split('\n') ?? [];
112+
const rv: Script[] = []
113+
let currentCommitDate: string | undefined;
114+
115+
for (let line of lines) {
116+
line = line.trim()
117+
118+
if (line.includes(' ')) { // Detect lines with commit hash and date
119+
currentCommitDate = line.split(' ')[1];
120+
} else if (line && currentCommitDate) {
121+
const filename = join(slug, line)
122+
if (!await exists(filename)) {
123+
// the file used to exist but has been deleted
124+
console.warn("skipping deleted: ", filename, line)
125+
continue
126+
}
127+
128+
console.error(line)
129+
130+
const repo_metadata = JSON.parse(await Deno.readTextFile(join(slug, 'metadata.json')))
131+
132+
const README = await extractMarkdownSection(join(slug, 'README.md'), basename(filename));
133+
const birthtime = new Date(currentCommitDate!);
134+
const avatar = repo_metadata.avatar
135+
const fullname = join(dirname(slug), ...stem(filename))
136+
const url = repo_metadata.url +'/scripts/' + basename(filename)
137+
const category = (([x, y]) => x?.length > 0 && y ? x : undefined)(basename(filename).split("-"))
138+
const description = README ? extract_description(README) : undefined
139+
const cmd = category ? `mash ${category} ${basename(filename).split('-').slice(1).join('-')}` : `mash ${fullname}`
140+
141+
rv.push({ fullname, birthtime, description, avatar, url, category, README, cmd })
142+
}
143+
}
144+
145+
return rv;
146+
147+
function stem(filename: string): string[] {
148+
const base = basename(filename)
149+
const parts = base.split('.')
150+
if (parts.length == 1) {
151+
return parts.slice(0, 1)
152+
} else {
153+
return parts.slice(0, -1) // no extension, but allow eg. foo.bar.js to be foo.bar
154+
}
155+
}
156+
}
157+
158+
function extract_description(input: string) {
159+
const regex = /^(.*?)\n#|^.*$/ms;
160+
const match = regex.exec(input);
161+
return match?.[1]?.trim();
162+
}

‎.github/scripts/trawl.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/usr/bin/env -S pkgx deno run --allow-run --allow-net --allow-env=GH_TOKEN --allow-write=.
2+
3+
import * as flags from "https://deno.land/std@0.206.0/flags/mod.ts";
4+
5+
const args = flags.parse(Deno.args);
6+
const outdir = args['out']
7+
8+
const ghToken = Deno.env.get("GH_TOKEN");
9+
if (!ghToken) {
10+
console.error("error: GitHub token is required. Set the GH_TOKEN environment variable.");
11+
Deno.exit(1)
12+
}
13+
14+
Deno.mkdirSync(outdir, { recursive: true });
15+
16+
async function cloneAllForks(user: string, repo: string) {
17+
let page = 1;
18+
while (true) {
19+
const response = await fetch(`https://api.github.com/repos/${user}/${repo}/forks?page=${page}`, {
20+
headers: {
21+
"Authorization": `token ${ghToken}`
22+
}
23+
});
24+
25+
if (!response.ok) {
26+
throw new Error(`err: ${response.statusText}`);
27+
}
28+
29+
const forks = await response.json();
30+
if (forks.length === 0) {
31+
break; // No more forks
32+
}
33+
34+
for (const fork of forks) {
35+
await clone(fork)
36+
37+
Deno.writeTextFileSync(`${outdir}/${fork.full_name}/metadata.json`, JSON.stringify({
38+
stars: fork.stargazers_count,
39+
license: fork.license?.spdx_id,
40+
avatar: fork.owner.avatar_url,
41+
url: fork.html_url + '/blob/' + fork.default_branch
42+
}, null, 2))
43+
}
44+
45+
page++;
46+
}
47+
}
48+
49+
async function clone({clone_url, full_name, ...fork}: any) {
50+
console.log(`Cloning ${clone_url}...`);
51+
const proc = new Deno.Command("git", { args: ["-C", outdir, "clone", clone_url, full_name]}).spawn()
52+
if (!(await proc.status).success) {
53+
throw new Error(`err: ${await proc.status}`)
54+
}
55+
}
56+
57+
await cloneAllForks('pkgxdev', 'mash');
58+
59+
// we have some general utility scripts here
60+
await clone({clone_url: 'https://github.com/pkgxdev/mash.git', full_name: 'pkgxdev/mash'});
61+
// deploy expects this and fails otherwise
62+
Deno.writeTextFileSync(`${outdir}/pkgxdev/mash/metadata.json`, `{
63+
"stars": 0,
64+
"license": "Apache-2.0",
65+
"avatar": "https://avatars.githubusercontent.com/u/140643783?v=4",
66+
"url": "https://github.com/pkgxdev/mash/blob/main"
67+
}`)

‎.github/workflows/ci.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
on:
2+
pull_request:
3+
paths: mash
4+
5+
jobs:
6+
test:
7+
runs-on: ubuntu-latest
8+
steps:
9+
- uses: actions/checkout@v4
10+
- uses: pkgxdev/setup@v2
11+
- run: ./mash pkgxdev/demo
12+
- run: ./mash pkgxdev/demo # check cache route works too

‎.github/workflows/deploy.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
on:
2+
push:
3+
branches: main
4+
paths:
5+
- .github/workflows/deploy.yml
6+
- .github/scripts/*
7+
pull_request:
8+
paths:
9+
- .github/workflows/deploy.yml
10+
schedule:
11+
- cron: '23 * * * *'
12+
workflow_dispatch:
13+
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.ref }}
16+
cancel-in-progress: true
17+
18+
jobs:
19+
build:
20+
if: github.repository == 'pkgxdev/mash'
21+
runs-on: ubuntu-latest
22+
steps:
23+
- uses: actions/checkout@v4
24+
- uses: pkgxdev/setup@v2
25+
26+
- run: .github/scripts/trawl.ts --out ./build
27+
env:
28+
GH_TOKEN: ${{ github.token }}
29+
30+
- run: |
31+
mkdir out
32+
.github/scripts/index.ts --input ./build > ./out/index.json
33+
34+
- run: .github/scripts/build.ts --input ./build --output ./out --index-json ./out/index.json
35+
36+
- uses: actions/configure-pages@v4
37+
- uses: actions/upload-pages-artifact@v3
38+
with:
39+
path: out
40+
41+
deploy:
42+
needs: build
43+
runs-on: ubuntu-latest
44+
if: ${{ github.event_name != 'pull_request' }}
45+
environment:
46+
name: github-pages
47+
url: ${{ steps.deployment.outputs.page_url }}
48+
permissions:
49+
pages: write # to deploy to Pages
50+
id-token: write # to verify the deployment originates from an appropriate source
51+
steps:
52+
- uses: actions/deploy-pages@v4
53+
id: deployment

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
.DS_Store
2+
/out
3+
/build

‎.vscode/settings.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"deno.enable": true,
3+
"deno.lint": true,
4+
"deno.unstable": true
5+
}

0 commit comments

Comments
 (0)