Skip to content

Commit 302839d

Browse files
hleepflhleepfl
authored
Knip part 2 (#1169)
* chore: fix gitignore patterns and format previously gitignored code * chore: make knip pass - adjust dependencies - remove unused exports - refine knip config * ci: add knip to ci
1 parent e9c13a4 commit 302839d

34 files changed

Lines changed: 1461 additions & 1375 deletions

‎.github/workflows/_static-analysis.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,10 @@ jobs:
99
- uses: actions/checkout@v7
1010
- uses: ./.github/actions/setup-node
1111
- run: pnpm run lint
12+
13+
knip:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v7
17+
- uses: ./.github/actions/setup-node
18+
- run: pnpm exec knip

‎.gitignore‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
# dependencies
22
/node_modules/
3-
/docs/examples/node_modules/
43
/*/node_modules/
4+
/docs/examples/node_modules/
55

66
# stored trained models
7-
models/
7+
/server/models/
8+
/models/
89

910
# tsc built
10-
dist/
11+
/*/dist/
12+
/docs/examples/dist/
1113
# tsc --build state
1214
*.tsbuildinfo
1315

‎.knip.json‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,27 @@
11
{
22
"workspaces": {
33
".": {
4-
"ignore": ["docs/**"]
4+
"ignore": ["docs/**"],
5+
"ignoreDependencies": ["typescript-eslint"]
56
},
67
"webapp": {
7-
"ignore": ["env.d.ts"]
8+
"ignore": ["env.d.ts"],
9+
"ignoreDependencies": ["buffer"]
10+
},
11+
"isomorphic-wrtc": {},
12+
"cli": {
13+
"entry": [
14+
"src/benchmark_gpt.ts",
15+
"src/hellaswag_gpt.ts",
16+
"src/train_gpt.ts"
17+
]
18+
},
19+
"onnx-converter": {
20+
"entry": [
21+
"src/convert_onnx.ts",
22+
"src/protobuf/onnx-proto.d.ts",
23+
"src/protobuf/onnx.d.ts"
24+
]
825
}
926
}
1027
}

‎cli/package.json‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,10 @@
1919
"@epfml/discojs-node": "workspace:",
2020
"@tensorflow/tfjs-node": "catalog:",
2121
"immutable": "catalog:",
22-
"server": "workspace:",
23-
"tslib": "2.8.1"
22+
"server": "workspace:"
2423
},
2524
"devDependencies": {
2625
"nodemon": "3.1.14",
27-
"ts-command-line-args": "2.5.1",
28-
"vitest": "catalog:"
26+
"ts-command-line-args": "2.5.1"
2927
}
3028
}

‎discojs-node/package.json‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919
"homepage": "https://github.com/epfml/disco#readme",
2020
"dependencies": {
2121
"@epfml/discojs": "workspace:",
22-
"@roamhq/wrtc": "0.10.0",
23-
"@tensorflow/tfjs-node": "catalog:",
2422
"csv-parse": "6.2.0",
2523
"debug": "catalog:",
2624
"node-fetch": "2.7.0",
@@ -31,7 +29,6 @@
3129
"@types/node-fetch": "2.6.13",
3230
"nodemon": "3.1.14",
3331
"tmp-promise": "3.0.3",
34-
"ts-node": "10.9.2",
3532
"vitest": "catalog:"
3633
}
3734
}

‎discojs-web/package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
"homepage": "https://github.com/epfml/disco#readme",
2020
"dependencies": {
2121
"@epfml/discojs": "workspace:",
22-
"@tensorflow/tfjs": "catalog:",
2322
"papaparse": "5.5.3"
2423
},
2524
"devDependencies": {

‎discojs/package.json‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,18 @@
2323
"@jimp/plugin-resize": "1.6.1",
2424
"@msgpack/msgpack": "3.1.3",
2525
"@tensorflow/tfjs": "catalog:",
26-
"@tensorflow/tfjs-node": "catalog:",
2726
"@xenova/transformers": "2.17.2",
2827
"chalk": "4.1.2",
2928
"debug": "catalog:",
3029
"immutable": "catalog:",
3130
"isomorphic-ws": "5.0.0",
3231
"simple-peer": "9.11.1",
33-
"tslib": "2.8.1",
3432
"ws": "8.21.0",
3533
"zod": "catalog:"
3634
},
3735
"devDependencies": {
3836
"@types/simple-peer": "9.11.9",
3937
"nodemon": "3.1.14",
40-
"ts-node": "10.9.2",
4138
"vitest": "catalog:"
4239
}
4340
}

‎discojs/src/models/gpt/config.ts‎

Lines changed: 54 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,39 @@
11
type GPTModelType =
2-
| 'gpt2'
3-
| 'gpt2-medium'
4-
| 'gpt2-large'
5-
| 'gpt2-xl'
6-
| 'gpt-mini'
7-
| 'gpt-micro'
8-
| 'gpt-nano'
2+
| "gpt2"
3+
| "gpt2-medium"
4+
| "gpt2-large"
5+
| "gpt2-xl"
6+
| "gpt-mini"
7+
| "gpt-micro"
8+
| "gpt-nano";
99

1010
export type GPTConfig = {
11-
lr: number
12-
contextLength: number
13-
vocabSize?: number
14-
modelType: GPTModelType
15-
evaluate?: boolean
16-
maxEvalBatches?: number
17-
evaluateEvery?: number
18-
maxIter?: number
19-
weightDecay?: number
20-
verbose?: 0 | 1
21-
debug?: boolean
22-
attnDrop?: number
23-
residDrop?: number
24-
embdDrop?: number
25-
nLayer?: number
26-
nHead?: number
27-
nEmbd?: number
28-
seed?: number,
29-
}
11+
lr: number;
12+
contextLength: number;
13+
vocabSize?: number;
14+
modelType: GPTModelType;
15+
evaluate?: boolean;
16+
maxEvalBatches?: number;
17+
evaluateEvery?: number;
18+
maxIter?: number;
19+
weightDecay?: number;
20+
verbose?: 0 | 1;
21+
debug?: boolean;
22+
attnDrop?: number;
23+
residDrop?: number;
24+
embdDrop?: number;
25+
nLayer?: number;
26+
nHead?: number;
27+
nEmbd?: number;
28+
seed?: number;
29+
};
3030
// for a benchmark of performance, see https://github.com/epfml/disco/pull/659
3131
export const DefaultGPTConfig: Required<GPTConfig> = {
3232
lr: 0.001,
3333
weightDecay: 0,
3434
maxIter: 10,
3535
verbose: 0,
36-
modelType: 'gpt-nano',
36+
modelType: "gpt-nano",
3737
evaluate: true,
3838
maxEvalBatches: 12,
3939
evaluateEvery: 100,
@@ -47,50 +47,50 @@ export const DefaultGPTConfig: Required<GPTConfig> = {
4747
nHead: 3,
4848
nEmbd: 48,
4949
seed: Math.random(),
50-
}
50+
};
5151

5252
export type ModelSize = {
53-
nLayer: number
54-
nHead: number
55-
nEmbd: number
56-
}
53+
nLayer: number;
54+
nHead: number;
55+
nEmbd: number;
56+
};
5757

58-
export function getModelSizes (modelType: GPTModelType): Required<ModelSize> {
58+
export function getModelSizes(modelType: GPTModelType): Required<ModelSize> {
5959
switch (modelType) {
60-
case 'gpt2':
61-
return { nLayer: 12, nHead: 12, nEmbd: 768 }
62-
case 'gpt2-medium':
63-
return { nLayer: 24, nHead: 16, nEmbd: 1024 }
64-
case 'gpt2-large':
65-
return { nLayer: 36, nHead: 20, nEmbd: 1280 }
66-
case 'gpt2-xl':
67-
return { nLayer: 48, nHead: 25, nEmbd: 1600 }
68-
case 'gpt-mini':
69-
return { nLayer: 6, nHead: 6, nEmbd: 192 }
70-
case 'gpt-micro':
71-
return { nLayer: 4, nHead: 4, nEmbd: 128 }
72-
case 'gpt-nano':
73-
return { nLayer: 3, nHead: 3, nEmbd: 48 }
60+
case "gpt2":
61+
return { nLayer: 12, nHead: 12, nEmbd: 768 };
62+
case "gpt2-medium":
63+
return { nLayer: 24, nHead: 16, nEmbd: 1024 };
64+
case "gpt2-large":
65+
return { nLayer: 36, nHead: 20, nEmbd: 1280 };
66+
case "gpt2-xl":
67+
return { nLayer: 48, nHead: 25, nEmbd: 1600 };
68+
case "gpt-mini":
69+
return { nLayer: 6, nHead: 6, nEmbd: 192 };
70+
case "gpt-micro":
71+
return { nLayer: 4, nHead: 4, nEmbd: 128 };
72+
case "gpt-nano":
73+
return { nLayer: 3, nHead: 3, nEmbd: 48 };
7474
}
7575
}
7676

7777
export interface GenerationConfig {
7878
// take random token weighted by its probability
7979
// If false, predict the token with the highest probability.
80-
doSample: boolean
80+
doSample: boolean;
8181
// the generation temperature (higher means more randomness).
8282
// Set to 0 for greedy decoding.
83-
temperature: number
84-
// only consider the topk most likely tokens for sampling.
83+
temperature: number;
84+
// only consider the topk most likely tokens for sampling.
8585
// used if doSample is true.
86-
topk: number
86+
topk: number;
8787
// random seed for sampling.
88-
seed: number
88+
seed: number;
8989
}
9090

9191
export const DefaultGenerationConfig: Required<GenerationConfig> = {
9292
temperature: 1.0,
9393
doSample: false,
9494
seed: Math.random(),
95-
topk: 50
96-
}
95+
topk: 50,
96+
};

‎discojs/src/models/gpt/evaluate.ts‎

Lines changed: 47 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,62 @@
1-
import * as tf from '@tensorflow/tfjs'
1+
import * as tf from "@tensorflow/tfjs";
22

33
interface DataPoint extends tf.TensorContainerObject {
4-
xs: tf.Tensor2D,
5-
ys: tf.Tensor3D,
4+
xs: tf.Tensor2D;
5+
ys: tf.Tensor3D;
66
}
77

8-
export default async function evaluate (
8+
export default async function evaluate(
99
model: tf.LayersModel,
1010
dataset: tf.data.Dataset<DataPoint>,
11-
maxEvalBatches: number
12-
): Promise<Record<'val_acc' | 'val_loss' | 'val_perplexity', number>> {
13-
let datasetSize = 0
14-
let totalLoss = 0
15-
const acc: [number, number] = [0, 0]
11+
maxEvalBatches: number,
12+
): Promise<Record<"val_acc" | "val_loss" | "val_perplexity", number>> {
13+
let datasetSize = 0;
14+
let totalLoss = 0;
15+
const acc: [number, number] = [0, 0];
1616

17-
await dataset.take(maxEvalBatches).map(({ xs, ys }) => {
18-
const logits = model.apply(xs)
19-
if (Array.isArray(logits)) {
20-
throw new Error('model output too many tensor')
21-
}
22-
if (logits instanceof tf.SymbolicTensor) {
23-
throw new Error('model output symbolic tensor')
24-
}
25-
xs.dispose()
17+
await dataset
18+
.take(maxEvalBatches)
19+
.map(({ xs, ys }) => {
20+
const logits = model.apply(xs);
21+
if (Array.isArray(logits)) {
22+
throw new Error("model output too many tensor");
23+
}
24+
if (logits instanceof tf.SymbolicTensor) {
25+
throw new Error("model output symbolic tensor");
26+
}
27+
xs.dispose();
2628

27-
return { logits, ys }
28-
}).mapAsync(async ({ logits, ys }) => {
29-
const lossTensor = tf.losses.softmaxCrossEntropy(ys, logits)
30-
const loss = await lossTensor.array()
31-
if (typeof loss !== 'number') {
32-
throw new Error('got multiple loss')
33-
}
29+
return { logits, ys };
30+
})
31+
.mapAsync(async ({ logits, ys }) => {
32+
const lossTensor = tf.losses.softmaxCrossEntropy(ys, logits);
33+
const loss = await lossTensor.array();
34+
if (typeof loss !== "number") {
35+
throw new Error("got multiple loss");
36+
}
3437

35-
const accTensor = tf.metrics.categoricalAccuracy(ys, logits)
36-
const accSize = accTensor.shape.reduce((l, r) => l * r, 1)
37-
const accSum = accTensor.sum()
38-
const accSummed = await accSum.array()
39-
if (typeof accSummed !== 'number') {
40-
throw new Error('got multiple accuracy sum')
41-
}
38+
const accTensor = tf.metrics.categoricalAccuracy(ys, logits);
39+
const accSize = accTensor.shape.reduce((l, r) => l * r, 1);
40+
const accSum = accTensor.sum();
41+
const accSummed = await accSum.array();
42+
if (typeof accSummed !== "number") {
43+
throw new Error("got multiple accuracy sum");
44+
}
4245

43-
tf.dispose([ys, logits, accTensor, accSum, lossTensor])
44-
return { loss, accSummed, accSize }
45-
}).forEachAsync(({ loss, accSummed, accSize }) => {
46-
datasetSize += 1
47-
totalLoss += loss
48-
acc[0] += accSummed
49-
acc[1] += accSize
50-
})
46+
tf.dispose([ys, logits, accTensor, accSum, lossTensor]);
47+
return { loss, accSummed, accSize };
48+
})
49+
.forEachAsync(({ loss, accSummed, accSize }) => {
50+
datasetSize += 1;
51+
totalLoss += loss;
52+
acc[0] += accSummed;
53+
acc[1] += accSize;
54+
});
5155

52-
const loss = totalLoss / datasetSize
56+
const loss = totalLoss / datasetSize;
5357
return {
5458
val_loss: loss,
5559
val_perplexity: Math.exp(loss),
56-
val_acc: acc[0] / acc[1]
57-
}
60+
val_acc: acc[0] / acc[1],
61+
};
5862
}

0 commit comments

Comments
 (0)