Skip to content

Commit e0a096d

Browse files
refactor-botWscats
authored andcommitted
refactor: complete JS to TypeScript migration
- Convert all 5 JS source files to TypeScript - Add generic type parameter to intersect<T>() function - Optimize Set creation: move outside filter callback (was creating new Set per element) - Extract CHUNK_SIZE, TOTAL_LINES, MAX_VALUE constants - Fix inverted error callback logic in writeResult - Fix parseInt(Math.random()) → Math.floor(Math.random()) - Type stream processing with fs.WriteStream - Replace require() with ES module imports - Replace module.exports with export default - Translate Chinese comments to English
1 parent 1672455 commit e0a096d

10 files changed

Lines changed: 143 additions & 125 deletions

File tree

index.js

Lines changed: 0 additions & 9 deletions
This file was deleted.

index.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/**
2+
* Intersect - Main entry point.
3+
* Reads the 3M dataset, then processes the 60M dataset to find intersections.
4+
*/
5+
import readSmallData from './library/data-3M';
6+
import processLargeData from './library/data-60M';
7+
8+
(async (): Promise<void> => {
9+
const smallData = await readSmallData();
10+
const result = await processLargeData(smallData);
11+
console.log(result);
12+
})();

library/create-60M.js

Lines changed: 0 additions & 33 deletions
This file was deleted.

library/create-60M.ts

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/**
2+
* Generate a 60M random number dataset and write to a text file.
3+
* Uses stream backpressure handling for efficient writing.
4+
*/
5+
import * as fs from 'fs';
6+
import * as path from 'path';
7+
8+
const OUTPUT_PATH = path.resolve(__dirname, '../database/data-60M.txt');
9+
const TOTAL_LINES = 600000;
10+
const MAX_VALUE = 60000000;
11+
12+
const writer = fs.createWriteStream(OUTPUT_PATH, { highWaterMark: 1 });
13+
14+
/** Write random numbers to the output file using stream backpressure. */
15+
function writeSixtyMillionTimes(writer: fs.WriteStream): void {
16+
let remaining = TOTAL_LINES;
17+
18+
const write = (): void => {
19+
let ok = true;
20+
do {
21+
remaining--;
22+
const data = Buffer.from(`${Math.floor(Math.random() * MAX_VALUE)}\n`);
23+
if (remaining === 0) {
24+
writer.write(data);
25+
} else {
26+
ok = writer.write(data);
27+
}
28+
} while (remaining > 0 && ok);
29+
30+
if (remaining > 0) {
31+
writer.once('drain', write);
32+
}
33+
};
34+
35+
write();
36+
}
37+
38+
writeSixtyMillionTimes(writer);

library/data-3M.js

Lines changed: 0 additions & 19 deletions
This file was deleted.

library/data-3M.ts

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/**
2+
* Read the 3M dataset line by line from a text file.
3+
* @returns Array of strings, one per line.
4+
*/
5+
import * as fs from 'fs';
6+
import * as readline from 'readline';
7+
8+
export default function readSmallData(): Promise<string[]> {
9+
return new Promise((resolve) => {
10+
const rl = readline.createInterface({
11+
input: fs.createReadStream('./database/data-3M.txt'),
12+
crlfDelay: Infinity,
13+
});
14+
const lines: string[] = [];
15+
rl.on('line', (line: string) => {
16+
lines.push(line);
17+
});
18+
rl.on('close', () => {
19+
resolve(lines);
20+
});
21+
});
22+
}

library/data-60M.js

Lines changed: 0 additions & 58 deletions
This file was deleted.

library/data-60M.ts

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/**
2+
* Process the 60M dataset in chunks, computing intersections with the small dataset.
3+
* Reads the large file as a stream, processing 600 lines at a time.
4+
*/
5+
import { createReadStream, appendFile } from 'fs';
6+
import * as readline from 'readline';
7+
import intersect from './intersect';
8+
9+
/** Append an intersection result to the output file. */
10+
function writeResult(element: string): void {
11+
appendFile('./result.txt', `${element}\n`, (err) => {
12+
if (err) {
13+
console.log('Write failed');
14+
}
15+
});
16+
}
17+
18+
/** Chunk size: number of lines to process per batch. */
19+
const CHUNK_SIZE = 600;
20+
21+
/**
22+
* Stream-process the 60M dataset, finding intersections with smallData.
23+
* @param smallData - The smaller dataset to intersect against.
24+
* @returns Promise that resolves when processing is complete.
25+
*/
26+
export default function processLargeData(smallData: string[]): Promise<string> {
27+
return new Promise((resolve) => {
28+
const rl = readline.createInterface({
29+
input: createReadStream('./database/data-60M.txt', {
30+
highWaterMark: 50,
31+
}),
32+
crlfDelay: Infinity,
33+
});
34+
35+
let lineCount = 0;
36+
let rawData: string[] = [];
37+
38+
rl.on('line', (line: string) => {
39+
rawData.push(line);
40+
console.log(line, rawData.length);
41+
lineCount++;
42+
43+
if (lineCount === CHUNK_SIZE) {
44+
console.log('Read count:', lineCount);
45+
console.log('Extracted data:', rawData);
46+
47+
rl.pause();
48+
49+
const intersectResult = intersect(rawData, smallData);
50+
intersectResult.forEach((element) => writeResult(element));
51+
52+
setTimeout(() => {
53+
rawData = [];
54+
lineCount = 0;
55+
rl.resume();
56+
}, 0);
57+
}
58+
});
59+
60+
rl.on('close', () => {
61+
resolve('Done');
62+
});
63+
});
64+
}

library/intersect.js

Lines changed: 0 additions & 6 deletions
This file was deleted.

library/intersect.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/**
2+
* Compute the intersection of two arrays using a Set for O(n+m) performance.
3+
*/
4+
export default function intersect<T>(a: T[], b: T[]): T[] {
5+
const setB = new Set(b);
6+
return a.filter((x) => setB.has(x));
7+
}

0 commit comments

Comments
 (0)