Skip to content

Commit 15fbcff

Browse files
mukulboromukulboro
committed
feat: better logging
1 parent b3c2475 commit 15fbcff

2 files changed

Lines changed: 117 additions & 239 deletions

File tree

‎src/core/framework-writer.js‎

Lines changed: 115 additions & 238 deletions
Original file line numberDiff line numberDiff line change
@@ -682,281 +682,158 @@ export class FrameworkWriter {
682682

683683
async downloadAssetsWithExactNames() {
684684
const axios = (await import('axios')).default;
685+
const CONCURRENCY = 8;
685686

686-
// Images
687-
if (this.cloner.assets.images.length) {
687+
const downloadTask = async (task, current, total, type) => {
688+
const { url, dest, buffer, headers } = task;
689+
const pct = Math.round((current / total) * 100);
690+
const label = path.basename(dest);
691+
688692
if (!this.cloner.options.quiet) {
689-
console.log(
690-
chalk.gray(
691-
` Downloading ${this.cloner.assets.images.length} images...`,
692-
),
693-
);
693+
process.stdout.write(chalk.gray(` [${current}/${total}] ${pct}% Downloading ${type}: ${label}...\r`));
694694
}
695-
for (const img of this.cloner.assets.images) {
696-
if (img.url && img.url.includes('/_next/image')) continue; // skip optimizer endpoints
697-
698-
const dest = path.join(
699-
this.cloner.options.outputDir,
700-
'assets',
701-
'images',
702-
img.filename,
703-
);
704-
await fs.ensureDir(path.dirname(dest));
705695

706-
if (img.buffer) {
707-
await fs.writeFile(dest, img.buffer);
708-
continue;
696+
try {
697+
await fs.ensureDir(path.dirname(dest));
698+
if (buffer) {
699+
await fs.writeFile(dest, buffer);
700+
return true;
709701
}
710702

711-
const tryUrls = [];
712-
if (img.url) tryUrls.push(img.url);
713-
if (img.nextJsUrl) tryUrls.push(this.cloner.resolveUrl(img.nextJsUrl));
703+
const res = await axios.get(url, {
704+
responseType: 'arraybuffer',
705+
timeout: 60000,
706+
headers: headers || { 'User-Agent': 'Mozilla/5.0' },
707+
validateStatus: () => true,
708+
});
714709

715-
let saved = false;
716-
for (const u of tryUrls) {
717-
try {
718-
const res = await axios.get(u, {
719-
responseType: 'arraybuffer',
720-
timeout: 60000,
721-
headers: {
722-
'User-Agent': 'Mozilla/5.0',
723-
Accept: 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
724-
Referer: this.cloner.url,
725-
},
726-
validateStatus: () => true,
727-
});
710+
if (res.status >= 200 && res.status < 300 && res.data?.byteLength > 0) {
711+
await fs.writeFile(dest, res.data);
712+
return true;
713+
}
714+
return false;
715+
} catch (e) {
716+
this.cloner.logger.warnNonCritical(type, url, e);
717+
return false;
718+
}
719+
};
728720

729-
const status = res.status || 0;
730-
const ctype = String(
731-
res.headers?.['content-type'] || '',
732-
).toLowerCase();
733-
734-
// Follow Microlink JSON if necessary
735-
if (
736-
/microlink\.io/i.test(u) &&
737-
ctype.includes('application/json')
738-
) {
739-
try {
740-
const j = await axios.get(u, {
741-
responseType: 'json',
742-
timeout: 60000,
743-
headers: { 'User-Agent': 'Mozilla/5.0' },
744-
});
745-
const target =
746-
j.data?.data?.image?.url ||
747-
j.data?.data?.screenshot?.url ||
748-
j.data?.data?.thumbnail?.url ||
749-
j.data?.image?.url ||
750-
j.data?.screenshot?.url;
751-
if (target) {
752-
const res2 = await axios.get(target, {
753-
responseType: 'arraybuffer',
754-
timeout: 60000,
755-
headers: {
756-
'User-Agent': 'Mozilla/5.0',
757-
Referer: this.cloner.url,
758-
Accept:
759-
'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
760-
},
761-
validateStatus: () => true,
762-
});
763-
if ((res2.status || 0) >= 200 && (res2.status || 0) < 300) {
764-
await fs.writeFile(dest, res2.data);
765-
saved = true;
766-
break;
767-
}
768-
}
769-
} catch (e) {
770-
this.cloner.logger.warnNonCritical('image', u, e);
771-
}
772-
}
721+
const runTasks = async (tasks, type) => {
722+
if (!tasks.length) return;
723+
if (!this.cloner.options.quiet) {
724+
console.log(chalk.blue(` 📥 Downloading ${tasks.length} ${type}...`));
725+
}
773726

774-
if (status >= 200 && status < 300 && res.data?.byteLength > 0) {
775-
await fs.writeFile(dest, res.data);
776-
saved = true;
777-
break;
778-
}
779-
} catch (e) {
780-
this.cloner.logger.warnNonCritical('image', u, e);
781-
}
782-
}
783-
if (!saved) {
784-
this.cloner.logger.warnNonCritical(
785-
'image',
786-
img.url || img.nextJsUrl,
787-
new Error('exhausted sources'),
788-
);
727+
let current = 0;
728+
const results = [];
729+
const executing = new Set();
730+
731+
for (const task of tasks) {
732+
const p = Promise.resolve().then(() => downloadTask(task, ++current, tasks.length, type));
733+
results.push(p);
734+
executing.add(p);
735+
p.finally(() => executing.delete(p));
736+
if (executing.size >= CONCURRENCY) {
737+
await Promise.race(executing);
789738
}
790739
}
791-
}
740+
await Promise.all(results);
741+
if (!this.cloner.options.quiet) process.stdout.write('\n');
742+
};
792743

793-
// CSS externals
744+
// 1. Images
745+
const imageTasks = this.cloner.assets.images
746+
.filter(img => img.url && !img.url.includes('/_next/image'))
747+
.map(img => ({
748+
url: img.url,
749+
dest: path.join(this.cloner.options.outputDir, 'assets', 'images', img.filename),
750+
buffer: img.buffer,
751+
headers: {
752+
'User-Agent': 'Mozilla/5.0',
753+
Accept: 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
754+
Referer: this.cloner.url,
755+
}
756+
}));
757+
await runTasks(imageTasks, 'images');
758+
759+
// 2. CSS
794760
const cssExternals = this.cloner.assets.styles.filter(
795761
(s) => s.url && s.type === 'external',
796762
);
797763
if (cssExternals.length) {
798764
if (!this.cloner.options.quiet) {
799-
console.log(
800-
chalk.gray(` Downloading ${cssExternals.length} CSS files...`),
801-
);
765+
console.log(chalk.blue(` 📥 Downloading ${cssExternals.length} CSS files...`));
802766
}
767+
let current = 0;
803768
for (const css of cssExternals) {
804-
const dest = path.join(
805-
this.cloner.options.outputDir,
806-
'assets',
807-
'css',
808-
css.filename,
809-
);
769+
current++;
770+
const dest = path.join(this.cloner.options.outputDir, 'assets', 'css', css.filename);
810771
try {
811772
const res = await axios.get(css.url, {
812773
responseType: 'text',
813774
timeout: 30000,
814775
headers: { 'User-Agent': 'Mozilla/5.0' },
815776
});
816777
let text = res.data || '';
817-
text = await this.rewriteCssUrlsAndDownload(text, css.url, axios, {
818-
fromInline: false,
819-
});
778+
text = await this.rewriteCssUrlsAndDownload(text, css.url, axios, { fromInline: false });
820779
await fs.ensureDir(path.dirname(dest));
821780
await fs.writeFile(dest, text, 'utf8');
781+
if (!this.cloner.options.quiet) {
782+
const pct = Math.round((current / cssExternals.length) * 100);
783+
process.stdout.write(chalk.gray(` [${current}/${cssExternals.length}] ${pct}% Processed CSS: ${css.filename}...\r`));
784+
}
822785
} catch (e) {
823786
this.cloner.logger.warnNonCritical('styles', css.url, e);
824787
}
825788
}
789+
if (!this.cloner.options.quiet) process.stdout.write('\n');
826790
}
827791

828-
// JS externals: download when JS is enabled (based on decision)
829-
const jsExternals = this.cloner.options.disableJs
830-
? []
831-
: this.cloner.assets.scripts.filter((s) => s.url);
832-
833-
if (jsExternals.length) {
834-
if (!this.cloner.options.quiet) {
835-
console.log(
836-
chalk.gray(` Downloading ${jsExternals.length} JS files...`),
837-
);
838-
}
839-
for (const s of jsExternals) {
840-
const dest = path.join(
841-
this.cloner.options.outputDir,
842-
'assets',
843-
'js',
844-
s.filename,
845-
);
846-
try {
847-
const res = await axios.get(s.url, {
848-
responseType: 'text',
849-
timeout: 30000,
850-
headers: { 'User-Agent': 'Mozilla/5.0' },
851-
});
852-
await fs.ensureDir(path.dirname(dest));
853-
await fs.writeFile(dest, res.data || '', 'utf8');
854-
} catch (e) {
855-
this.cloner.logger.warnNonCritical('scripts', s.url, e);
856-
}
857-
}
858-
}
859-
860-
// Fonts
861-
if (this.cloner.assets.fonts.length) {
862-
if (!this.cloner.options.quiet) {
863-
console.log(
864-
chalk.gray(
865-
` Downloading ${this.cloner.assets.fonts.length} fonts...`,
866-
),
867-
);
868-
}
869-
for (const f of this.cloner.assets.fonts) {
870-
const dest = path.join(
871-
this.cloner.options.outputDir,
872-
'assets',
873-
'fonts',
874-
f.filename,
875-
);
876-
if (!f.url) continue;
877-
try {
878-
const res = await axios.get(f.url, {
879-
responseType: 'arraybuffer',
880-
timeout: 30000,
881-
headers: { 'User-Agent': 'Mozilla/5.0' },
882-
});
883-
await fs.ensureDir(path.dirname(dest));
884-
await fs.writeFile(dest, res.data);
885-
} catch (e) {
886-
this.cloner.logger.warnNonCritical('fonts', f.url, e);
887-
}
888-
}
889-
}
890-
891-
// Icons
892-
if (this.cloner.assets.icons.length) {
893-
if (!this.cloner.options.quiet) {
894-
console.log(
895-
chalk.gray(
896-
` Downloading ${this.cloner.assets.icons.length} icons...`,
897-
),
898-
);
899-
}
900-
for (const i of this.cloner.assets.icons) {
901-
const dest = path.join(
902-
this.cloner.options.outputDir,
903-
'assets',
904-
'icons',
905-
i.filename,
906-
);
907-
if (!i.url) continue;
908-
try {
909-
const res = await axios.get(i.url, {
910-
responseType: 'arraybuffer',
911-
timeout: 30000,
912-
headers: { 'User-Agent': 'Mozilla/5.0' },
913-
});
914-
await fs.ensureDir(path.dirname(dest));
915-
await fs.writeFile(dest, res.data);
916-
} catch (e) {
917-
this.cloner.logger.warnNonCritical('icon', i.url, e);
918-
}
919-
}
792+
// 3. JS
793+
if (!this.cloner.options.disableJs) {
794+
const jsTasks = this.cloner.assets.scripts
795+
.filter(s => s.url)
796+
.map(s => ({
797+
url: s.url,
798+
dest: path.join(this.cloner.options.outputDir, 'assets', 'js', s.filename),
799+
headers: { 'User-Agent': 'Mozilla/5.0' }
800+
}));
801+
await runTasks(jsTasks, 'scripts');
920802
}
921803

922-
// Media
923-
if (this.cloner.assets.media.length) {
924-
if (!this.cloner.options.quiet) {
925-
console.log(
926-
chalk.gray(
927-
` Downloading ${this.cloner.assets.media.length} media files...`,
928-
),
929-
);
930-
}
931-
for (const media of this.cloner.assets.media) {
932-
const dest = path.join(
933-
this.cloner.options.outputDir,
934-
'assets',
935-
'media',
936-
media.filename,
937-
);
938-
if (!media.url) continue;
939-
try {
940-
const res = await axios.get(media.url, {
941-
responseType: 'arraybuffer',
942-
timeout: 120000,
943-
headers: {
944-
'User-Agent': 'Mozilla/5.0',
945-
Accept: 'video/*;q=0.9,audio/*;q=0.9,*/*;q=0.5',
946-
Referer: this.cloner.url,
947-
},
948-
});
949-
await fs.ensureDir(path.dirname(dest));
950-
await fs.writeFile(dest, res.data);
951-
} catch (e) {
952-
this.cloner.logger.warnNonCritical(
953-
media.type || 'media',
954-
media.url,
955-
e,
956-
);
804+
// 4. Fonts
805+
const fontTasks = this.cloner.assets.fonts
806+
.filter(f => f.url)
807+
.map(f => ({
808+
url: f.url,
809+
dest: path.join(this.cloner.options.outputDir, 'assets', 'fonts', f.filename),
810+
headers: { 'User-Agent': 'Mozilla/5.0' }
811+
}));
812+
await runTasks(fontTasks, 'fonts');
813+
814+
// 5. Icons
815+
const iconTasks = this.cloner.assets.icons
816+
.filter(i => i.url)
817+
.map(i => ({
818+
url: i.url,
819+
dest: path.join(this.cloner.options.outputDir, 'assets', 'icons', i.filename),
820+
headers: { 'User-Agent': 'Mozilla/5.0' }
821+
}));
822+
await runTasks(iconTasks, 'icons');
823+
824+
// 6. Media
825+
const mediaTasks = this.cloner.assets.media
826+
.filter(m => m.url)
827+
.map(m => ({
828+
url: m.url,
829+
dest: path.join(this.cloner.options.outputDir, 'assets', 'media', m.filename),
830+
headers: {
831+
'User-Agent': 'Mozilla/5.0',
832+
Accept: 'video/*;q=0.9,audio/*;q=0.9,*/*;q=0.5',
833+
Referer: this.cloner.url,
957834
}
958-
}
959-
}
835+
}));
836+
await runTasks(mediaTasks, 'media');
960837
}
961838

962839
async rewriteCssUrlsAndDownload(

0 commit comments

Comments
 (0)