Add ability to provide initial files to crawl (#121)

* Add ability to provide initial files to crawl

* Fix lint issue

Author: chrischrischris

nx/public/utils/tree.js

@@ -80,22 +80,27 @@ function calculateCrawlTime(startTime) {
 /**
  * Assign the project to an employee.
  * @param {Object} options - The crawl options.
- * @param {string} options.path - The parent path to crawl.
+ * @param {string|string[]} options.path - The parent path(s) to crawl.
+ * @param {Object[]} options.files - Optional array of file objects to include in the crawl.
  * @param {function} options.callback - The callback to run when a file is found.
  * @param {number} options.concurrent - The amount of concurrent requests for the callback queue.
  * @param {number} options.throttle - How much to throttle the crawl.
  */
-export function crawl({ path, callback, concurrent, throttle = 100 }) {
+export function crawl({ path, files: initialFiles = [], callback, concurrent, throttle = 100 }) {
   let time;
   let isCanceled = false;
-  const files = [];
+  const files = [...initialFiles];
   const errors = [];
   const folders = Array.isArray(path) ? [...path] : [path];
   const inProgress = [];
   const startTime = Date.now();
   const queue = new Queue(callback, concurrent, (item, err) => errors.push({ item, err }));
 
   const results = new Promise((resolve) => {
+    if (callback && initialFiles.length > 0) {
+      Promise.allSettled(initialFiles.map((file) => queue.push(file)));
+    }
+
     const interval = setInterval(async () => {
       if (folders.length > 0) {
         inProgress.push(true);

test/utils/tree.test.js

@@ -639,4 +639,108 @@ describe('crawl', () => {
     expect(files.some((f) => f.name === 'file2')).to.equal(true);
     expect(files.some((f) => f.name === 'deep2')).to.equal(true);
   });
+
+  it('Includes initial files in results without crawling', async () => {
+    // eslint-disable-next-line no-unused-vars
+    window.fetch = async (url) => ({
+      ok: true,
+      json: async () => [],
+      headers: { get: () => null },
+    });
+
+    const initialFiles = [
+      { path: '/custom/file1.html', name: 'file1', ext: 'html', lastModified: 123456789 },
+      { path: '/custom/file2.json', name: 'file2', ext: 'json', lastModified: 987654321 },
+    ];
+
+    const { results } = crawl({
+      path: '/empty',
+      files: initialFiles,
+      callback: null,
+      concurrent: 10,
+      throttle: 10,
+    });
+
+    const files = await results;
+    expect(files.length).to.equal(2);
+    expect(files.some((f) => f.name === 'file1' && f.path === '/custom/file1.html')).to.equal(true);
+    expect(files.some((f) => f.name === 'file2' && f.path === '/custom/file2.json')).to.equal(true);
+  });
+
+  it('Includes initial files and crawled files together', async () => {
+    window.fetch = async (url) => ({
+      ok: true,
+      json: async () => (url.includes('/folder') ? mockFilesOnlyResponse : []),
+      headers: { get: () => null },
+    });
+
+    const initialFiles = [
+      { path: '/custom/custom1.html', name: 'custom1', ext: 'html', lastModified: 123456789 },
+    ];
+
+    const { results } = crawl({
+      path: '/folder',
+      files: initialFiles,
+      callback: null,
+      concurrent: 10,
+      throttle: 10,
+    });
+
+    const files = await results;
+    expect(files.length).to.equal(3);
+    expect(files.some((f) => f.name === 'custom1')).to.equal(true);
+    expect(files.some((f) => f.name === 'file1')).to.equal(true);
+    expect(files.some((f) => f.name === 'file2')).to.equal(true);
+  });
+
+  it('Executes callback for initial files', async () => {
+    // eslint-disable-next-line no-unused-vars
+    window.fetch = async (url) => ({
+      ok: true,
+      json: async () => [],
+      headers: { get: () => null },
+    });
+
+    const initialFiles = [
+      { path: '/custom/file1.html', name: 'file1', ext: 'html', lastModified: 123456789 },
+      { path: '/custom/file2.json', name: 'file2', ext: 'json', lastModified: 987654321 },
+    ];
+
+    const callbackResults = [];
+    const callback = async (file) => {
+      callbackResults.push(file.name);
+    };
+
+    const { results } = crawl({
+      path: '/empty',
+      files: initialFiles,
+      callback,
+      concurrent: 10,
+      throttle: 10,
+    });
+
+    await results;
+    expect(callbackResults).to.include('file1');
+    expect(callbackResults).to.include('file2');
+    expect(callbackResults.length).to.equal(2);
+  });
+
+  it('Works without initial files parameter (backward compatibility)', async () => {
+    // eslint-disable-next-line no-unused-vars
+    window.fetch = async (url) => ({
+      ok: true,
+      json: async () => mockFilesOnlyResponse,
+      headers: { get: () => null },
+    });
+
+    const { results } = crawl({
+      path: '/test',
+      callback: null,
+      concurrent: 10,
+      throttle: 10,
+    });
+
+    const files = await results;
+    expect(files.length).to.equal(2);
+  });
 });