feat: add support for files and folders beginning with a dot (#206)

Author: sethvargo

src/upload-helper.ts

@@ -123,7 +123,7 @@ export class UploadHelper {
     metadata?: Metadata,
   ): Promise<UploadResponse[]> {
     // by default we just use directoryPath with empty glob '', which globby evaluates to directory/**/*
-    const filesList = await globby([path.posix.join(directoryPath, glob)]);
+    const filesList = await expandGlob(directoryPath, glob);
     const uploader = async (filePath: string): Promise<UploadResponse> => {
       const destination = await getDestinationFromPath(
         filePath,
@@ -145,3 +145,34 @@ export class UploadHelper {
     return await pMap(filesList, uploader, { concurrency });
   }
 }
+
+/**
+ * expandGlob compiles the list of all files in the given directory for
+ * the provided glob.
+ *
+ * @param directoryPath The path to the directory.
+ * @param glob Glob pattern to use for searching. If the empty string, a
+ * match-all pattern is used instead.
+ * @return Sorted list of files in posix form.
+ */
+export async function expandGlob(
+  directoryPath: string,
+  glob: string,
+): Promise<string[]> {
+  const pth = toPosixPath(path.posix.join(directoryPath, glob));
+  const filesList = await globby([pth], {
+    dot: true,
+  });
+  return filesList.sort();
+}
+
+/**
+ * toPosixPath converts the given path to the posix form. On Windows, \\ will be
+ * replaced with /.
+ *
+ * @param pth. Path to transform.
+ * @return Posix-path.
+ */
+export function toPosixPath(pth: string): string {
+  return pth.split(path.sep).join(path.posix.sep);
+}

tests/upload-helper.test.ts

@@ -17,8 +17,11 @@
 import 'mocha';
 import { expect } from 'chai';
 
-import * as sinon from 'sinon';
+import { promises as fs } from 'fs';
+import * as os from 'os';
 import * as path from 'path';
+import * as sinon from 'sinon';
+
 import { Storage, Bucket } from '@google-cloud/storage';
 
 import {
@@ -33,7 +36,7 @@ import {
   TXT_FILES_IN_DIR,
   TXT_FILES_IN_TOP_DIR,
 } from './constants.test';
-import { UploadHelper } from '../src/upload-helper';
+import { UploadHelper, expandGlob, toPosixPath } from '../src/upload-helper';
 /**
  * Unit Test uploadFile method in uploadHelper.
  */
@@ -216,6 +219,7 @@ describe('Unit Test uploadDir', function () {
       expect(destination.split('/')[0]).eq(EXAMPLE_PREFIX);
     });
   });
+
   it('uploads a dir at bucket root with globstar txt', async function () {
     const uploader = new UploadHelper(new Storage());
     await uploader.uploadDirectory(
@@ -272,3 +276,88 @@ describe('Unit Test uploadDir', function () {
     expect(destinations).to.have.members(TXT_FILES_IN_TOP_DIR);
   });
 });
+
+describe('#expandGlob', () => {
+  beforeEach(async function () {
+    // Make a temporary directory and make the path relative to cwd, then make
+    // it posix.
+    const tmp = os.tmpdir();
+    this.tmpdir = await fs.mkdtemp(path.join(tmp, 'gha-'));
+    this.tmpdir = path.relative(process.cwd(), this.tmpdir);
+  });
+
+  afterEach(async function () {
+    if (this.tmpdir) {
+      try {
+        // TODO(sethvargo): switch to just fs.rm once we upgrade to Node 16
+        // only. This function is deprecated in 16, but the replacement doesn't
+        // exist in 12.
+        await fs.rmdir(this.tmpdir, { recursive: true });
+      } catch (err) {
+        console.error(`failed to remove directory: ${err}`);
+      }
+    }
+  });
+
+  it('returns the empty list when the directory is empty', async function () {
+    const list = await expandGlob(this.tmpdir, '');
+    expect(list).to.eql([]);
+  });
+
+  it('returns one file', async function () {
+    const a = path.join(this.tmpdir, 'a');
+    await fs.writeFile(a, 'test');
+    const list = await expandGlob(this.tmpdir, '');
+    expect(list).to.eql([toPosixPath(a)]);
+  });
+
+  it('returns multiple files', async function () {
+    const a = path.join(this.tmpdir, 'a');
+    await fs.writeFile(a, 'test');
+
+    const b = path.join(this.tmpdir, 'b');
+    await fs.writeFile(b, 'test');
+
+    const list = await expandGlob(this.tmpdir, '');
+    expect(list).to.eql([toPosixPath(a), toPosixPath(b)]);
+  });
+
+  it('returns files in subdirectories', async function () {
+    const a = path.join(this.tmpdir, 'a');
+    await fs.writeFile(a, 'test');
+
+    const pth = path.join(this.tmpdir, 'sub', 'directory');
+    await fs.mkdir(pth, { recursive: true });
+    const b = path.join(pth, 'b');
+    await fs.writeFile(b, 'test');
+
+    const list = await expandGlob(this.tmpdir, '');
+    expect(list).to.eql([toPosixPath(a), toPosixPath(b)]);
+  });
+
+  it('returns files beginning with a dot', async function () {
+    const a = path.join(this.tmpdir, '.a');
+    await fs.writeFile(a, 'test');
+
+    const pth = path.join(this.tmpdir, 'sub', 'directory');
+    await fs.mkdir(pth, { recursive: true });
+    const b = path.join(pth, '.b');
+    await fs.writeFile(b, 'test');
+
+    const list = await expandGlob(this.tmpdir, '');
+    expect(list).to.eql([toPosixPath(a), toPosixPath(b)]);
+  });
+
+  it('returns files with non-ascii characters', async function () {
+    const a = path.join(this.tmpdir, '🚀');
+    await fs.writeFile(a, 'test');
+
+    const pth = path.join(this.tmpdir, 'sub', 'directory');
+    await fs.mkdir(pth, { recursive: true });
+    const b = path.join(pth, '.🚀');
+    await fs.writeFile(b, 'test');
+
+    const list = await expandGlob(this.tmpdir, '');
+    expect(list).to.eql([toPosixPath(b), toPosixPath(a)]);
+  });
+});