Skip to content

Commit 023e0d1

Browse files
FyorlFyorl
committed
Add --omitVolatile flag.
1 parent ab50d1c commit 023e0d1

4 files changed

Lines changed: 119 additions & 23 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
### Breaking Changes
77
- (Jeff Hitchcock) When passing both `--expandAdventures` and `--folders`, the Adventure's contents will now be grouped by Document type in separate folders. The `--recursive` flag must be supplied when packing entries extracted in this way.
88

9+
### Improvements
10+
- Added the `--omitVolatile` command-line flag, and corresponding `omitVolatile` parameter to `extractPact`. When used, changes will only be written if at least one non-volatile field has changed between the entry on disk and the candidate entry being extracted. Currently, `_stats.createdTime`, `_stats.modifiedTime`, and `_stats.lastModifiedBy` are considered volatile.
11+
912
## 1.1.0
1013

1114
### Improvements

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ Extract the contents of a compendium pack into individual source files for each
214214
* **transformName:** *(entry: object): Promise<string|void>* A function that is called on every entry. The value returned from this will be used as the entry's filename and must include the appropriate file extension. If nothing is returned, an auto-generated name will be used instead.
215215
* **transformFolderName:** *(entry: object): Promise<string|void>* A function used to generate a directory name for an extracted Folder document when the `folders` option is used.
216216
* **expandAdventures:** *boolean* Write documents embedded in Adventures to their own files. If the `folders` option is also supplied, the Adventure is treated like a folder, and written to `_Adventure.{yml|json}` instead of `_Folder.{yml|json}`. Additionally, all its entries are grouped into sub-folders by Document type.
217+
* **omitVolatile:** *boolean* When unpacking, diff the candidate entry against an existing one and only write it if non-volatile fields have changed. Currently, `_stats.createdTime`, `_stats.modifiedTime`, and `_stats.lastModifiedBy` are considered volatile.
217218
* **jsonOptions:** *object*
218219
* **replacer:** *(key: string, value: any): any|Array<string|number>* A replacer function or an array of property names in the object to include in the resulting string.
219220
* **space:** *string|number* A number of spaces or a string to use as indentation.

‎commands/package.mjs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ import { compilePack, extractPack, TYPE_COLLECTION_MAP } from "../lib/package.mj
3030
* compendium folders.
3131
* @property {boolean} [expandAdventures] When unpacking, extract adventure documents into a folder with
3232
* each contained document as its own entry in a folder.
33+
* @property {boolean} [omitVolatile] When unpacking, diff the candidate entry against an existing one
34+
* and only write it if non-volatile fields have changed.
35+
* Currently, _stats.createdTime, _stats.modifiedTime, and
36+
* _stats.lastModifiedBy are considered volatile.
3337
*/
3438

3539
/**
@@ -139,6 +143,11 @@ export function getCommand() {
139143
type: "boolean"
140144
});
141145

146+
yargs.option("omitVolatile", {
147+
describe: "When unpacking, diff the candidate entry against an existing one and only write it if non-volatile fields have changed. Currently, _stats.createdTime, _stats.modifiedTime, and _stats.lastModifiedBy are considered volatile.",
148+
type: "boolean"
149+
});
150+
142151
return yargs;
143152
},
144153
handler: async argv => {
@@ -390,7 +399,7 @@ async function handleUnpack(argv) {
390399
}
391400

392401
let documentType;
393-
const { nedb, yaml, clean, folders, expandAdventures } = argv;
402+
const { nedb, yaml, clean, folders, expandAdventures, omitVolatile } = argv;
394403
if ( nedb ) {
395404
documentType = determineDocumentType(pack, argv);
396405
if ( !documentType ) {
@@ -410,7 +419,9 @@ async function handleUnpack(argv) {
410419
console.log(`[${dbMode}] Unpacking "${chalk.blue(pack)}" to "${chalk.blue(source)}"`);
411420

412421
try {
413-
await extractPack(pack, source, { nedb, yaml, documentType, clean, folders, expandAdventures, log: true });
422+
await extractPack(pack, source, {
423+
nedb, yaml, documentType, clean, folders, expandAdventures, omitVolatile, log: true
424+
});
414425
} catch ( err ) {
415426
console.error(err);
416427
process.exitCode = 1;

‎lib/package.mjs‎

Lines changed: 102 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
import fs from "fs";
2-
import path from "path";
1+
import fs from "node:fs";
2+
import os from "node:os";
3+
import path from "node:path";
34
import Datastore from "nedb-promises";
45
import chalk from "chalk";
56
import { default as YAML } from "js-yaml";
@@ -50,6 +51,10 @@ import { ClassicLevel } from "classic-level";
5051
* folders option is also supplied, the Adventure is treated like a
5152
* folder, and all its entries are grouped into sub-folders by
5253
* Document type.
54+
* @property {boolean} [omitVolatile] Do not overwrite an existing entry if the new one has changes to
55+
* non-volatile fields. Currently, _stats.createdTime,
56+
* _stats.modifiedTime, and _stats.lastModifiedBy are considered
57+
* volatile.
5358
* @property {DocumentCollection} [collection] Required only for NeDB packs in order to generate a correct key.
5459
* Can be used instead of documentType if known.
5560
* @property {NameTransformer} [transformName] A function that is used to generate a filename for the extracted
@@ -414,7 +419,7 @@ async function compactClassicLevel(db) {
414419
*/
415420
export async function extractPack(src, dest, {
416421
nedb=false, yaml=false, yamlOptions={}, jsonOptions={}, log=false, documentType, collection, clean, folders,
417-
expandAdventures, transformEntry, transformName, transformFolderName
422+
expandAdventures, omitVolatile, transformEntry, transformName, transformFolderName
418423
}={}) {
419424
if ( nedb && (path.extname(src) !== ".db") ) {
420425
throw new Error("The nedb option was passed to extractPacks, but the target pack does not have a .db extension.");
@@ -423,28 +428,40 @@ export async function extractPack(src, dest, {
423428
if ( nedb && !collection ) {
424429
throw new Error("For NeDB operations, a documentType or collection must be provided.");
425430
}
426-
if ( clean ) fs.rmSync(dest, { force: true, recursive: true, maxRetries: 10 });
431+
const tmp = path.join(os.tmpdir(), "foundryvtt-cli",
432+
`${Date.now()}-${performance.now().toString().replace(".", "-")}`);
427433
// Create the output directory if it doesn't exist already.
428434
fs.mkdirSync(dest, { recursive: true });
429-
if ( nedb ) {
430-
return extractNedb(src, dest, { yaml, yamlOptions, jsonOptions, log, collection, transformEntry, transformName });
435+
fs.mkdirSync(tmp, { recursive: true });
436+
try {
437+
if ( nedb ) {
438+
await extractNedb(src, tmp, {
439+
yaml, yamlOptions, jsonOptions, omitVolatile, log, collection, transformEntry, transformName, existing: dest
440+
});
441+
}
442+
await extractClassicLevel(src, tmp, {
443+
yaml, log, yamlOptions, jsonOptions, folders, expandAdventures, omitVolatile, transformEntry, transformName,
444+
transformFolderName, existing: dest
445+
});
446+
if ( clean ) fs.rmSync(dest, { force: true, recursive: true, maxRetries: 10 });
447+
fs.cpSync(tmp, dest, { force: true, recursive: true });
448+
} finally {
449+
fs.rmSync(tmp, { force: true, recursive: true, maxRetries: 10 });
431450
}
432-
return extractClassicLevel(src, dest, {
433-
yaml, log, yamlOptions, jsonOptions, folders, expandAdventures, transformEntry, transformName, transformFolderName
434-
});
435451
}
436452

437453
/* -------------------------------------------- */
438454

439455
/**
440456
* Extract a NeDB compendium pack into individual source files for each primary Document.
441-
* @param {string} pack The source compendium pack.
442-
* @param {string} dest The root output directory.
457+
* @param {string} pack The source compendium pack.
458+
* @param {string} dest The root output directory.
443459
* @param {Partial<ExtractOptions>} [options]
460+
* @param {string} [options.existing] The location of existing serialized Documents.
444461
* @returns {Promise<void>}
445462
*/
446463
async function extractNedb(pack, dest, {
447-
yaml, yamlOptions, jsonOptions, log, collection, transformEntry, transformName
464+
yaml, yamlOptions, jsonOptions, omitVolatile, log, collection, transformEntry, transformName, existing
448465
}={}) {
449466
// Load the NeDB file.
450467
const db = new Datastore({ filename: pack, autoload: true });
@@ -466,7 +483,9 @@ async function extractNedb(pack, dest, {
466483
name = `${doc.name ? `${getSafeFilename(doc.name)}_${doc._id}` : doc._id}.${yaml ? "yml" : "json"}`;
467484
}
468485
const filename = path.join(dest, name);
469-
serializeDocument(doc, filename, { yaml, yamlOptions, jsonOptions });
486+
serializeDocument(checkVolatile(doc, name, { omitVolatile, existing, yaml }), filename, {
487+
yaml, yamlOptions, jsonOptions
488+
});
470489
if ( log ) console.log(`Wrote ${chalk.blue(name)}`);
471490
}
472491
}
@@ -475,13 +494,15 @@ async function extractNedb(pack, dest, {
475494

476495
/**
477496
* Extract a LevelDB pack into individual source files for each primary Document.
478-
* @param {string} pack The source compendium pack.
479-
* @param {string} dest The root output directory.
497+
* @param {string} pack The source compendium pack.
498+
* @param {string} dest The root output directory.
480499
* @param {Partial<ExtractOptions>} [options]
500+
* @param {string} [options.existing] The location of existing serialized Documents.
481501
* @returns {Promise<void>}
482502
*/
483503
async function extractClassicLevel(pack, dest, {
484-
yaml, yamlOptions, jsonOptions, log, folders, expandAdventures, transformEntry, transformName, transformFolderName
504+
yaml, yamlOptions, jsonOptions, log, folders, expandAdventures, omitVolatile, transformEntry, transformName,
505+
transformFolderName, existing
485506
}={}) {
486507
// Load the directory as a ClassicLevel DB.
487508
const db = new ClassicLevel(pack, { keyEncoding: "utf8", valueEncoding: "json", createIfMissing: false });
@@ -511,7 +532,7 @@ async function extractClassicLevel(pack, dest, {
511532
if ( await transformEntry?.(doc) === false ) continue;
512533
if ( key.startsWith("!adventures") && expandAdventures ) {
513534
await extractAdventure(doc, dest, { folderMap }, {
514-
yaml, yamlOptions, jsonOptions, log, folders, transformEntry, transformName
535+
yaml, yamlOptions, jsonOptions, log, folders, omitVolatile, transformEntry, transformName, existing
515536
});
516537
continue;
517538
}
@@ -527,7 +548,9 @@ async function extractClassicLevel(pack, dest, {
527548
if ( folder ) name = path.join(folder, name);
528549
}
529550
const filename = path.join(dest, name);
530-
serializeDocument(doc, filename, { yaml, yamlOptions, jsonOptions });
551+
serializeDocument(checkVolatile(doc, name, { omitVolatile, existing, yaml }), filename, {
552+
yaml, yamlOptions, jsonOptions
553+
});
531554
if ( log ) console.log(`Wrote ${chalk.blue(name)}`);
532555
}
533556

@@ -543,9 +566,10 @@ async function extractClassicLevel(pack, dest, {
543566
* @param {object} [adventureOptions] Options to configure adventure extraction behavior.
544567
* @param {Map<string, FolderDescriptor>} [adventureOptions.folderMap] Folder hierarchy.
545568
* @param {Partial<ExtractOptions>} [extractOptions] Options to configure serialization behavior.
569+
* @param {string} [extractOptions.existing] The location of existing serialized Documents.
546570
*/
547571
async function extractAdventure(doc, dest, { folderMap }={}, {
548-
yaml, yamlOptions, jsonOptions, log, folders, transformEntry, transformName, transformFolderName
572+
yaml, yamlOptions, jsonOptions, log, folders, omitVolatile, transformEntry, transformName, transformFolderName, existing
549573
}={}) {
550574
let adventureFolder;
551575

@@ -587,15 +611,19 @@ async function extractAdventure(doc, dest, { folderMap }={}, {
587611
}
588612
const filename = path.join(dest, embeddedName);
589613
paths.push(adventureFolder ? path.relative(adventureFolder, embeddedName) : path.basename(embeddedName));
590-
serializeDocument(embeddedDoc, filename, { yaml, yamlOptions, jsonOptions });
614+
serializeDocument(checkVolatile(embeddedDoc, embeddedName, { omitVolatile, existing, yaml }), filename, {
615+
yaml, yamlOptions, jsonOptions
616+
});
591617
if ( log ) console.log(`Wrote ${chalk.blue(embeddedName)}`);
592618
}
593619
doc[embeddedCollectionName] = paths;
594620
}
595621

596622
// Write the adventure itself
597623
const filename = path.join(dest, name);
598-
serializeDocument(doc, filename, { yaml, yamlOptions, jsonOptions });
624+
serializeDocument(checkVolatile(doc, name, { omitVolatile, existing, yaml }), filename, {
625+
yaml, yamlOptions, jsonOptions
626+
});
599627
if ( log ) console.log(`Wrote ${chalk.blue(name)}`);
600628
}
601629

@@ -722,6 +750,59 @@ function serializeDocument(doc, filename, { yaml, yamlOptions, jsonOptions }={})
722750

723751
/* -------------------------------------------- */
724752

753+
/**
754+
* Check if only a Document's volatile fields have changed, and if so, do not write those changes.
755+
* @param {object} doc The candidate Document data.
756+
* @param {string} name The destination file name.
757+
* @param {object} [options]
758+
* @param {boolean} [options.omitVolatile] Only write changes if some non-volatile fields have changed.
759+
* @param {string} [options.existing] The location of the existing file.
760+
* @param {boolean} [options.yaml] Whether the existing files are formatted with YAML.
761+
* @returns {object} The data to write. Either the candidate data, or the existing data.
762+
*/
763+
function checkVolatile(doc, name, { omitVolatile, existing, yaml }={}) {
764+
if ( !omitVolatile || !existing || !("_stats" in doc) ) return doc;
765+
const parse = yaml ? YAML.load : JSON.parse;
766+
try {
767+
const base = parse(fs.readFileSync(path.join(existing, name), { encoding: "utf8" }));
768+
if ( !base || !("_stats" in base) ) return doc;
769+
const copy = structuredClone(doc);
770+
if ( "createdTime" in base._stats ) copy._stats.createdTime = base._stats.createdTime;
771+
if ( "modifiedTime" in base._stats ) copy._stats.modifiedTime = base._stats.modifiedTime;
772+
if ( "lastModifiedBy" in base._stats ) copy._stats.lastModifiedBy = base._stats.lastModifiedBy;
773+
return testEquality(base, copy) ? base : doc;
774+
} catch {
775+
return doc;
776+
}
777+
}
778+
779+
/* -------------------------------------------- */
780+
781+
/**
782+
* Determine if two pieces of data are the same.
783+
* @param {string|number|boolean|object} a
784+
* @param {string|number|boolean|object} b
785+
* @param {number} [depth]
786+
* @returns {boolean}
787+
*/
788+
function testEquality(a, b, depth=0) {
789+
if ( depth > 100 ) throw new Error("cyclic structure detected");
790+
if ( typeof a !== typeof b ) return false;
791+
if ( (a === null) || (typeof a !== "object") ) return a === b;
792+
if ( Array.isArray(a) ) {
793+
if ( a.length !== b.length ) return false;
794+
return a.every((v, i) => testEquality(v, b[i], depth + 1));
795+
}
796+
const keysA = Object.keys(a);
797+
if ( keysA.length !== Object.keys(b).length ) return false;
798+
for ( const p of keysA ) {
799+
if ( !testEquality(a[p], b[p], depth + 1) ) return false;
800+
}
801+
return true;
802+
}
803+
804+
/* -------------------------------------------- */
805+
725806
/**
726807
* Join non-blank key parts.
727808
* @param {...string} args Key parts.

0 commit comments

Comments
 (0)