Skip to content

Commit 550a47d

Browse files
jwunderljwunderl
authored
Fixes for local github cache expiration, projects loading while offline (#11342)
* fix caching regression / clearing of all caches * glue in extensions to hex as well in extra space * cache hex info as well, so ideally we can warm cache even when offline * backup -> fallback, clean up unused test marker, shallow clones instead of jsonifying
1 parent 99c5833 commit 550a47d

8 files changed

Lines changed: 296 additions & 77 deletions

File tree

‎cli/cli.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -276,8 +276,8 @@ class FileGithubDb implements pxt.github.IGithubDb {
276276
return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t));
277277
}
278278

279-
loadPackageAsync(repopath: string, tag: string): Promise<pxt.github.CachedPackage> {
280-
return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t));
279+
loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<pxt.github.CachedPackage> {
280+
return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, fallbackPackageFiles));
281281
}
282282

283283
loadTutorialMarkdown(repopath: string, tag?: string): Promise<pxt.github.CachedPackage> {

‎pxtcompiler/simpledriver.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ namespace pxt {
5252
return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t));
5353
}
5454

55-
loadPackageAsync(repopath: string, tag: string): Promise<pxt.github.CachedPackage> {
56-
return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t));
55+
loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<pxt.github.CachedPackage> {
56+
return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, fallbackPackageFiles));
5757
}
5858

5959
loadTutorialMarkdown(repopath: string, tag?: string): Promise<pxt.github.CachedPackage> {

‎pxtlib/cpp.ts‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1516,6 +1516,21 @@ namespace pxt.hexloader {
15161516
})
15171517
}
15181518

1519+
export function stringifyHexInfoForCache(hexInfo: pxtc.HexInfo) {
1520+
if (!hexInfo?.hex) return undefined;
1521+
1522+
const cachedMeta = {
1523+
...hexInfo,
1524+
hex: compressHex(hexInfo.hex)
1525+
};
1526+
return JSON.stringify(cachedMeta);
1527+
}
1528+
1529+
export function storeHexInfoCacheEntryAsync(host: Host, sha: string, cachedHexInfo: string) {
1530+
if (!sha || !cachedHexInfo) return Promise.resolve();
1531+
return storeWithLimitAsync(host, "hex-keys", "hex-" + sha, cachedHexInfo);
1532+
}
1533+
15191534
export function getHexInfoAsync(host: Host, extInfo: pxtc.ExtensionInfo, cloudModule?: any): Promise<pxtc.HexInfo> {
15201535
if (!extInfo.sha)
15211536
return Promise.resolve<any>(null)
@@ -1545,10 +1560,7 @@ namespace pxt.hexloader {
15451560
else {
15461561
return downloadHexInfoAsync(extInfo)
15471562
.then(meta => {
1548-
let origHex = meta.hex
1549-
meta.hex = compressHex(meta.hex)
1550-
let store = JSON.stringify(meta)
1551-
meta.hex = origHex
1563+
let store = stringifyHexInfoForCache(meta)
15521564
return storeWithLimitAsync(host, "hex-keys", key, store)
15531565
.then(() => meta)
15541566
}).catch(e => {

‎pxtlib/github.ts‎

Lines changed: 70 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -147,11 +147,24 @@ namespace pxt.github {
147147
files: Map<string>;
148148
}
149149

150+
function copyCachedPackage(cachedPackage: CachedPackage): CachedPackage {
151+
return {
152+
files: { ...cachedPackage.files }
153+
};
154+
}
155+
156+
function cachedPackageFromFallback(fallbackPackageFiles?: pxt.Map<string>): CachedPackage {
157+
if (!fallbackPackageFiles) return undefined;
158+
return {
159+
files: { ...fallbackPackageFiles }
160+
};
161+
}
162+
150163
// caching
151164
export interface IGithubDb {
152165
latestVersionAsync(repopath: string, config: PackagesConfig): Promise<string>;
153166
loadConfigAsync(repopath: string, tag: string): Promise<pxt.PackageConfig>;
154-
loadPackageAsync(repopath: string, tag: string): Promise<CachedPackage>;
167+
loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<CachedPackage>;
155168
loadTutorialMarkdown(repopath: string, tag?: string): Promise<CachedPackage>;
156169
cacheReposAsync(response: GHTutorialResponse): Promise<void>;
157170
}
@@ -273,7 +286,7 @@ namespace pxt.github {
273286
return resolved
274287
}
275288

276-
async loadPackageAsync(repopath: string, tag: string): Promise<CachedPackage> {
289+
async loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<CachedPackage> {
277290
if (!tag) {
278291
pxt.debug(`load pkg: default to master branch`)
279292
tag = "master";
@@ -282,48 +295,49 @@ namespace pxt.github {
282295
// try using github proxy first
283296
if (hasProxy()) {
284297
try {
285-
return await this.proxyWithCdnLoadPackageAsync(repopath, tag).then(v => U.clone(v));
298+
return await this.proxyWithCdnLoadPackageAsync(repopath, tag).then(copyCachedPackage);
286299
} catch (e) {
287300
ghProxyHandleException(e);
288301
}
289302
}
290303

291304
// try using github apis
292-
return await this.githubLoadPackageAsync(repopath, tag);
305+
return await this.githubLoadPackageAsync(repopath, tag, fallbackPackageFiles);
293306
}
294307

295-
private githubLoadPackageAsync(repopath: string, tag: string): Promise<CachedPackage> {
296-
return tagToShaAsync(repopath, tag)
297-
.then(sha => {
298-
// cache lookup
299-
const key = `${repopath}/${sha}`;
300-
let res = this.packages[key];
301-
if (res) {
302-
pxt.debug(`github cache ${repopath}/${tag}/text`);
303-
return Promise.resolve(U.clone(res));
304-
}
308+
private async githubLoadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<CachedPackage> {
309+
try {
310+
const sha = await tagToShaAsync(repopath, tag);
311+
312+
// cache lookup
313+
const key = `${repopath}/${sha}`;
314+
let res = this.packages[key];
315+
if (res) {
316+
pxt.debug(`github cache ${repopath}/${tag}/text`);
317+
return copyCachedPackage(res);
318+
}
305319

306-
// load and cache
307-
pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`)
308-
return downloadTextAsync(repopath, sha, pxt.CONFIG_NAME)
309-
.then(pkg => {
310-
const current: CachedPackage = {
311-
files: {}
312-
}
313-
current.files[pxt.CONFIG_NAME] = pkg
314-
const cfg: pxt.PackageConfig = JSON.parse(pkg)
315-
return U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1),
316-
fn => downloadTextAsync(repopath, sha, fn)
317-
.then(text => {
318-
current.files[fn] = text
319-
}))
320-
.then(() => {
321-
// cache!
322-
this.packages[key] = current;
323-
return U.clone(current);
324-
})
325-
})
326-
})
320+
// load and cache
321+
pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`)
322+
const pkg = await downloadTextAsync(repopath, sha, pxt.CONFIG_NAME);
323+
const current: CachedPackage = {
324+
files: {}
325+
}
326+
current.files[pxt.CONFIG_NAME] = pkg
327+
const cfg: pxt.PackageConfig = JSON.parse(pkg)
328+
await U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1), async fn => {
329+
current.files[fn] = await downloadTextAsync(repopath, sha, fn);
330+
});
331+
332+
// cache!
333+
this.packages[key] = current;
334+
return copyCachedPackage(current);
335+
}
336+
catch (e) {
337+
const fallbackPackage = cachedPackageFromFallback(fallbackPackageFiles);
338+
if (fallbackPackage) return fallbackPackage;
339+
throw e;
340+
}
327341
}
328342

329343
async loadTutorialMarkdown(repopath: string, tag?: string) {
@@ -731,7 +745,7 @@ namespace pxt.github {
731745
return await db.loadConfigAsync(repopath, tag)
732746
}
733747

734-
export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig): Promise<CachedPackage> {
748+
export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig, fallbackPackageFiles?: pxt.Map<string>): Promise<CachedPackage> {
735749
const p = parseRepoId(repoWithTag)
736750
if (!p) {
737751
pxt.log('Unknown GitHub syntax');
@@ -746,9 +760,16 @@ namespace pxt.github {
746760

747761
// always try to upgrade unbound versions
748762
if (!p.tag) {
749-
p.tag = await db.latestVersionAsync(p.slug, config)
763+
try {
764+
p.tag = await db.latestVersionAsync(p.slug, config)
765+
}
766+
catch (e) {
767+
const fallbackPackage = cachedPackageFromFallback(fallbackPackageFiles);
768+
if (fallbackPackage) return fallbackPackage;
769+
throw e;
770+
}
750771
}
751-
const cached = await db.loadPackageAsync(p.fullName, p.tag)
772+
const cached = await db.loadPackageAsync(p.fullName, p.tag, fallbackPackageFiles)
752773
const dv = upgradedDisablesVariants(config, repoWithTag)
753774
if (dv) {
754775
const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME])
@@ -775,7 +796,11 @@ namespace pxt.github {
775796
return { version, config };
776797
}
777798

778-
export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise<void> {
799+
export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, fallbackPackageFilesById?: pxt.Map<pxt.Map<string>>): Promise<void> {
800+
await cacheProjectDependenciesCoreAsync(cfg, {}, fallbackPackageFilesById);
801+
}
802+
803+
async function cacheProjectDependenciesCoreAsync(cfg: pxt.PackageConfig, checked: pxt.Map<boolean>, fallbackPackageFilesById?: pxt.Map<pxt.Map<string>>): Promise<void> {
779804
const ghExtensions = Object.keys(cfg.dependencies)
780805
?.filter(dep => isGithubId(cfg.dependencies[dep]));
781806

@@ -786,10 +811,15 @@ namespace pxt.github {
786811
ghExtensions.map(
787812
async ext => {
788813
const extSrc = cfg.dependencies[ext];
789-
const ghPkg = await downloadPackageAsync(extSrc, pkgConfig);
814+
if (checked[extSrc]) return;
815+
checked[extSrc] = true;
816+
817+
const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, fallbackPackageFilesById?.[extSrc]);
790818
if (!ghPkg) {
791819
throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc));
792820
}
821+
const ghPkgCfg = Package.parseAndValidConfig(ghPkg.files[pxt.CONFIG_NAME]);
822+
if (ghPkgCfg) await cacheProjectDependenciesCoreAsync(ghPkgCfg, checked, fallbackPackageFilesById);
793823
}
794824
)
795825
);

‎pxtlib/main.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,8 @@ namespace pxt {
541541
export const TUTORIAL_CODE_STOP = "_onCodeStop.ts";
542542
export const TUTORIAL_INFO_FILE = "tutorial-info-cache.json";
543543
export const TUTORIAL_CUSTOM_TS = "tutorial.custom.ts";
544+
export const PACKAGED_EXTENSIONS = "_packaged-extensions.json";
545+
export const PACKAGED_EXT_INFO = "_packaged-ext-info.json";
544546
export const BREAKPOINT_TABLET = 991; // TODO (shakao) revisit when tutorial stuff is more settled
545547
export const PALETTES_FILE = "_palettes.json";
546548
export const HISTORY_FILE = "_history";

‎pxtlib/package.ts‎

Lines changed: 77 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1281,7 +1281,10 @@ namespace pxt {
12811281
!opts.target.isNative
12821282

12831283
if (!noFileEmbed) {
1284-
const files = await this.filesToBePublishedAsync(true)
1284+
const files = await this.filesToBePublishedAsync(true, !appTarget.compile.useUF2)
1285+
const packagedExtInfo = this.packagedExtensionHexInfo(opts);
1286+
if (Object.keys(packagedExtInfo).length)
1287+
files[pxt.PACKAGED_EXT_INFO] = JSON.stringify(packagedExtInfo);
12851288
const headerString = JSON.stringify({
12861289
name: this.config.name,
12871290
comment: this.config.description,
@@ -1357,24 +1360,81 @@ namespace pxt {
13571360
return cfg;
13581361
}
13591362

1360-
filesToBePublishedAsync(allowPrivate = false) {
1363+
async filesToBePublishedAsync(allowPrivate = false, packExternalExtensions = false): Promise<Map<string>> {
13611364
const files: Map<string> = {};
1362-
return this.loadAsync()
1363-
.then(() => {
1364-
if (!allowPrivate && !this.config.public)
1365-
U.userError('Only packages with "public":true can be published')
1366-
const cfg = this.prepareConfigToBePublished();
1367-
files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg);
1368-
for (let f of this.getFiles()) {
1369-
// already stored
1370-
if (f === pxt.CONFIG_NAME || f === HISTORY_FILE) continue;
1371-
let str = this.readFile(f)
1372-
if (str == null)
1373-
U.userError("referenced file missing: " + f)
1374-
files[f] = str
1365+
await this.loadAsync();
1366+
1367+
if (!allowPrivate && !this.config.public)
1368+
U.userError('Only packages with "public":true can be published')
1369+
1370+
const cfg = this.prepareConfigToBePublished();
1371+
files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg);
1372+
for (let f of this.getFiles()) {
1373+
// already stored
1374+
if (f === pxt.CONFIG_NAME || f === HISTORY_FILE) continue;
1375+
let str = this.readFile(f)
1376+
if (str == null)
1377+
U.userError("referenced file missing: " + f)
1378+
files[f] = str
1379+
}
1380+
1381+
if (packExternalExtensions) {
1382+
const packagedExtensions = this.packagedExternalExtensions();
1383+
if (Object.keys(packagedExtensions).length)
1384+
files[pxt.PACKAGED_EXTENSIONS] = JSON.stringify(packagedExtensions);
1385+
}
1386+
1387+
return U.sortObjectFields(files)
1388+
}
1389+
1390+
private packagedExternalExtensions(): pxt.Map<pxt.Map<string>> {
1391+
const packaged: pxt.Map<pxt.Map<string>> = {};
1392+
const seen: pxt.Map<boolean> = {};
1393+
1394+
const packDeps = (pkg: Package) => {
1395+
for (const dep of pkg.resolvedDependencies()) {
1396+
if (!dep) continue;
1397+
1398+
const seenKey = `${dep.id}:${dep.version()}`;
1399+
if (seen[seenKey]) continue;
1400+
seen[seenKey] = true;
1401+
1402+
if (dep.verProtocol() === "github" || dep.verProtocol() === "pub") {
1403+
const depFiles: pxt.Map<string> = {};
1404+
for (const fn of dep.getFiles()) {
1405+
if (fn === pxt.CONFIG_NAME || fn === HISTORY_FILE) continue;
1406+
const content = dep.readFile(fn);
1407+
if (content != null) depFiles[fn] = content;
1408+
}
1409+
depFiles[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(dep.config);
1410+
packaged[dep._verspec] = depFiles;
1411+
if (dep.version() !== dep._verspec)
1412+
packaged[dep.version()] = depFiles;
1413+
if (dep.verProtocol() === "pub")
1414+
packaged[dep.verArgument()] = depFiles;
13751415
}
1376-
return U.sortObjectFields(files)
1377-
})
1416+
1417+
packDeps(dep);
1418+
}
1419+
}
1420+
1421+
packDeps(this);
1422+
return packaged;
1423+
}
1424+
1425+
private packagedExtensionHexInfo(opts: pxtc.CompileOptions): pxt.Map<string> {
1426+
const packaged: pxt.Map<string> = {};
1427+
const targets = [opts, ...(opts.otherMultiVariants || [])];
1428+
1429+
for (const target of targets) {
1430+
const extInfo = target.extinfo;
1431+
if (!extInfo?.sha || !extInfo.hexinfo?.hex) continue;
1432+
1433+
const serialized = pxt.hexloader.stringifyHexInfoForCache(extInfo.hexinfo);
1434+
if (serialized) packaged[extInfo.sha] = serialized;
1435+
}
1436+
1437+
return packaged;
13781438
}
13791439

13801440
saveToJsonAsync(): Promise<pxt.cpp.HexFile> {

0 commit comments

Comments
 (0)