From d336d8336178dcd4a5d822df188f2d47aca73599 Mon Sep 17 00:00:00 2001 From: Yury Bondarenko Date: Thu, 13 Feb 2025 17:20:33 +0100 Subject: [PATCH 01/21] Cache-bust dynamic configuration files DSpace Angular's production server can modify the configuration it serves to CSR-only clients through YAML or environment variables. However, these files can remain cached in the browser and leave users with out-of-date configuration until the TTL runs out or the user does a "hard refresh". On build time this sort of problem is solved by saving the content hash as part of the path of each file (JS, CSS, ...) We introduce HashedFileMapping to bridge the same gap for dynamic content generated _after_ the server is built: - Files added to this mapping on the server are copied to a hashed path - A copy is injected into index.html, where clients can read it out and resolve the hashed paths - If a given path is not found in the mapping, the client will fall back to the original version (to prevent errors in development mode) With this mechanism we can ensure updates to config.json and similar files take immediate effect without losing the performance benefit of client-side caching. --- package.json | 1 + server.ts | 12 +- src/config/config.server.ts | 11 +- src/main.browser.ts | 5 +- .../hashed-file-mapping.browser.ts | 38 ++++++ .../hashed-file-mapping.server.ts | 115 ++++++++++++++++++ .../dynamic-hash/hashed-file-mapping.ts | 30 +++++ yarn.lock | 26 +++- 8 files changed, 229 insertions(+), 9 deletions(-) create mode 100644 src/modules/dynamic-hash/hashed-file-mapping.browser.ts create mode 100644 src/modules/dynamic-hash/hashed-file-mapping.server.ts create mode 100644 src/modules/dynamic-hash/hashed-file-mapping.ts diff --git a/package.json b/package.json index 719b13b23b6..d747d057782 100644 --- a/package.json +++ b/package.json @@ -121,6 +121,7 @@ "ngx-pagination": "6.0.3", "ngx-sortablejs": "^11.1.0", "ngx-ui-switch": "^14.0.3", + "node-html-parser": "^7.0.1", "nouislider": "^14.6.3", "pem": "1.14.7", "prop-types": "^15.8.1", diff --git a/server.ts b/server.ts index 23327c2058e..cee6fc73916 100644 --- a/server.ts +++ b/server.ts @@ -52,6 +52,7 @@ import { ServerAppModule } from './src/main.server'; import { buildAppConfig } from './src/config/config.server'; import { APP_CONFIG, AppConfig } from './src/config/app-config.interface'; import { extendEnvironmentWithAppConfig } from './src/config/config.util'; +import { ServerHashedFileMapping } from './src/modules/dynamic-hash/hashed-file-mapping.server'; import { logStartupMessage } from './startup-message'; import { TOKENITEM } from './src/app/core/auth/models/auth-token-info.model'; @@ -67,7 +68,10 @@ const indexHtml = join(DIST_FOLDER, 'index.html'); const cookieParser = require('cookie-parser'); -const appConfig: AppConfig = buildAppConfig(join(DIST_FOLDER, 'assets/config.json')); +const configJson = join(DIST_FOLDER, 'assets/config.json'); +const hashedFileMapping = new ServerHashedFileMapping(DIST_FOLDER, 'index.html'); +const appConfig: AppConfig = buildAppConfig(configJson, hashedFileMapping); +hashedFileMapping.save(); // cache of SSR pages for known bots, only enabled in production mode let botCache: LRU; @@ -256,7 +260,7 @@ function ngApp(req, res) { */ function serverSideRender(req, res, sendToUser: boolean = true) { // Render the page via SSR (server side rendering) - res.render(indexHtml, { + res.render(hashedFileMapping.resolve(indexHtml), { req, res, preboot: environment.universal.preboot, @@ -298,7 +302,7 @@ function serverSideRender(req, res, sendToUser: boolean = true) { * @param res current response */ function clientSideRender(req, res) { - res.sendFile(indexHtml); + res.sendFile(hashedFileMapping.resolve(indexHtml)); } @@ -487,7 +491,7 @@ function saveToCache(req, page: any) { */ function hasNotSucceeded(statusCode) { const rgx = new RegExp(/^20+/); - return !rgx.test(statusCode) + return !rgx.test(statusCode); } function retrieveHeaders(response) { diff --git a/src/config/config.server.ts b/src/config/config.server.ts index 65daede0a84..258c94567b6 100644 --- a/src/config/config.server.ts +++ b/src/config/config.server.ts @@ -2,6 +2,7 @@ import { red, blue, green, bold } from 'colors'; import { existsSync, readFileSync, writeFileSync } from 'fs'; import { load } from 'js-yaml'; import { join } from 'path'; +import { ServerHashedFileMapping } from '../modules/dynamic-hash/hashed-file-mapping.server'; import { AppConfig } from './app-config.interface'; import { Config } from './config.interface'; @@ -159,6 +160,7 @@ const buildBaseUrl = (config: ServerConfig): void => { ].join(''); }; + /** * Build app config with the following chain of override. * @@ -169,7 +171,7 @@ const buildBaseUrl = (config: ServerConfig): void => { * @param destConfigPath optional path to save config file * @returns app config */ -export const buildAppConfig = (destConfigPath?: string): AppConfig => { +export const buildAppConfig = (destConfigPath?: string, mapping?: ServerHashedFileMapping): AppConfig => { // start with default app config const appConfig: AppConfig = new DefaultAppConfig(); @@ -236,7 +238,12 @@ export const buildAppConfig = (destConfigPath?: string): AppConfig => { buildBaseUrl(appConfig.rest); if (isNotEmpty(destConfigPath)) { - writeFileSync(destConfigPath, JSON.stringify(appConfig, null, 2)); + const content = JSON.stringify(appConfig, null, 2); + + writeFileSync(destConfigPath, content); + if (mapping !== undefined) { + mapping.add(destConfigPath, content); + } console.log(`Angular ${bold('config.json')} file generated correctly at ${bold(destConfigPath)} \n`); } diff --git a/src/main.browser.ts b/src/main.browser.ts index 43b2ffbaf40..1b94d2de752 100644 --- a/src/main.browser.ts +++ b/src/main.browser.ts @@ -3,14 +3,15 @@ import 'reflect-metadata'; import 'core-js/es/reflect'; import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; - import { BrowserAppModule } from './modules/app/browser-app.module'; import { environment } from './environments/environment'; import { AppConfig } from './config/app-config.interface'; import { extendEnvironmentWithAppConfig } from './config/config.util'; import { enableProdMode } from '@angular/core'; +import { BrowserHashedFileMapping } from './modules/dynamic-hash/hashed-file-mapping.browser'; +const hashedFileMapping = new BrowserHashedFileMapping(); const bootstrap = () => platformBrowserDynamic() .bootstrapModule(BrowserAppModule, {}); @@ -32,7 +33,7 @@ const main = () => { return bootstrap(); } else { // Configuration must be fetched explicitly - return fetch('assets/config.json') + return fetch(hashedFileMapping.resolve('assets/config.json')) .then((response) => response.json()) .then((appConfig: AppConfig) => { // extend environment with app config for browser when not prerendered diff --git a/src/modules/dynamic-hash/hashed-file-mapping.browser.ts b/src/modules/dynamic-hash/hashed-file-mapping.browser.ts new file mode 100644 index 00000000000..02df8d6d545 --- /dev/null +++ b/src/modules/dynamic-hash/hashed-file-mapping.browser.ts @@ -0,0 +1,38 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +import { Injectable } from '@angular/core'; +import isObject from 'lodash/isObject'; +import isString from 'lodash/isString'; +import { hasValue } from '../../app/shared/empty.util'; +import { + HashedFileMapping, + ID, +} from './hashed-file-mapping'; + +/** + * Client-side implementation of {@link HashedFileMapping}. + * Reads out the mapping from index.html before the app is bootstrapped. + * Afterwards, {@link resolve} can be used to grab the latest file. + */ +@Injectable() +export class BrowserHashedFileMapping extends HashedFileMapping { + constructor() { + super(); + const element = document.querySelector(`script#${ID}`); + + if (hasValue(element?.textContent)) { + const mapping = JSON.parse(element.textContent); + + if (isObject(mapping)) { + Object.entries(mapping) + .filter(([key, value]) => isString(key) && isString(value)) + .forEach(([plainPath, hashPath]) => this.map.set(plainPath, hashPath)); + } + } + } +} diff --git a/src/modules/dynamic-hash/hashed-file-mapping.server.ts b/src/modules/dynamic-hash/hashed-file-mapping.server.ts new file mode 100644 index 00000000000..c709bab2782 --- /dev/null +++ b/src/modules/dynamic-hash/hashed-file-mapping.server.ts @@ -0,0 +1,115 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +import crypto from 'crypto'; +import { + readFileSync, + rmSync, + writeFileSync, +} from 'fs'; +import glob from 'glob'; +import { parse } from 'node-html-parser'; +import { + extname, + join, + relative, +} from 'path'; +import zlib from 'zlib'; +import { + HashedFileMapping, + ID, +} from './hashed-file-mapping'; + +/** + * Server-side implementation of {@link HashedFileMapping}. + * Registers dynamically hashed files and stores them in index.html for the browser to use. + */ +export class ServerHashedFileMapping extends HashedFileMapping { + public readonly indexPath: string; + private readonly indexContent: string; + + constructor( + private readonly root: string, + file: string, + ) { + super(); + this.root = join(root, ''); + this.indexPath = join(root, file); + this.indexContent = readFileSync(this.indexPath).toString(); + } + + /** + * Add a new file to the mapping by an absolute path (within the root directory). + * If {@link content} is provided, the {@link path} itself does not have to exist. + * Otherwise, it is read out from the original path. + * The original path is never overwritten. + */ + add(path: string, content?: string, compress = false) { + if (content === undefined) { + readFileSync(path); + } + + // remove previous files + const ext = extname(path); + glob.GlobSync(path.replace(`${ext}`, `.*${ext}*`)) + .found + .forEach(p => rmSync(p)); + + // hash the content + const hash = crypto.createHash('md5') + .update(content) + .digest('hex'); + + // add the hash to the path + const hashPath = path.replace(`${ext}`, `.${hash}${ext}`); + + // store it in the mapping + this.map.set(path, hashPath); + + // write the file + writeFileSync(hashPath, content); + + if (compress) { + // write the file as .br + zlib.brotliCompress(content, (err, compressed) => { + if (err) { + throw new Error('Brotli compress failed'); + } else { + writeFileSync(hashPath + '.br', compressed); + } + }); + + // write the file as .gz + zlib.gzip(content, (err, compressed) => { + if (err) { + throw new Error('Gzip compress failed'); + } else { + writeFileSync(hashPath + '.gz', compressed); + } + }); + } + } + + /** + * Save the mapping as JSON in the index file. + * The updated index file itself is hashed as well, and must be sent {@link resolve}. + */ + save(): void { + const out = Array.from(this.map.entries()) + .reduce((object, [plain, hashed]) => { + object[relative(this.root, plain)] = relative(this.root, hashed); + return object; + }, {}); + + let root = parse(this.indexContent); + root.querySelector(`head > script#${ID}`)?.remove(); + root.querySelector('head') + .appendChild(`` as any); + + this.add(this.indexPath, root.toString()); + } +} diff --git a/src/modules/dynamic-hash/hashed-file-mapping.ts b/src/modules/dynamic-hash/hashed-file-mapping.ts new file mode 100644 index 00000000000..a3348b3fd1b --- /dev/null +++ b/src/modules/dynamic-hash/hashed-file-mapping.ts @@ -0,0 +1,30 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +export const ID = 'hashed-file-mapping'; + +/** + * A mapping of plain path to hashed path, used for cache-busting files that are created dynamically after DSpace has been built. + * The production server can register hashed files here and attach the map to index.html. + * The browser can then use it to resolve hashed files and circumvent the browser cache. + * + * This can ensure that e.g. configuration changes are consistently served to all CSR clients. + */ +export abstract class HashedFileMapping { + protected readonly map: Map = new Map(); + + /** + * Resolve a hashed path based on a plain path. + */ + resolve(plainPath: string): string { + if (this.map.has(plainPath)) { + const hashPath = this.map.get(plainPath); + return hashPath; + } + return plainPath; + } +} diff --git a/yarn.lock b/yarn.lock index 730966fcdb5..8f60a5d4127 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4372,6 +4372,17 @@ css-select@^4.2.0, css-select@^4.3.0: domutils "^2.8.0" nth-check "^2.0.1" +css-select@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.1.0.tgz#b8ebd6554c3637ccc76688804ad3f6a6fdaea8a6" + integrity sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg== + dependencies: + boolbase "^1.0.0" + css-what "^6.1.0" + domhandler "^5.0.2" + domutils "^3.0.1" + nth-check "^2.0.1" + css-vendor@^2.0.8: version "2.0.8" resolved "https://registry.npmjs.org/css-vendor/-/css-vendor-2.0.8.tgz" @@ -4380,7 +4391,7 @@ css-vendor@^2.0.8: "@babel/runtime" "^7.8.3" is-in-browser "^1.0.2" -css-what@^6.0.1: +css-what@^6.0.1, css-what@^6.1.0: version "6.1.0" resolved "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz" integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== @@ -6184,6 +6195,11 @@ hdr-histogram-percentiles-obj@^3.0.0: resolved "https://registry.npmjs.org/hdr-histogram-percentiles-obj/-/hdr-histogram-percentiles-obj-3.0.0.tgz" integrity sha512-7kIufnBqdsBGcSZLPJwqHT3yhk1QTsSlFsVD3kx5ixH/AlgBs9yM1q6DPhXZ8f8gtdqgh7N7/5btRLpQsS2gHw== +he@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" + integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== + hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.2: version "3.3.2" resolved "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz" @@ -8277,6 +8293,14 @@ node-gyp@^9.0.0: tar "^6.1.2" which "^2.0.2" +node-html-parser@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-7.0.1.tgz#e3056550bae48517ebf161a0b0638f4b0123dfe3" + integrity sha512-KGtmPY2kS0thCWGK0VuPyOS+pBKhhe8gXztzA2ilAOhbUbxa9homF1bOyKvhGzMLXUoRds9IOmr/v5lr/lqNmA== + dependencies: + css-select "^5.1.0" + he "1.2.0" + node-releases@^2.0.8: version "2.0.10" resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz" From fdc6e8aaec25ab11f9cee3bcb93e65ce1415e907 Mon Sep 17 00:00:00 2001 From: Yury Bondarenko Date: Mon, 17 Feb 2025 12:33:37 +0100 Subject: [PATCH 02/21] Don't try to 'run' JSON on load The JSON mapping needs to be declared as a data block, otherwise browers may complain (nothing seems to really break though, except for Cypress) See https://developer.mozilla.org/en-US/docs/Web/HTML/Element/script#any_other_value --- src/modules/dynamic-hash/hashed-file-mapping.server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/dynamic-hash/hashed-file-mapping.server.ts b/src/modules/dynamic-hash/hashed-file-mapping.server.ts index c709bab2782..41e83e6b19c 100644 --- a/src/modules/dynamic-hash/hashed-file-mapping.server.ts +++ b/src/modules/dynamic-hash/hashed-file-mapping.server.ts @@ -106,9 +106,9 @@ export class ServerHashedFileMapping extends HashedFileMapping { }, {}); let root = parse(this.indexContent); - root.querySelector(`head > script#${ID}`)?.remove(); + root.querySelector(`script#${ID}`)?.remove(); root.querySelector('head') - .appendChild(`` as any); + .appendChild(`` as any); this.add(this.indexPath, root.toString()); } From 2f235991057154531f6d244e4b5df59b17078dd6 Mon Sep 17 00:00:00 2001 From: Yury Bondarenko Date: Mon, 17 Feb 2025 17:10:23 +0100 Subject: [PATCH 03/21] Add theme CSS to hashed file map --- server.ts | 1 + src/app/app.module.ts | 6 +++++ .../theme-support/theme.service.spec.ts | 13 +++++++++ src/app/shared/theme-support/theme.service.ts | 7 ++++- src/main.browser.ts | 2 +- .../hashed-file-mapping.browser.ts | 13 ++++++--- .../hashed-file-mapping.server.ts | 27 +++++++++++++++++-- 7 files changed, 62 insertions(+), 7 deletions(-) diff --git a/server.ts b/server.ts index cee6fc73916..e46f044923e 100644 --- a/server.ts +++ b/server.ts @@ -71,6 +71,7 @@ const cookieParser = require('cookie-parser'); const configJson = join(DIST_FOLDER, 'assets/config.json'); const hashedFileMapping = new ServerHashedFileMapping(DIST_FOLDER, 'index.html'); const appConfig: AppConfig = buildAppConfig(configJson, hashedFileMapping); +hashedFileMapping.addThemeStyles(); hashedFileMapping.save(); // cache of SSR pages for known bots, only enabled in production mode diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 89e361821ba..3ba227c134a 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -10,6 +10,8 @@ import { MetaReducer, StoreModule, USER_PROVIDED_META_REDUCERS } from '@ngrx/sto import { TranslateModule } from '@ngx-translate/core'; import { ScrollToModule } from '@nicky-lenaers/ngx-scroll-to'; import { DYNAMIC_MATCHER_PROVIDERS } from '@ng-dynamic-forms/core'; +import { HashedFileMapping } from '../modules/dynamic-hash/hashed-file-mapping'; +import { BrowserHashedFileMapping } from '../modules/dynamic-hash/hashed-file-mapping.browser'; import { AppRoutingModule } from './app-routing.module'; import { AppComponent } from './app.component'; @@ -103,6 +105,10 @@ const PROVIDERS = [ useClass: LogInterceptor, multi: true }, + { + provide: HashedFileMapping, + useClass: BrowserHashedFileMapping, + }, // register the dynamic matcher used by form. MUST be provided by the app module ...DYNAMIC_MATCHER_PROVIDERS, ]; diff --git a/src/app/shared/theme-support/theme.service.spec.ts b/src/app/shared/theme-support/theme.service.spec.ts index f56fb86cbc1..677bf065f6c 100644 --- a/src/app/shared/theme-support/theme.service.spec.ts +++ b/src/app/shared/theme-support/theme.service.spec.ts @@ -1,6 +1,7 @@ import { of as observableOf } from 'rxjs'; import { TestBed } from '@angular/core/testing'; import { provideMockActions } from '@ngrx/effects/testing'; +import { HashedFileMapping } from '../../../modules/dynamic-hash/hashed-file-mapping'; import { LinkService } from '../../core/cache/builders/link.service'; import { hot } from 'jasmine-marbles'; import { SetThemeAction } from './theme.actions'; @@ -46,6 +47,10 @@ class MockLinkService { } } +const mockHashedFileMapping = { + resolve: (path: string) => path, +}; + describe('ThemeService', () => { let themeService: ThemeService; let linkService: LinkService; @@ -96,6 +101,10 @@ describe('ThemeService', () => { provideMockActions(() => mockActions), { provide: DSpaceObjectDataService, useValue: mockDsoService }, { provide: Router, useValue: new RouterMock() }, + { + provide: HashedFileMapping, + useValue: mockHashedFileMapping, + }, ] }); @@ -393,6 +402,10 @@ describe('ThemeService', () => { provideMockStore({ initialState }), { provide: DSpaceObjectDataService, useValue: mockDsoService }, { provide: Router, useValue: new RouterMock() }, + { + provide: HashedFileMapping, + useValue: mockHashedFileMapping, + }, ] }); diff --git a/src/app/shared/theme-support/theme.service.ts b/src/app/shared/theme-support/theme.service.ts index 6d2939a5f88..8ffa3a2eb1c 100644 --- a/src/app/shared/theme-support/theme.service.ts +++ b/src/app/shared/theme-support/theme.service.ts @@ -1,6 +1,7 @@ import { Injectable, Inject, Injector } from '@angular/core'; import { Store, createFeatureSelector, createSelector, select } from '@ngrx/store'; import { BehaviorSubject, EMPTY, Observable, of as observableOf } from 'rxjs'; +import { HashedFileMapping } from '../../../modules/dynamic-hash/hashed-file-mapping'; import { ThemeState } from './theme.reducer'; import { SetThemeAction, ThemeActionTypes } from './theme.actions'; import { expand, filter, map, switchMap, take, toArray } from 'rxjs/operators'; @@ -57,6 +58,7 @@ export class ThemeService { @Inject(GET_THEME_CONFIG_FOR_FACTORY) private gtcf: (str) => ThemeConfig, private router: Router, @Inject(DOCUMENT) private document: any, + private hashedFileMapping: HashedFileMapping, ) { // Create objects from the theme configs in the environment file this.themes = environment.themes.map((themeConfig: ThemeConfig) => themeFactory(themeConfig, injector)); @@ -182,7 +184,10 @@ export class ThemeService { link.setAttribute('rel', 'stylesheet'); link.setAttribute('type', 'text/css'); link.setAttribute('class', 'theme-css'); - link.setAttribute('href', `${encodeURIComponent(themeName)}-theme.css`); + link.setAttribute( + 'href', + this.hashedFileMapping.resolve(`${encodeURIComponent(themeName)}-theme.css`), + ); // wait for the new css to download before removing the old one to prevent a // flash of unstyled content link.onload = () => { diff --git a/src/main.browser.ts b/src/main.browser.ts index 1b94d2de752..a73494cda2a 100644 --- a/src/main.browser.ts +++ b/src/main.browser.ts @@ -11,7 +11,7 @@ import { extendEnvironmentWithAppConfig } from './config/config.util'; import { enableProdMode } from '@angular/core'; import { BrowserHashedFileMapping } from './modules/dynamic-hash/hashed-file-mapping.browser'; -const hashedFileMapping = new BrowserHashedFileMapping(); +const hashedFileMapping = new BrowserHashedFileMapping(document); const bootstrap = () => platformBrowserDynamic() .bootstrapModule(BrowserAppModule, {}); diff --git a/src/modules/dynamic-hash/hashed-file-mapping.browser.ts b/src/modules/dynamic-hash/hashed-file-mapping.browser.ts index 02df8d6d545..576dc386314 100644 --- a/src/modules/dynamic-hash/hashed-file-mapping.browser.ts +++ b/src/modules/dynamic-hash/hashed-file-mapping.browser.ts @@ -5,7 +5,12 @@ * * http://www.dspace.org/license/ */ -import { Injectable } from '@angular/core'; +import { DOCUMENT } from '@angular/common'; +import { + Inject, + Injectable, + Optional, +} from '@angular/core'; import isObject from 'lodash/isObject'; import isString from 'lodash/isString'; import { hasValue } from '../../app/shared/empty.util'; @@ -21,9 +26,11 @@ import { */ @Injectable() export class BrowserHashedFileMapping extends HashedFileMapping { - constructor() { + constructor( + @Optional() @Inject(DOCUMENT) protected document: any, + ) { super(); - const element = document.querySelector(`script#${ID}`); + const element = document?.querySelector(`script#${ID}`); if (hasValue(element?.textContent)) { const mapping = JSON.parse(element.textContent); diff --git a/src/modules/dynamic-hash/hashed-file-mapping.server.ts b/src/modules/dynamic-hash/hashed-file-mapping.server.ts index 41e83e6b19c..822b3f5864f 100644 --- a/src/modules/dynamic-hash/hashed-file-mapping.server.ts +++ b/src/modules/dynamic-hash/hashed-file-mapping.server.ts @@ -10,6 +10,8 @@ import { readFileSync, rmSync, writeFileSync, + copyFileSync, + existsSync, } from 'fs'; import glob from 'glob'; import { parse } from 'node-html-parser'; @@ -48,9 +50,9 @@ export class ServerHashedFileMapping extends HashedFileMapping { * Otherwise, it is read out from the original path. * The original path is never overwritten. */ - add(path: string, content?: string, compress = false) { + add(path: string, content?: string, compress = false): string { if (content === undefined) { - readFileSync(path); + content = readFileSync(path).toString(); } // remove previous files @@ -92,6 +94,27 @@ export class ServerHashedFileMapping extends HashedFileMapping { } }); } + + return hashPath; + } + + addThemeStyles() { + glob.GlobSync(`${this.root}/*-theme.css`) + .found + .forEach(p => { + const hp = this.add(p); + this.ensureCompressedFilesAssumingUnchangedContent(p, hp, '.br'); + this.ensureCompressedFilesAssumingUnchangedContent(p, hp, '.gz'); + }); + } + + private ensureCompressedFilesAssumingUnchangedContent(path: string, hashedPath: string, compression: string) { + const compressedPath = `${path}${compression}`; + const compressedHashedPath = `${hashedPath}${compression}`; + + if (existsSync(compressedPath) && !existsSync(compressedHashedPath)) { + copyFileSync(compressedPath, compressedHashedPath); + } } /** From 02bbd7cbdb9da59cd4ee6e4cc816b70af0f1fa79 Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Thu, 16 Jan 2025 09:26:42 +0100 Subject: [PATCH 04/21] 124369: Add PrefetchConfig --- config/config.example.yml | 8 ++++++- .../dspace-rest/raw-rest-response.model.ts | 21 +++++++++++++++++++ src/config/app-config.interface.ts | 2 ++ src/config/default-app-config.ts | 8 +++++++ src/config/prefetch-config.ts | 18 ++++++++++++++++ src/environments/environment.test.ts | 8 ++++++- 6 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 src/config/prefetch-config.ts diff --git a/config/config.example.yml b/config/config.example.yml index ea38303fa36..d40d18ea8ee 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -379,4 +379,10 @@ vocabularies: # Default collection/community sorting order at Advanced search, Create/update community and collection when there are not a query. comcolSelectionSort: sortField: 'dc.title' - sortDirection: 'ASC' \ No newline at end of file + sortDirection: 'ASC' + +prefetch: + urls: + - /api + - /api/discover + refreshInterval: 60000 diff --git a/src/app/core/dspace-rest/raw-rest-response.model.ts b/src/app/core/dspace-rest/raw-rest-response.model.ts index 57f487f205d..f6ecbd22f95 100644 --- a/src/app/core/dspace-rest/raw-rest-response.model.ts +++ b/src/app/core/dspace-rest/raw-rest-response.model.ts @@ -11,3 +11,24 @@ export interface RawRestResponse { statusCode: number; statusText: string; } + +export interface RawBootstrapResponse { + payload: { + [name: string]: any; + _embedded?: any; + _links?: any; + page?: any; + }; + headers?: Record; + statusCode: number; + statusText: string; +} + +export function rawBootstrapToRawRestResponse(bootstrapResponse: RawBootstrapResponse): RawRestResponse { + return { + payload: bootstrapResponse.payload, + headers: new HttpHeaders(bootstrapResponse.headers), + statusCode: bootstrapResponse.statusCode, + statusText: bootstrapResponse.statusText, + }; +} diff --git a/src/config/app-config.interface.ts b/src/config/app-config.interface.ts index 84a30549a72..171cb8eb17f 100644 --- a/src/config/app-config.interface.ts +++ b/src/config/app-config.interface.ts @@ -22,6 +22,7 @@ import { HomeConfig } from './homepage-config.interface'; import { MarkdownConfig } from './markdown-config.interface'; import { FilterVocabularyConfig } from './filter-vocabulary-config'; import { DiscoverySortConfig } from './discovery-sort.config'; +import { PrefetchConfig } from './prefetch-config'; interface AppConfig extends Config { ui: UIServerConfig; @@ -48,6 +49,7 @@ interface AppConfig extends Config { markdown: MarkdownConfig; vocabularies: FilterVocabularyConfig[]; comcolSelectionSort: DiscoverySortConfig; + prefetch: PrefetchConfig; } /** diff --git a/src/config/default-app-config.ts b/src/config/default-app-config.ts index a6e9e092e46..f744772611c 100644 --- a/src/config/default-app-config.ts +++ b/src/config/default-app-config.ts @@ -22,6 +22,7 @@ import { HomeConfig } from './homepage-config.interface'; import { MarkdownConfig } from './markdown-config.interface'; import { FilterVocabularyConfig } from './filter-vocabulary-config'; import { DiscoverySortConfig } from './discovery-sort.config'; +import { PrefetchConfig } from './prefetch-config'; export class DefaultAppConfig implements AppConfig { production = false; @@ -432,4 +433,11 @@ export class DefaultAppConfig implements AppConfig { sortField:'dc.title', sortDirection:'ASC', }; + + // EndpointMap prefetching configuration + prefetch: PrefetchConfig = { + urls: [], + bootstrap: {}, + refreshInterval: 60000, // refresh every minute + }; } diff --git a/src/config/prefetch-config.ts b/src/config/prefetch-config.ts new file mode 100644 index 00000000000..e456f538e49 --- /dev/null +++ b/src/config/prefetch-config.ts @@ -0,0 +1,18 @@ +import { Config } from './config.interface'; +import { RawBootstrapResponse } from '../app/core/dspace-rest/raw-rest-response.model'; + +export interface PrefetchConfig extends Config { + /** + * The URLs that should be pre-fetched + */ + urls: string[]; + /** + * A mapping of URL to response + * bootstrap values should not be set manually, they will be overwritten. + */ + bootstrap: Record; + /** + * How often the HAL endpoints should be refreshed + */ + refreshInterval: number; +} diff --git a/src/environments/environment.test.ts b/src/environments/environment.test.ts index cb9d2c71303..448699f8a0d 100644 --- a/src/environments/environment.test.ts +++ b/src/environments/environment.test.ts @@ -313,5 +313,11 @@ export const environment: BuildConfig = { vocabulary: 'srsc', enabled: true } - ] + ], + + prefetch: { + urls: [], + bootstrap: {}, + refreshInterval: 60000, + } }; From 4024840c8a72105ea9172b00ddcb81a4cdfbd7e3 Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Wed, 19 Feb 2025 14:01:05 +0100 Subject: [PATCH 05/21] 124369: Prefetch requests and write to config --- server.ts | 17 +++++++++------ src/config/config.server.ts | 43 ++++++++++++++++++++++++++++++++++++- src/config/config.util.ts | 11 ++++++---- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/server.ts b/server.ts index 23327c2058e..55959459907 100644 --- a/server.ts +++ b/server.ts @@ -49,7 +49,7 @@ import { UIServerConfig } from './src/config/ui-server-config.interface'; import { ServerAppModule } from './src/main.server'; -import { buildAppConfig } from './src/config/config.server'; +import { buildAppConfig, setupEndpointPrefetching } from './src/config/config.server'; import { APP_CONFIG, AppConfig } from './src/config/app-config.interface'; import { extendEnvironmentWithAppConfig } from './src/config/config.util'; import { logStartupMessage } from './startup-message'; @@ -67,7 +67,8 @@ const indexHtml = join(DIST_FOLDER, 'index.html'); const cookieParser = require('cookie-parser'); -const appConfig: AppConfig = buildAppConfig(join(DIST_FOLDER, 'assets/config.json')); +const destConfigPath = join(DIST_FOLDER, 'assets/config.json'); +const appConfig: AppConfig = buildAppConfig(destConfigPath); // cache of SSR pages for known bots, only enabled in production mode let botCache: LRU; @@ -487,7 +488,7 @@ function saveToCache(req, page: any) { */ function hasNotSucceeded(statusCode) { const rgx = new RegExp(/^20+/); - return !rgx.test(statusCode) + return !rgx.test(statusCode); } function retrieveHeaders(response) { @@ -612,8 +613,12 @@ function healthCheck(req, res) { declare const __non_webpack_require__: NodeRequire; const mainModule = __non_webpack_require__.main; const moduleFilename = (mainModule && mainModule.filename) || ''; -if (moduleFilename === __filename || moduleFilename.includes('iisnode')) { - start(); -} +setupEndpointPrefetching(appConfig, destConfigPath, environment).then(() => { + if (moduleFilename === __filename || moduleFilename.includes('iisnode')) { + start(); + } +}).catch((error) => { + console.error('Errored while prefetching Endpoint Maps', error); +}); export * from './src/main.server'; diff --git a/src/config/config.server.ts b/src/config/config.server.ts index 65daede0a84..7ae8856d546 100644 --- a/src/config/config.server.ts +++ b/src/config/config.server.ts @@ -7,8 +7,9 @@ import { AppConfig } from './app-config.interface'; import { Config } from './config.interface'; import { DefaultAppConfig } from './default-app-config'; import { ServerConfig } from './server-config.interface'; -import { mergeConfig } from './config.util'; +import { mergeConfig, extendEnvironmentWithAppConfig } from './config.util'; import { isNotEmpty } from '../app/shared/empty.util'; +import { RawBootstrapResponse } from '../app/core/dspace-rest/raw-rest-response.model'; const CONFIG_PATH = join(process.cwd(), 'config'); @@ -243,3 +244,43 @@ export const buildAppConfig = (destConfigPath?: string): AppConfig => { return appConfig; }; + +export const setupEndpointPrefetching = async (appConfig: AppConfig, destConfigPath: string, env: any): Promise => { + await prefetchResponses(appConfig, destConfigPath, env); + + setInterval(() => void prefetchResponses(appConfig, destConfigPath, env), appConfig.prefetch.refreshInterval); +}; + +export const prefetchResponses = async (appConfig: AppConfig, destConfigPath: string, env: any): Promise => { + console.info('Prefetching endpoint maps'); + const restConfig = appConfig.rest; + const prefetchConfig = appConfig.prefetch; + + const baseUrl = restConfig.baseUrl; + const mapping: Record = {}; + + for (const relativeUrl of prefetchConfig.urls) { + const url = baseUrl + relativeUrl; + + const response = await fetch(url); + + const headers: Record = {}; + response.headers.forEach((value, header) => { + headers[header] = value; + }); + + const rawBootstrapResponse: RawBootstrapResponse = { + payload: await response.json(), + headers: headers, + statusCode: response.status, + statusText: response.statusText, + }; + + mapping[url] = rawBootstrapResponse; + } + + prefetchConfig.bootstrap = mapping; + + extendEnvironmentWithAppConfig(env, appConfig, false); + writeFileSync(destConfigPath, JSON.stringify(appConfig, null, 2)); +}; diff --git a/src/config/config.util.ts b/src/config/config.util.ts index 3d152d4563c..33af94aa278 100644 --- a/src/config/config.util.ts +++ b/src/config/config.util.ts @@ -10,12 +10,15 @@ import { ThemeConfig } from './theme.model'; /** * Extend Angular environment with app config. * - * @param env environment object - * @param appConfig app config + * @param env environment object + * @param appConfig app config + * @param logToConsole whether to log a message in the console */ -const extendEnvironmentWithAppConfig = (env: any, appConfig: AppConfig): void => { +const extendEnvironmentWithAppConfig = (env: any, appConfig: AppConfig, logToConsole = true): void => { mergeConfig(env, appConfig); - console.log(`Environment extended with app config`); + if (logToConsole) { + console.log(`Environment extended with app config`); + } }; /** From 66cdb69b3b17029922871cb6293f93f53b9f936c Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Fri, 31 Jan 2025 10:00:00 +0100 Subject: [PATCH 06/21] 124369: Separate request performing into method --- src/app/core/dspace-rest/dspace-rest.service.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/app/core/dspace-rest/dspace-rest.service.ts b/src/app/core/dspace-rest/dspace-rest.service.ts index ea4e8c28310..663aed70624 100644 --- a/src/app/core/dspace-rest/dspace-rest.service.ts +++ b/src/app/core/dspace-rest/dspace-rest.service.ts @@ -104,6 +104,11 @@ export class DspaceRestService { // Because HttpHeaders is immutable, the set method returns a new object instead of updating the existing headers requestOptions.headers = requestOptions.headers.set('Content-Type', DEFAULT_CONTENT_TYPE); } + + return this.performRequest(method, url, requestOptions); + } + + performRequest(method: RestRequestMethod, url: string, requestOptions: HttpOptions): Observable { return this.http.request(method, url, requestOptions).pipe( map((res) => ({ payload: res.body, From 7abd9bb191a325cb7f7cd0449230e624b578021b Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Fri, 31 Jan 2025 10:23:52 +0100 Subject: [PATCH 07/21] 124369: Retrieve prefetched response --- src/app/core/core.module.ts | 9 +++-- .../core/dspace-rest/dspace-rest.service.ts | 40 +++++++++++++++---- .../endpoint-mocking-rest.service.spec.ts | 3 +- .../endpoint-mocking-rest.service.ts | 6 ++- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/app/core/core.module.ts b/src/app/core/core.module.ts index dbca773375a..0121d316ec2 100644 --- a/src/app/core/core.module.ts +++ b/src/app/core/core.module.ts @@ -182,16 +182,17 @@ import { FlatBrowseDefinition } from './shared/flat-browse-definition.model'; import { ValueListBrowseDefinition } from './shared/value-list-browse-definition.model'; import { NonHierarchicalBrowseDefinition } from './shared/non-hierarchical-browse-definition'; import { BulkAccessConditionOptions } from './config/models/bulk-access-condition-options.model'; +import { APP_CONFIG, AppConfig } from '../../config/app-config.interface'; /** * When not in production, endpoint responses can be mocked for testing purposes * If there is no mock version available for the endpoint, the actual REST response will be used just like in production mode */ -export const restServiceFactory = (mocks: ResponseMapMock, http: HttpClient) => { +export const restServiceFactory = (mocks: ResponseMapMock, http: HttpClient, appConfig: AppConfig) => { if (environment.production) { - return new DspaceRestService(http); + return new DspaceRestService(http, appConfig); } else { - return new EndpointMockingRestService(mocks, http); + return new EndpointMockingRestService(mocks, http, appConfig); } }; @@ -212,7 +213,7 @@ const PROVIDERS = [ SiteDataService, DSOResponseParsingService, { provide: MOCK_RESPONSE_MAP, useValue: mockResponseMap }, - { provide: DspaceRestService, useFactory: restServiceFactory, deps: [MOCK_RESPONSE_MAP, HttpClient] }, + { provide: DspaceRestService, useFactory: restServiceFactory, deps: [MOCK_RESPONSE_MAP, HttpClient, APP_CONFIG] }, EPersonDataService, LinkHeadService, HALEndpointService, diff --git a/src/app/core/dspace-rest/dspace-rest.service.ts b/src/app/core/dspace-rest/dspace-rest.service.ts index 663aed70624..84733c6de6b 100644 --- a/src/app/core/dspace-rest/dspace-rest.service.ts +++ b/src/app/core/dspace-rest/dspace-rest.service.ts @@ -1,12 +1,13 @@ -import { Observable, throwError as observableThrowError } from 'rxjs'; -import { catchError, map } from 'rxjs/operators'; -import { Injectable } from '@angular/core'; +import { Observable, of as observableOf, throwError as observableThrowError } from 'rxjs'; +import { catchError, map, switchMap } from 'rxjs/operators'; +import { Injectable, Inject } from '@angular/core'; import { HttpClient, HttpHeaders, HttpParams, HttpResponse } from '@angular/common/http'; -import { RawRestResponse } from './raw-rest-response.model'; +import { RawRestResponse, rawBootstrapToRawRestResponse } from './raw-rest-response.model'; import { RestRequestMethod } from '../data/rest-request-method'; import { hasNoValue, hasValue, isNotEmpty } from '../../shared/empty.util'; import { DSpaceObject } from '../shared/dspace-object.model'; +import { APP_CONFIG, AppConfig } from '../../../config/app-config.interface'; export const DEFAULT_CONTENT_TYPE = 'application/json; charset=utf-8'; export interface HttpOptions { @@ -25,8 +26,10 @@ export interface HttpOptions { @Injectable() export class DspaceRestService { - constructor(protected http: HttpClient) { - + constructor( + protected http: HttpClient, + @Inject(APP_CONFIG) protected appConfig: AppConfig, + ) { } /** @@ -105,7 +108,14 @@ export class DspaceRestService { requestOptions.headers = requestOptions.headers.set('Content-Type', DEFAULT_CONTENT_TYPE); } - return this.performRequest(method, url, requestOptions); + return this.retrieveResponseFromConfig(method, url).pipe(switchMap(response => { + if (hasValue(response)) { + return observableOf(response); + } else { + return this.performRequest(method, url, requestOptions); + } + }) + ); } performRequest(method: RestRequestMethod, url: string, requestOptions: HttpOptions): Observable { @@ -150,4 +160,20 @@ export class DspaceRestService { return form; } + /** + * Returns an observable which emits a RawRestResponse if the request has been prefetched and stored in the config. + * Emits null otherwise. + */ + retrieveResponseFromConfig(method: RestRequestMethod, url: string): Observable { + if (method !== RestRequestMethod.OPTIONS && method !== RestRequestMethod.GET) { + return observableOf(null); + } + + const prefetchedResponses = this.appConfig?.prefetch?.bootstrap; + if (hasValue(prefetchedResponses?.[url])) { + return observableOf(rawBootstrapToRawRestResponse(prefetchedResponses[url])); + } else { + return observableOf(null); + } + } } diff --git a/src/app/shared/mocks/dspace-rest/endpoint-mocking-rest.service.spec.ts b/src/app/shared/mocks/dspace-rest/endpoint-mocking-rest.service.spec.ts index 0b6b218164b..777b734d215 100644 --- a/src/app/shared/mocks/dspace-rest/endpoint-mocking-rest.service.spec.ts +++ b/src/app/shared/mocks/dspace-rest/endpoint-mocking-rest.service.spec.ts @@ -3,6 +3,7 @@ import { of as observableOf } from 'rxjs'; import { RestRequestMethod } from '../../../core/data/rest-request-method'; import { EndpointMockingRestService } from './endpoint-mocking-rest.service'; import { ResponseMapMock } from './mocks/response-map.mock'; +import { environment } from '../../../../environments/environment.test'; describe('EndpointMockingRestService', () => { let service: EndpointMockingRestService; @@ -27,7 +28,7 @@ describe('EndpointMockingRestService', () => { request: observableOf(serverHttpResponse) }); - service = new EndpointMockingRestService(mockResponseMap, httpStub); + service = new EndpointMockingRestService(mockResponseMap, httpStub, environment); }); describe('get', () => { diff --git a/src/app/shared/mocks/dspace-rest/endpoint-mocking-rest.service.ts b/src/app/shared/mocks/dspace-rest/endpoint-mocking-rest.service.ts index 0358451557a..4fefc282c07 100644 --- a/src/app/shared/mocks/dspace-rest/endpoint-mocking-rest.service.ts +++ b/src/app/shared/mocks/dspace-rest/endpoint-mocking-rest.service.ts @@ -8,6 +8,7 @@ import { RawRestResponse } from '../../../core/dspace-rest/raw-rest-response.mod import { DspaceRestService, HttpOptions } from '../../../core/dspace-rest/dspace-rest.service'; import { MOCK_RESPONSE_MAP, ResponseMapMock } from './mocks/response-map.mock'; import { environment } from '../../../../environments/environment'; +import { APP_CONFIG, AppConfig } from '../../../../config/app-config.interface'; /** * Service to access DSpace's REST API. @@ -21,9 +22,10 @@ export class EndpointMockingRestService extends DspaceRestService { constructor( @Inject(MOCK_RESPONSE_MAP) protected mockResponseMap: ResponseMapMock, - protected http: HttpClient + protected http: HttpClient, + @Inject(APP_CONFIG) protected appConfig: AppConfig, ) { - super(http); + super(http, appConfig); } /** From 932bde7efd48772d30353ba7d5e729f9638a9f4f Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Mon, 3 Feb 2025 14:08:51 +0100 Subject: [PATCH 08/21] 124369: Fill request cache with prefetched requests in server & browser --- src/app/init.service.ts | 17 +++++++++++++++-- src/modules/app/browser-init.service.ts | 21 +++++++++++++++++++-- src/modules/app/server-init.service.ts | 7 ++++++- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/app/init.service.ts b/src/app/init.service.ts index 9fef2ca4edd..253dc0190aa 100644 --- a/src/app/init.service.ts +++ b/src/app/init.service.ts @@ -21,9 +21,11 @@ import { MetadataService } from './core/metadata/metadata.service'; import { BreadcrumbsService } from './breadcrumbs/breadcrumbs.service'; import { ThemeService } from './shared/theme-support/theme.service'; import { isAuthenticationBlocking } from './core/auth/selectors'; -import { distinctUntilChanged, find } from 'rxjs/operators'; +import { distinctUntilChanged, find, take } from 'rxjs/operators'; import { Observable } from 'rxjs'; import { MenuService } from './shared/menu/menu.service'; +import { hasValue } from './shared/empty.util'; +import { HrefOnlyDataService } from './core/data/href-only-data.service'; /** * Performs the initialization of the app. @@ -53,7 +55,7 @@ export abstract class InitService { protected breadcrumbsService: BreadcrumbsService, protected themeService: ThemeService, protected menuService: MenuService, - + protected hrefOnlyDataService: HrefOnlyDataService, ) { } @@ -201,4 +203,15 @@ export abstract class InitService { find((b: boolean) => b === false) ); } + + /** + * Use the bootstrapped requests to prefill the cache + */ + protected initBootstrapEndpoints() { + if (hasValue(this.appConfig?.prefetch?.bootstrap)) { + for (let url of Object.getOwnPropertyNames(this.appConfig.prefetch.bootstrap)) { + this.hrefOnlyDataService.findByHref(url, false).pipe(take(1)).subscribe(); + } + } + } } diff --git a/src/modules/app/browser-init.service.ts b/src/modules/app/browser-init.service.ts index 61d57f10f98..60295a99399 100644 --- a/src/modules/app/browser-init.service.ts +++ b/src/modules/app/browser-init.service.ts @@ -27,11 +27,12 @@ import { ThemeService } from '../../app/shared/theme-support/theme.service'; import { StoreAction, StoreActionTypes } from '../../app/store.actions'; import { coreSelector } from '../../app/core/core.selectors'; import { filter, find, map } from 'rxjs/operators'; -import { isNotEmpty } from '../../app/shared/empty.util'; +import { isNotEmpty, hasValue } from '../../app/shared/empty.util'; import { logStartupMessage } from '../../../startup-message'; import { MenuService } from '../../app/shared/menu/menu.service'; import { RootDataService } from '../../app/core/data/root-data.service'; import { firstValueFrom, Subscription } from 'rxjs'; +import { HrefOnlyDataService } from '../../app/core/data/href-only-data.service'; /** * Performs client-side initialization. @@ -56,7 +57,8 @@ export class BrowserInitService extends InitService { protected authService: AuthService, protected themeService: ThemeService, protected menuService: MenuService, - private rootDataService: RootDataService + protected rootDataService: RootDataService, + protected hrefOnlyDataService: HrefOnlyDataService, ) { super( store, @@ -69,6 +71,7 @@ export class BrowserInitService extends InitService { breadcrumbsService, themeService, menuService, + hrefOnlyDataService, ); } @@ -101,6 +104,8 @@ export class BrowserInitService extends InitService { this.initKlaro(); + this.initBootstrapEndpoints(); + await this.authenticationReady$().toPromise(); return true; @@ -172,4 +177,16 @@ export class BrowserInitService extends InitService { }); } + /** + * Use the bootstrapped requests from the server to prefill the cache on the client + */ + override initBootstrapEndpoints() { + super.initBootstrapEndpoints(); + + if (hasValue(this.appConfig?.prefetch?.bootstrap)) { + // Clear bootstrap once finished so the dspace-rest.service does not keep using the bootstrap + this.appConfig.prefetch.bootstrap = undefined; + } + } + } diff --git a/src/modules/app/server-init.service.ts b/src/modules/app/server-init.service.ts index 715f872cd9e..d87275b5af2 100644 --- a/src/modules/app/server-init.service.ts +++ b/src/modules/app/server-init.service.ts @@ -21,6 +21,7 @@ import { BreadcrumbsService } from '../../app/breadcrumbs/breadcrumbs.service'; import { ThemeService } from '../../app/shared/theme-support/theme.service'; import { take } from 'rxjs/operators'; import { MenuService } from '../../app/shared/menu/menu.service'; +import { HrefOnlyDataService } from '../../app/core/data/href-only-data.service'; /** * Performs server-side initialization. @@ -38,7 +39,8 @@ export class ServerInitService extends InitService { protected metadata: MetadataService, protected breadcrumbsService: BreadcrumbsService, protected themeService: ThemeService, - protected menuService: MenuService + protected menuService: MenuService, + protected hrefOnlyDataService: HrefOnlyDataService, ) { super( store, @@ -51,6 +53,7 @@ export class ServerInitService extends InitService { breadcrumbsService, themeService, menuService, + hrefOnlyDataService, ); } @@ -67,6 +70,8 @@ export class ServerInitService extends InitService { this.initRouteListeners(); this.themeService.listenForThemeChanges(false); + this.initBootstrapEndpoints(); + await this.authenticationReady$().toPromise(); return true; From 509ed7ad7800496cbcb321ab3a26a61cb5b0dfa2 Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Tue, 4 Feb 2025 15:07:56 +0100 Subject: [PATCH 09/21] 124369: Add StatisticsEndpoint model to models array of core.module --- src/app/core/core.module.ts | 4 +++- src/app/statistics/statistics.module.ts | 8 -------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/app/core/core.module.ts b/src/app/core/core.module.ts index 0121d316ec2..5e7c0a55dfe 100644 --- a/src/app/core/core.module.ts +++ b/src/app/core/core.module.ts @@ -183,6 +183,7 @@ import { ValueListBrowseDefinition } from './shared/value-list-browse-definition import { NonHierarchicalBrowseDefinition } from './shared/non-hierarchical-browse-definition'; import { BulkAccessConditionOptions } from './config/models/bulk-access-condition-options.model'; import { APP_CONFIG, AppConfig } from '../../config/app-config.interface'; +import { StatisticsEndpoint } from '../statistics/statistics-endpoint.model'; /** * When not in production, endpoint responses can be mocked for testing purposes @@ -381,7 +382,8 @@ export const models = IdentifierData, Subscription, ItemRequest, - BulkAccessConditionOptions + BulkAccessConditionOptions, + StatisticsEndpoint, ]; @NgModule({ diff --git a/src/app/statistics/statistics.module.ts b/src/app/statistics/statistics.module.ts index 4870e4fbf0b..f4d515030c5 100644 --- a/src/app/statistics/statistics.module.ts +++ b/src/app/statistics/statistics.module.ts @@ -3,14 +3,6 @@ import { CommonModule } from '@angular/common'; import { CoreModule } from '../core/core.module'; import { SharedModule } from '../shared/shared.module'; import { ViewTrackerComponent } from './angulartics/dspace/view-tracker.component'; -import { StatisticsEndpoint } from './statistics-endpoint.model'; - -/** - * Declaration needed to make sure all decorator functions are called in time - */ -export const models = [ - StatisticsEndpoint -]; @NgModule({ imports: [ From 93dbe13882ce950de20c60c3d654c2435294eaa0 Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Wed, 19 Feb 2025 15:40:54 +0100 Subject: [PATCH 10/21] 124369: Fix tests by providing APP_CONFIG --- src/app/core/auth/auth.interceptor.spec.ts | 3 +++ src/app/core/dspace-rest/dspace-rest.service.spec.ts | 7 ++++++- .../forward-client-ip.interceptor.spec.ts | 5 ++++- src/app/core/locale/locale.interceptor.spec.ts | 3 +++ src/app/core/log/log.interceptor.spec.ts | 3 +++ src/app/core/xsrf/xsrf.interceptor.spec.ts | 5 ++++- src/app/init.service.spec.ts | 3 +++ 7 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/app/core/auth/auth.interceptor.spec.ts b/src/app/core/auth/auth.interceptor.spec.ts index 04bbc4acaf0..dc6998ef7d4 100644 --- a/src/app/core/auth/auth.interceptor.spec.ts +++ b/src/app/core/auth/auth.interceptor.spec.ts @@ -13,6 +13,8 @@ import { RouterStub } from '../../shared/testing/router.stub'; import { TruncatablesState } from '../../shared/truncatable/truncatable.reducer'; import { AuthServiceStub } from '../../shared/testing/auth-service.stub'; import { RestRequestMethod } from '../data/rest-request-method'; +import { APP_CONFIG } from '../../../config/app-config.interface'; +import { environment } from 'src/environments/environment.test'; describe(`AuthInterceptor`, () => { let service: DspaceRestService; @@ -39,6 +41,7 @@ describe(`AuthInterceptor`, () => { multi: true, }, { provide: Store, useValue: store }, + { provide: APP_CONFIG, useValue: environment }, ], }); diff --git a/src/app/core/dspace-rest/dspace-rest.service.spec.ts b/src/app/core/dspace-rest/dspace-rest.service.spec.ts index 2188522dc0c..e9437086d09 100644 --- a/src/app/core/dspace-rest/dspace-rest.service.spec.ts +++ b/src/app/core/dspace-rest/dspace-rest.service.spec.ts @@ -5,6 +5,8 @@ import { DEFAULT_CONTENT_TYPE, DspaceRestService } from './dspace-rest.service'; import { DSpaceObject } from '../shared/dspace-object.model'; import { RestRequestMethod } from '../data/rest-request-method'; import { HttpHeaders } from '@angular/common/http'; +import { APP_CONFIG } from '../../../config/app-config.interface'; +import { environment } from '../../../environments/environment.test'; describe('DspaceRestService', () => { let dspaceRestService: DspaceRestService; @@ -19,7 +21,10 @@ describe('DspaceRestService', () => { beforeEach(() => { TestBed.configureTestingModule({ imports: [HttpClientTestingModule], - providers: [DspaceRestService] + providers: [ + DspaceRestService, + { provide: APP_CONFIG, useValue: environment }, + ], }); dspaceRestService = TestBed.inject(DspaceRestService); diff --git a/src/app/core/forward-client-ip/forward-client-ip.interceptor.spec.ts b/src/app/core/forward-client-ip/forward-client-ip.interceptor.spec.ts index bd9e4f90764..e6525e51e11 100644 --- a/src/app/core/forward-client-ip/forward-client-ip.interceptor.spec.ts +++ b/src/app/core/forward-client-ip/forward-client-ip.interceptor.spec.ts @@ -4,6 +4,8 @@ import { TestBed } from '@angular/core/testing'; import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; import { HTTP_INTERCEPTORS } from '@angular/common/http'; import { REQUEST } from '@nguniversal/express-engine/tokens'; +import { APP_CONFIG } from '../../../config/app-config.interface'; +import { environment } from 'src/environments/environment.test'; describe('ForwardClientIpInterceptor', () => { let service: DspaceRestService; @@ -25,7 +27,8 @@ describe('ForwardClientIpInterceptor', () => { useClass: ForwardClientIpInterceptor, multi: true, }, - { provide: REQUEST, useValue: { get: () => undefined, connection: { remoteAddress: clientIp } } } + { provide: REQUEST, useValue: { get: () => undefined, connection: { remoteAddress: clientIp } } }, + { provide: APP_CONFIG, useValue: environment }, ], }); diff --git a/src/app/core/locale/locale.interceptor.spec.ts b/src/app/core/locale/locale.interceptor.spec.ts index e96126d19c8..13bf2c56f76 100644 --- a/src/app/core/locale/locale.interceptor.spec.ts +++ b/src/app/core/locale/locale.interceptor.spec.ts @@ -7,6 +7,8 @@ import { RestRequestMethod } from '../data/rest-request-method'; import { LocaleService } from './locale.service'; import { LocaleInterceptor } from './locale.interceptor'; import { of } from 'rxjs'; +import { APP_CONFIG } from '../../../config/app-config.interface'; +import { environment } from '../../../environments/environment.test'; describe(`LocaleInterceptor`, () => { let service: DspaceRestService; @@ -31,6 +33,7 @@ describe(`LocaleInterceptor`, () => { multi: true, }, { provide: LocaleService, useValue: mockLocaleService }, + { provide: APP_CONFIG, useValue: environment }, ], }); diff --git a/src/app/core/log/log.interceptor.spec.ts b/src/app/core/log/log.interceptor.spec.ts index cae9c322029..29f51de196e 100644 --- a/src/app/core/log/log.interceptor.spec.ts +++ b/src/app/core/log/log.interceptor.spec.ts @@ -13,6 +13,8 @@ import { CorrelationIdService } from '../../correlation-id/correlation-id.servic import { UUIDService } from '../shared/uuid.service'; import { StoreModule } from '@ngrx/store'; import { appReducers, storeModuleConfig } from '../../app.reducer'; +import { APP_CONFIG } from '../../../config/app-config.interface'; +import { environment } from '../../../environments/environment.test'; describe('LogInterceptor', () => { @@ -49,6 +51,7 @@ describe('LogInterceptor', () => { { provide: Router, useValue: router }, { provide: CorrelationIdService, useClass: CorrelationIdService }, { provide: UUIDService, useClass: UUIDService }, + { provide: APP_CONFIG, useValue: environment }, ], }); diff --git a/src/app/core/xsrf/xsrf.interceptor.spec.ts b/src/app/core/xsrf/xsrf.interceptor.spec.ts index 4a78b60fc17..9fd5215ffe9 100644 --- a/src/app/core/xsrf/xsrf.interceptor.spec.ts +++ b/src/app/core/xsrf/xsrf.interceptor.spec.ts @@ -7,6 +7,8 @@ import { CookieService } from '../services/cookie.service'; import { CookieServiceMock } from '../../shared/mocks/cookie.service.mock'; import { XsrfInterceptor } from './xsrf.interceptor'; import { HttpXsrfTokenExtractorMock } from '../../shared/mocks/http-xsrf-token-extractor.mock'; +import { APP_CONFIG } from '../../../config/app-config.interface'; +import { environment } from 'src/environments/environment.test'; describe(`XsrfInterceptor`, () => { let service: DspaceRestService; @@ -35,7 +37,8 @@ describe(`XsrfInterceptor`, () => { multi: true, }, { provide: HttpXsrfTokenExtractor, useValue: new HttpXsrfTokenExtractorMock(testToken) }, - { provide: CookieService, useValue: new CookieServiceMock() } + { provide: CookieService, useValue: new CookieServiceMock() }, + { provide: APP_CONFIG, useValue: environment }, ], }); diff --git a/src/app/init.service.spec.ts b/src/app/init.service.spec.ts index a05f06c78fa..59cfe2a5098 100644 --- a/src/app/init.service.spec.ts +++ b/src/app/init.service.spec.ts @@ -31,6 +31,8 @@ import objectContaining = jasmine.objectContaining; import createSpyObj = jasmine.createSpyObj; import SpyObj = jasmine.SpyObj; import { getTestScheduler } from 'jasmine-marbles'; +import { HrefOnlyDataService } from './core/data/href-only-data.service'; +import { getMockHrefOnlyDataService } from './shared/mocks/href-only-data.service.mock'; let spy: SpyObj; @@ -180,6 +182,7 @@ describe('InitService', () => { { provide: ActivatedRoute, useValue: new MockActivatedRoute() }, { provide: MenuService, useValue: menuServiceSpy }, { provide: ThemeService, useValue: getMockThemeService() }, + { provide: HrefOnlyDataService, useValue: getMockHrefOnlyDataService() }, provideMockStore({ initialState }), AppComponent, RouteService, From ba91fe55bc544a31a4cbf18dea705fcfa28affa9 Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Thu, 20 Feb 2025 10:53:00 +0100 Subject: [PATCH 11/21] 124369: Retrieve 'browse' model constructors by their 'browseType' --- .../dspace-rest-response-parsing.service.ts | 26 ++++++++++++------- .../endpoint-map-response-parsing.service.ts | 2 +- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/app/core/data/dspace-rest-response-parsing.service.ts b/src/app/core/data/dspace-rest-response-parsing.service.ts index 500afc4aff6..604a0959ce4 100644 --- a/src/app/core/data/dspace-rest-response-parsing.service.ts +++ b/src/app/core/data/dspace-rest-response-parsing.service.ts @@ -191,13 +191,12 @@ export class DspaceRestResponseParsingService implements ResponseParsingService } protected deserialize(obj): any { - const type = obj.type; - const objConstructor = this.getConstructorFor(type); + const objConstructor = this.getConstructorFor(obj); if (hasValue(objConstructor)) { const serializer = new this.serializerConstructor(objConstructor); return serializer.deserialize(obj); } else { - console.warn('cannot deserialize type ' + type); + console.warn('cannot deserialize type ', obj?.type); return null; } } @@ -206,15 +205,24 @@ export class DspaceRestResponseParsingService implements ResponseParsingService * Returns the constructor for the given type, or null if there isn't a registered model for that * type * - * @param type the object to find the constructor for. + * @param obj the object to find the constructor for. * @protected */ - protected getConstructorFor(type: string): GenericConstructor { - if (hasValue(type)) { - return getClassForType(type) as GenericConstructor; - } else { - return null; + protected getConstructorFor(obj: any): GenericConstructor { + if (hasValue(obj?.type)) { + const constructor = getClassForType(obj.type) as GenericConstructor; + + if (hasValue(constructor)) { + return constructor; + } + + // Browses have a subtype, so to get the correct constructor it has to be retrieved based on 'browseType' + if (obj.type === 'browse') { + return getClassForType(obj.browseType) as GenericConstructor; + } } + + return null; } /** diff --git a/src/app/core/data/endpoint-map-response-parsing.service.ts b/src/app/core/data/endpoint-map-response-parsing.service.ts index 728714876c4..cb4a6991b18 100644 --- a/src/app/core/data/endpoint-map-response-parsing.service.ts +++ b/src/app/core/data/endpoint-map-response-parsing.service.ts @@ -110,7 +110,7 @@ export class EndpointMapResponseParsingService extends DspaceRestResponseParsing return; } - if (hasValue(this.getConstructorFor((co as any).type))) { + if (hasValue(this.getConstructorFor(co as any))) { this.objectCache.add(co, hasValue(request.responseMsToLive) ? request.responseMsToLive : environment.cache.msToLive.default, request.uuid, alternativeURL); } } From 38ed2481474375a4bf1b108a24b338c4fe9831eb Mon Sep 17 00:00:00 2001 From: Yury Bondarenko Date: Mon, 24 Feb 2025 12:07:47 +0100 Subject: [PATCH 12/21] Follow REST model for browe definitions more closely --- .../browse/browse-definition-data.service.ts | 49 ++----------------- .../core/cache/builders/build-decorators.ts | 44 +++++++++++++++-- .../builders/remote-data-build.service.ts | 4 +- src/app/core/cache/object-cache.service.ts | 4 +- .../data/base-response-parsing.service.ts | 4 +- src/app/core/data/base/create-data.ts | 32 +++++++----- .../data/default-change-analyzer.service.ts | 14 +++--- .../dspace-rest-response-parsing.service.ts | 44 +++++++++-------- .../endpoint-map-response-parsing.service.ts | 22 ++++----- src/app/core/data/request.effects.ts | 38 +++++++++----- .../core/shared/browse-definition.model.ts | 7 +++ .../shared/flat-browse-definition.model.ts | 8 +-- .../hierarchical-browse-definition.model.ts | 11 +++-- .../value-list-browse-definition.model.ts | 8 +-- 14 files changed, 161 insertions(+), 128 deletions(-) diff --git a/src/app/core/browse/browse-definition-data.service.ts b/src/app/core/browse/browse-definition-data.service.ts index bc495a51f4f..f8d02ffa912 100644 --- a/src/app/core/browse/browse-definition-data.service.ts +++ b/src/app/core/browse/browse-definition-data.service.ts @@ -6,56 +6,17 @@ import { RemoteDataBuildService } from '../cache/builders/remote-data-build.serv import { ObjectCacheService } from '../cache/object-cache.service'; import { HALEndpointService } from '../shared/hal-endpoint.service'; import { FollowLinkConfig } from '../../shared/utils/follow-link-config.model'; -import { Observable, of as observableOf } from 'rxjs'; +import { Observable } from 'rxjs'; import { RemoteData } from '../data/remote-data'; import { PaginatedList } from '../data/paginated-list.model'; import { FindListOptions } from '../data/find-list-options.model'; import { IdentifiableDataService } from '../data/base/identifiable-data.service'; import { FindAllData, FindAllDataImpl } from '../data/base/find-all-data'; import { dataService } from '../data/base/data-service.decorator'; -import { isNotEmpty, isNotEmptyOperator, hasValue } from '../../shared/empty.util'; -import { take } from 'rxjs/operators'; -import { BrowseDefinitionRestRequest } from '../data/request.models'; import { RequestParam } from '../cache/models/request-param.model'; import { SearchData, SearchDataImpl } from '../data/base/search-data'; import { BrowseDefinition } from '../shared/browse-definition.model'; -/** - * Create a GET request for the given href, and send it. - * Use a GET request specific for BrowseDefinitions. - */ -export const createAndSendBrowseDefinitionGetRequest = (requestService: RequestService, - responseMsToLive: number, - href$: string | Observable, - useCachedVersionIfAvailable: boolean = true): void => { - if (isNotEmpty(href$)) { - if (typeof href$ === 'string') { - href$ = observableOf(href$); - } - - href$.pipe( - isNotEmptyOperator(), - take(1) - ).subscribe((href: string) => { - const requestId = requestService.generateRequestId(); - const request = new BrowseDefinitionRestRequest(requestId, href); - if (hasValue(responseMsToLive)) { - request.responseMsToLive = responseMsToLive; - } - requestService.send(request, useCachedVersionIfAvailable); - }); - } -}; - -/** - * Custom extension of {@link FindAllDataImpl} to be able to send BrowseDefinitionRestRequests - */ -class BrowseDefinitionFindAllDataImpl extends FindAllDataImpl { - createAndSendGetRequest(href$: string | Observable, useCachedVersionIfAvailable: boolean = true) { - createAndSendBrowseDefinitionGetRequest(this.requestService, this.responseMsToLive, href$, useCachedVersionIfAvailable); - } -} - /** * Data service responsible for retrieving browse definitions from the REST server */ @@ -64,7 +25,7 @@ class BrowseDefinitionFindAllDataImpl extends FindAllDataImpl }) @dataService(BROWSE_DEFINITION) export class BrowseDefinitionDataService extends IdentifiableDataService implements FindAllData, SearchData { - private findAllData: BrowseDefinitionFindAllDataImpl; + private findAllData: FindAllData; private searchData: SearchDataImpl; constructor( @@ -75,7 +36,7 @@ export class BrowseDefinitionDataService extends IdentifiableDataService, useCachedVersionIfAvailable: boolean = true) { - createAndSendBrowseDefinitionGetRequest(this.requestService, this.responseMsToLive, href$, useCachedVersionIfAvailable); - } } diff --git a/src/app/core/cache/builders/build-decorators.ts b/src/app/core/cache/builders/build-decorators.ts index 9e5ebaed854..69f4f10b124 100644 --- a/src/app/core/cache/builders/build-decorators.ts +++ b/src/app/core/cache/builders/build-decorators.ts @@ -27,15 +27,51 @@ const linkMap = new Map(); * @param target the typed class to map */ export function typedObject(target: TypedObject) { - typeMap.set(target.type.value, target); + typeMap.set(target.type.value, new Map([[null, target]])); +} + +/** + * Decorator function to map a ResourceType and sub-type to its class + */ +export function typedObjectWithSubType(subTypeProperty: string) { + return function(target: TypedObject) { + if (hasNoValue(target[subTypeProperty]?.value)) { + throw new Error(`Class ${(target as any).name} has no static property '${subTypeProperty}' to define as the sub-type`); + } + + if (!typeMap.has(target.type.value)) { + typeMap.set(target.type.value, new Map()); + } + if (!typeMap.get(target.type.value).has(subTypeProperty)) { + typeMap.get(target.type.value) + .set(subTypeProperty, new Map()); + } + + typeMap.get(target.type.value) + .get(subTypeProperty) + .set(target[subTypeProperty].value, target); + }; } /** * Returns the mapped class for the given type - * @param type The resource type */ -export function getClassForType(type: string | ResourceType) { - return typeMap.get(getResourceTypeValueFor(type)); +export function getClassForObject(obj: any) { + const map = typeMap.get(getResourceTypeValueFor(obj.type)); + + if (hasValue(map)) { + for (const subTypeProperty of map.keys()) { + if (subTypeProperty === null) { + // Regular class without subtype + return map.get(subTypeProperty); + } else if (hasValue(obj?.[subTypeProperty])) { + // Class with subtype + return map.get(subTypeProperty).get(getResourceTypeValueFor(obj?.[subTypeProperty])); + } + } + } + + return undefined; } /** diff --git a/src/app/core/cache/builders/remote-data-build.service.ts b/src/app/core/cache/builders/remote-data-build.service.ts index 075bf3ca0ca..7fd8c5aaf0d 100644 --- a/src/app/core/cache/builders/remote-data-build.service.ts +++ b/src/app/core/cache/builders/remote-data-build.service.ts @@ -16,7 +16,7 @@ import { ObjectCacheService } from '../object-cache.service'; import { LinkService } from './link.service'; import { HALLink } from '../../shared/hal-link.model'; import { GenericConstructor } from '../../shared/generic-constructor'; -import { getClassForType } from './build-decorators'; +import { getClassForObject } from './build-decorators'; import { HALResource } from '../../shared/hal-resource.model'; import { PAGINATED_LIST } from '../../data/paginated-list.resource-type'; import { getUrlWithoutEmbedParams } from '../../index/index.selectors'; @@ -90,7 +90,7 @@ export class RemoteDataBuildService { * @param obj The object to turn in to a class instance based on its type property */ private plainObjectToInstance(obj: any): T { - const type: GenericConstructor = getClassForType(obj.type); + const type: GenericConstructor = getClassForObject(obj); if (typeof type === 'function') { return Object.assign(new type(), obj) as T; } else { diff --git a/src/app/core/cache/object-cache.service.ts b/src/app/core/cache/object-cache.service.ts index 9ca02162108..91822dd12b4 100644 --- a/src/app/core/cache/object-cache.service.ts +++ b/src/app/core/cache/object-cache.service.ts @@ -10,7 +10,7 @@ import { coreSelector } from '../core.selectors'; import { RestRequestMethod } from '../data/rest-request-method'; import { selfLinkFromAlternativeLinkSelector, selfLinkFromUuidSelector } from '../index/index.selectors'; import { GenericConstructor } from '../shared/generic-constructor'; -import { getClassForType } from './builders/build-decorators'; +import { getClassForObject } from './builders/build-decorators'; import { LinkService } from './builders/link.service'; import { AddDependentsObjectCacheAction, AddPatchObjectCacheAction, AddToObjectCacheAction, ApplyPatchObjectCacheAction, RemoveDependentsObjectCacheAction, RemoveFromObjectCacheAction } from './object-cache.actions'; @@ -139,7 +139,7 @@ export class ObjectCacheService { } ), map((entry: ObjectCacheEntry) => { - const type: GenericConstructor = getClassForType((entry.data as any).type); + const type: GenericConstructor = getClassForObject(entry.data as any); if (typeof type !== 'function') { throw new Error(`${type} is not a valid constructor for ${JSON.stringify(entry.data)}`); } diff --git a/src/app/core/data/base-response-parsing.service.ts b/src/app/core/data/base-response-parsing.service.ts index 18e6623683f..ff91ef411ce 100644 --- a/src/app/core/data/base-response-parsing.service.ts +++ b/src/app/core/data/base-response-parsing.service.ts @@ -6,7 +6,7 @@ import { PageInfo } from '../shared/page-info.model'; import { ObjectCacheService } from '../cache/object-cache.service'; import { GenericConstructor } from '../shared/generic-constructor'; import { PaginatedList, buildPaginatedList } from './paginated-list.model'; -import { getClassForType } from '../cache/builders/build-decorators'; +import { getClassForObject } from '../cache/builders/build-decorators'; import { environment } from '../../../environments/environment'; import { CacheableObject } from '../cache/cacheable-object.model'; import { RestRequest } from './rest-request.model'; @@ -111,7 +111,7 @@ export abstract class BaseResponseParsingService { protected deserialize(obj): any { const type: string = obj.type; if (hasValue(type)) { - const objConstructor = getClassForType(type) as GenericConstructor; + const objConstructor = getClassForObject(obj) as GenericConstructor; if (hasValue(objConstructor)) { const serializer = new this.serializerConstructor(objConstructor); diff --git a/src/app/core/data/base/create-data.ts b/src/app/core/data/base/create-data.ts index 3ffcd9adf20..85beb96255c 100644 --- a/src/app/core/data/base/create-data.ts +++ b/src/app/core/data/base/create-data.ts @@ -5,22 +5,30 @@ * * http://www.dspace.org/license/ */ +import { Observable } from 'rxjs'; +import { + distinctUntilChanged, + map, + take, + takeWhile, +} from 'rxjs/operators'; +import { + hasValue, + isNotEmptyOperator, +} from '../../../shared/empty.util'; +import { NotificationOptions } from '../../../shared/notifications/models/notification-options.model'; +import { NotificationsService } from '../../../shared/notifications/notifications.service'; +import { getClassForObject } from '../../cache/builders/build-decorators'; +import { RemoteDataBuildService } from '../../cache/builders/remote-data-build.service'; import { CacheableObject } from '../../cache/cacheable-object.model'; -import { BaseDataService } from './base-data.service'; import { RequestParam } from '../../cache/models/request-param.model'; -import { Observable } from 'rxjs'; -import { RemoteData } from '../remote-data'; -import { hasValue, isNotEmptyOperator } from '../../../shared/empty.util'; -import { distinctUntilChanged, map, take, takeWhile } from 'rxjs/operators'; +import { ObjectCacheService } from '../../cache/object-cache.service'; import { DSpaceSerializer } from '../../dspace-rest/dspace.serializer'; -import { getClassForType } from '../../cache/builders/build-decorators'; +import { HALEndpointService } from '../../shared/hal-endpoint.service'; +import { RemoteData } from '../remote-data'; import { CreateRequest } from '../request.models'; -import { NotificationOptions } from '../../../shared/notifications/models/notification-options.model'; import { RequestService } from '../request.service'; -import { RemoteDataBuildService } from '../../cache/builders/remote-data-build.service'; -import { HALEndpointService } from '../../shared/hal-endpoint.service'; -import { NotificationsService } from '../../../shared/notifications/notifications.service'; -import { ObjectCacheService } from '../../cache/object-cache.service'; +import { BaseDataService } from './base-data.service'; /** * Interface for a data service that can create objects. @@ -78,7 +86,7 @@ export class CreateDataImpl extends BaseDataService): Observable> { const requestId = this.requestService.generateRequestId(); - const serializedObject = new DSpaceSerializer(getClassForType(object.type)).serialize(object); + const serializedObject = new DSpaceSerializer(getClassForObject(object)).serialize(object); endpoint$.pipe( take(1), diff --git a/src/app/core/data/default-change-analyzer.service.ts b/src/app/core/data/default-change-analyzer.service.ts index 70c45bbc2de..ecb8325eefa 100644 --- a/src/app/core/data/default-change-analyzer.service.ts +++ b/src/app/core/data/default-change-analyzer.service.ts @@ -1,10 +1,12 @@ import { Injectable } from '@angular/core'; -import { compare } from 'fast-json-patch'; -import { Operation } from 'fast-json-patch'; -import { getClassForType } from '../cache/builders/build-decorators'; +import { + compare, + Operation, +} from 'fast-json-patch'; +import { getClassForObject } from '../cache/builders/build-decorators'; +import { TypedObject } from '../cache/typed-object.model'; import { DSpaceNotNullSerializer } from '../dspace-rest/dspace-not-null.serializer'; import { ChangeAnalyzer } from './change-analyzer'; -import { TypedObject } from '../cache/typed-object.model'; /** * A class to determine what differs between two @@ -22,8 +24,8 @@ export class DefaultChangeAnalyzer implements ChangeAnaly * The second object to compare */ diff(object1: T, object2: T): Operation[] { - const serializer1 = new DSpaceNotNullSerializer(getClassForType(object1.type)); - const serializer2 = new DSpaceNotNullSerializer(getClassForType(object2.type)); + const serializer1 = new DSpaceNotNullSerializer(getClassForObject(object1)); + const serializer2 = new DSpaceNotNullSerializer(getClassForObject(object2)); return compare(serializer1.serialize(object1), serializer2.serialize(object2)); } } diff --git a/src/app/core/data/dspace-rest-response-parsing.service.ts b/src/app/core/data/dspace-rest-response-parsing.service.ts index 604a0959ce4..7c2736d6a1a 100644 --- a/src/app/core/data/dspace-rest-response-parsing.service.ts +++ b/src/app/core/data/dspace-rest-response-parsing.service.ts @@ -1,25 +1,34 @@ /* eslint-disable max-classes-per-file */ -import { hasNoValue, hasValue, isNotEmpty } from '../../shared/empty.util'; -import { DSpaceSerializer } from '../dspace-rest/dspace.serializer'; -import { Serializer } from '../serializer'; -import { PageInfo } from '../shared/page-info.model'; -import { ObjectCacheService } from '../cache/object-cache.service'; -import { GenericConstructor } from '../shared/generic-constructor'; -import { PaginatedList, buildPaginatedList } from './paginated-list.model'; -import { getClassForType } from '../cache/builders/build-decorators'; +import { Injectable } from '@angular/core'; import { environment } from '../../../environments/environment'; +import { + hasNoValue, + hasValue, + isNotEmpty, +} from '../../shared/empty.util'; +import { getClassForObject } from '../cache/builders/build-decorators'; +import { CacheableObject } from '../cache/cacheable-object.model'; +import { ObjectCacheService } from '../cache/object-cache.service'; +import { ParsedResponse } from '../cache/response.models'; +import { DSpaceSerializer } from '../dspace-rest/dspace.serializer'; import { RawRestResponse } from '../dspace-rest/raw-rest-response.model'; +import { + getEmbedSizeParams, + getUrlWithoutEmbedParams, +} from '../index/index.selectors'; +import { Serializer } from '../serializer'; import { DSpaceObject } from '../shared/dspace-object.model'; -import { Injectable } from '@angular/core'; +import { GenericConstructor } from '../shared/generic-constructor'; +import { PageInfo } from '../shared/page-info.model'; +import { URLCombiner } from '../url-combiner/url-combiner'; +import { + buildPaginatedList, + PaginatedList, +} from './paginated-list.model'; import { ResponseParsingService } from './parsing.service'; -import { ParsedResponse } from '../cache/response.models'; import { RestRequestMethod } from './rest-request-method'; -import { getUrlWithoutEmbedParams, getEmbedSizeParams } from '../index/index.selectors'; -import { URLCombiner } from '../url-combiner/url-combiner'; -import { CacheableObject } from '../cache/cacheable-object.model'; import { RestRequest } from './rest-request.model'; - /** * Return true if obj has a value for `_links.self` * @@ -210,16 +219,11 @@ export class DspaceRestResponseParsingService implements ResponseParsingService */ protected getConstructorFor(obj: any): GenericConstructor { if (hasValue(obj?.type)) { - const constructor = getClassForType(obj.type) as GenericConstructor; + const constructor = getClassForObject(obj) as GenericConstructor; if (hasValue(constructor)) { return constructor; } - - // Browses have a subtype, so to get the correct constructor it has to be retrieved based on 'browseType' - if (obj.type === 'browse') { - return getClassForType(obj.browseType) as GenericConstructor; - } } return null; diff --git a/src/app/core/data/endpoint-map-response-parsing.service.ts b/src/app/core/data/endpoint-map-response-parsing.service.ts index cb4a6991b18..e2f2fb2de20 100644 --- a/src/app/core/data/endpoint-map-response-parsing.service.ts +++ b/src/app/core/data/endpoint-map-response-parsing.service.ts @@ -1,17 +1,17 @@ import { Injectable } from '@angular/core'; +import { environment } from '../../../environments/environment'; +import { hasValue } from '../../shared/empty.util'; +import { getClassForObject } from '../cache/builders/build-decorators'; +import { CacheableObject } from '../cache/cacheable-object.model'; +import { ParsedResponse } from '../cache/response.models'; +import { RawRestResponse } from '../dspace-rest/raw-rest-response.model'; +import { DSpaceObject } from '../shared/dspace-object.model'; +import { GenericConstructor } from '../shared/generic-constructor'; import { DspaceRestResponseParsingService, - isCacheableObject + isCacheableObject, } from './dspace-rest-response-parsing.service'; -import { hasValue } from '../../shared/empty.util'; -import { getClassForType } from '../cache/builders/build-decorators'; -import { GenericConstructor } from '../shared/generic-constructor'; -import { RawRestResponse } from '../dspace-rest/raw-rest-response.model'; -import { ParsedResponse } from '../cache/response.models'; -import { DSpaceObject } from '../shared/dspace-object.model'; -import { environment } from '../../../environments/environment'; -import { CacheableObject } from '../cache/cacheable-object.model'; import { RestRequest } from './rest-request.model'; /** @@ -42,7 +42,7 @@ export class EndpointMapResponseParsingService extends DspaceRestResponseParsing const type: string = processRequestDTO.type; let objConstructor; if (hasValue(type)) { - objConstructor = getClassForType(type); + objConstructor = getClassForObject(processRequestDTO); } if (isCacheableObject(processRequestDTO) && hasValue(objConstructor)) { @@ -73,7 +73,7 @@ export class EndpointMapResponseParsingService extends DspaceRestResponseParsing protected deserialize(obj): any { const type: string = obj.type; if (hasValue(type)) { - const objConstructor = getClassForType(type) as GenericConstructor; + const objConstructor = getClassForObject(obj) as GenericConstructor; if (hasValue(objConstructor)) { const serializer = new this.serializerConstructor(objConstructor); diff --git a/src/app/core/data/request.effects.ts b/src/app/core/data/request.effects.ts index 889d909bfa3..745402f2ff6 100644 --- a/src/app/core/data/request.effects.ts +++ b/src/app/core/data/request.effects.ts @@ -1,26 +1,42 @@ -import { Injectable, Injector } from '@angular/core'; +import { + Injectable, + Injector, +} from '@angular/core'; -import { Actions, createEffect, ofType } from '@ngrx/effects'; -import { catchError, filter, map, mergeMap, take } from 'rxjs/operators'; +import { + Actions, + createEffect, + ofType, +} from '@ngrx/effects'; +import { + catchError, + filter, + map, + mergeMap, + take, +} from 'rxjs/operators'; -import { hasValue, isNotEmpty } from '../../shared/empty.util'; +import { + hasValue, + isNotEmpty, +} from '../../shared/empty.util'; import { StoreActionTypes } from '../../store.actions'; -import { getClassForType } from '../cache/builders/build-decorators'; -import { RawRestResponse } from '../dspace-rest/raw-rest-response.model'; +import { getClassForObject } from '../cache/builders/build-decorators'; +import { ParsedResponse } from '../cache/response.models'; import { DspaceRestService } from '../dspace-rest/dspace-rest.service'; import { DSpaceSerializer } from '../dspace-rest/dspace.serializer'; +import { RawRestResponse } from '../dspace-rest/raw-rest-response.model'; +import { RequestEntry } from './request-entry.model'; +import { RequestError } from './request-error.model'; import { RequestActionTypes, RequestErrorAction, RequestExecuteAction, RequestSuccessAction, - ResetResponseTimestampsAction + ResetResponseTimestampsAction, } from './request.actions'; import { RequestService } from './request.service'; -import { ParsedResponse } from '../cache/response.models'; -import { RequestError } from './request-error.model'; import { RestRequestWithResponseParser } from './rest-request-with-response-parser.model'; -import { RequestEntry } from './request-entry.model'; @Injectable() export class RequestEffects { @@ -37,7 +53,7 @@ export class RequestEffects { mergeMap((request: RestRequestWithResponseParser) => { let body = request.body; if (isNotEmpty(request.body) && !request.isMultipart) { - const serializer = new DSpaceSerializer(getClassForType(request.body.type)); + const serializer = new DSpaceSerializer(getClassForObject(request.body)); body = serializer.serialize(request.body); } return this.restApi.request(request.method, request.href, body, request.options, request.isMultipart).pipe( diff --git a/src/app/core/shared/browse-definition.model.ts b/src/app/core/shared/browse-definition.model.ts index a5bed53c9fd..d052f34fafc 100644 --- a/src/app/core/shared/browse-definition.model.ts +++ b/src/app/core/shared/browse-definition.model.ts @@ -1,10 +1,17 @@ import { autoserialize } from 'cerialize'; import { CacheableObject } from '../cache/cacheable-object.model'; +import { excludeFromEquals } from '../utilities/equals.decorators'; +import { BROWSE_DEFINITION } from './browse-definition.resource-type'; /** * Base class for BrowseDefinition models */ export abstract class BrowseDefinition extends CacheableObject { + static type = BROWSE_DEFINITION; + + @excludeFromEquals + type = BROWSE_DEFINITION; + @autoserialize id: string; diff --git a/src/app/core/shared/flat-browse-definition.model.ts b/src/app/core/shared/flat-browse-definition.model.ts index 086fca891bb..09afaeab32f 100644 --- a/src/app/core/shared/flat-browse-definition.model.ts +++ b/src/app/core/shared/flat-browse-definition.model.ts @@ -1,5 +1,5 @@ import { inheritSerialization, deserialize } from 'cerialize'; -import { typedObject } from '../cache/builders/build-decorators'; +import { typedObjectWithSubType } from '../cache/builders/build-decorators'; import { excludeFromEquals } from '../utilities/equals.decorators'; import { FLAT_BROWSE_DEFINITION } from './flat-browse-definition.resource-type'; import { ResourceType } from './resource-type'; @@ -9,16 +9,16 @@ import { HALLink } from './hal-link.model'; /** * BrowseDefinition model for browses of type 'flatBrowse' */ -@typedObject +@typedObjectWithSubType('browseType') @inheritSerialization(NonHierarchicalBrowseDefinition) export class FlatBrowseDefinition extends NonHierarchicalBrowseDefinition { - static type = FLAT_BROWSE_DEFINITION; + static browseType = FLAT_BROWSE_DEFINITION; /** * The object type */ @excludeFromEquals - type: ResourceType = FLAT_BROWSE_DEFINITION; + browseType: ResourceType = FLAT_BROWSE_DEFINITION; get self(): string { return this._links.self.href; diff --git a/src/app/core/shared/hierarchical-browse-definition.model.ts b/src/app/core/shared/hierarchical-browse-definition.model.ts index d561fff643f..c7dff8333f9 100644 --- a/src/app/core/shared/hierarchical-browse-definition.model.ts +++ b/src/app/core/shared/hierarchical-browse-definition.model.ts @@ -1,5 +1,8 @@ import { autoserialize, autoserializeAs, deserialize, inheritSerialization } from 'cerialize'; -import { typedObject } from '../cache/builders/build-decorators'; +import { + typedObject, + typedObjectWithSubType, +} from '../cache/builders/build-decorators'; import { excludeFromEquals } from '../utilities/equals.decorators'; import { HIERARCHICAL_BROWSE_DEFINITION } from './hierarchical-browse-definition.resource-type'; import { HALLink } from './hal-link.model'; @@ -9,16 +12,16 @@ import { BrowseDefinition } from './browse-definition.model'; /** * BrowseDefinition model for browses of type 'hierarchicalBrowse' */ -@typedObject +@typedObjectWithSubType('browseType') @inheritSerialization(BrowseDefinition) export class HierarchicalBrowseDefinition extends BrowseDefinition { - static type = HIERARCHICAL_BROWSE_DEFINITION; + static browseType = HIERARCHICAL_BROWSE_DEFINITION; /** * The object type */ @excludeFromEquals - type: ResourceType = HIERARCHICAL_BROWSE_DEFINITION; + browseType: ResourceType = HIERARCHICAL_BROWSE_DEFINITION; @autoserialize facetType: string; diff --git a/src/app/core/shared/value-list-browse-definition.model.ts b/src/app/core/shared/value-list-browse-definition.model.ts index 3378263962e..6726c8044f6 100644 --- a/src/app/core/shared/value-list-browse-definition.model.ts +++ b/src/app/core/shared/value-list-browse-definition.model.ts @@ -1,5 +1,5 @@ import { inheritSerialization, deserialize } from 'cerialize'; -import { typedObject } from '../cache/builders/build-decorators'; +import { typedObjectWithSubType } from '../cache/builders/build-decorators'; import { excludeFromEquals } from '../utilities/equals.decorators'; import { VALUE_LIST_BROWSE_DEFINITION } from './value-list-browse-definition.resource-type'; import { ResourceType } from './resource-type'; @@ -9,16 +9,16 @@ import { HALLink } from './hal-link.model'; /** * BrowseDefinition model for browses of type 'valueList' */ -@typedObject +@typedObjectWithSubType('browseType') @inheritSerialization(NonHierarchicalBrowseDefinition) export class ValueListBrowseDefinition extends NonHierarchicalBrowseDefinition { - static type = VALUE_LIST_BROWSE_DEFINITION; + static browseType = VALUE_LIST_BROWSE_DEFINITION; /** * The object type */ @excludeFromEquals - type: ResourceType = VALUE_LIST_BROWSE_DEFINITION; + browseType: ResourceType = VALUE_LIST_BROWSE_DEFINITION; get self(): string { return this._links.self.href; From 168a9671f9c8653ae3c2c3d0724914a8863b932a Mon Sep 17 00:00:00 2001 From: Yury Bondarenko Date: Mon, 24 Feb 2025 13:44:45 +0100 Subject: [PATCH 13/21] Compress hashed config.json --- src/config/config.server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.server.ts b/src/config/config.server.ts index 952aac5a15b..40796a0539d 100644 --- a/src/config/config.server.ts +++ b/src/config/config.server.ts @@ -243,7 +243,7 @@ export const buildAppConfig = (destConfigPath?: string, mapping?: ServerHashedFi writeFileSync(destConfigPath, content); if (mapping !== undefined) { - mapping.add(destConfigPath, content); + mapping.add(destConfigPath, content, true); } console.log(`Angular ${bold('config.json')} file generated correctly at ${bold(destConfigPath)} \n`); From 79d4da1c02edf409e3db0dd8073395893a3c58d9 Mon Sep 17 00:00:00 2001 From: Yury Bondarenko Date: Mon, 24 Feb 2025 17:04:50 +0100 Subject: [PATCH 14/21] Ensure bootstrapped requests are included in hashed config --- server.ts | 2 +- src/config/config.server.ts | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/server.ts b/server.ts index 192fcb812f4..37452c135d0 100644 --- a/server.ts +++ b/server.ts @@ -617,7 +617,7 @@ function healthCheck(req, res) { declare const __non_webpack_require__: NodeRequire; const mainModule = __non_webpack_require__.main; const moduleFilename = (mainModule && mainModule.filename) || ''; -setupEndpointPrefetching(appConfig, destConfigPath, environment).then(() => { +setupEndpointPrefetching(appConfig, destConfigPath, environment, hashedFileMapping).then(() => { if (moduleFilename === __filename || moduleFilename.includes('iisnode')) { start(); } diff --git a/src/config/config.server.ts b/src/config/config.server.ts index 40796a0539d..0d306dd0880 100644 --- a/src/config/config.server.ts +++ b/src/config/config.server.ts @@ -252,13 +252,13 @@ export const buildAppConfig = (destConfigPath?: string, mapping?: ServerHashedFi return appConfig; }; -export const setupEndpointPrefetching = async (appConfig: AppConfig, destConfigPath: string, env: any): Promise => { - await prefetchResponses(appConfig, destConfigPath, env); +export const setupEndpointPrefetching = async (appConfig: AppConfig, destConfigPath: string, env: any, hfm: ServerHashedFileMapping): Promise => { + await prefetchResponses(appConfig, destConfigPath, env, hfm); - setInterval(() => void prefetchResponses(appConfig, destConfigPath, env), appConfig.prefetch.refreshInterval); + setInterval(() => void prefetchResponses(appConfig, destConfigPath, env, hfm), appConfig.prefetch.refreshInterval); }; -export const prefetchResponses = async (appConfig: AppConfig, destConfigPath: string, env: any): Promise => { +export const prefetchResponses = async (appConfig: AppConfig, destConfigPath: string, env: any, hfm: ServerHashedFileMapping): Promise => { console.info('Prefetching endpoint maps'); const restConfig = appConfig.rest; const prefetchConfig = appConfig.prefetch; @@ -288,6 +288,8 @@ export const prefetchResponses = async (appConfig: AppConfig, destConfigPath: st prefetchConfig.bootstrap = mapping; + const content = JSON.stringify(appConfig, null, 2); extendEnvironmentWithAppConfig(env, appConfig, false); - writeFileSync(destConfigPath, JSON.stringify(appConfig, null, 2)); + hfm.add(destConfigPath, content, true); + hfm.save(); }; From 13e039ee61ac58718e17d09d87029b137c26c05f Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Wed, 26 Feb 2025 16:20:38 +0100 Subject: [PATCH 15/21] Fix browse response parsing test --- .../core/data/browse-response-parsing.service.spec.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/app/core/data/browse-response-parsing.service.spec.ts b/src/app/core/data/browse-response-parsing.service.spec.ts index 9fa7239ef7c..a4b3919c851 100644 --- a/src/app/core/data/browse-response-parsing.service.spec.ts +++ b/src/app/core/data/browse-response-parsing.service.spec.ts @@ -4,6 +4,7 @@ import { ObjectCacheService } from '../cache/object-cache.service'; import { HIERARCHICAL_BROWSE_DEFINITION } from '../shared/hierarchical-browse-definition.resource-type'; import { FLAT_BROWSE_DEFINITION } from '../shared/flat-browse-definition.resource-type'; import { VALUE_LIST_BROWSE_DEFINITION } from '../shared/value-list-browse-definition.resource-type'; +import { BROWSE_DEFINITION } from '../shared/browse-definition.resource-type'; class TestService extends BrowseResponseParsingService { constructor(protected objectCache: ObjectCacheService) { @@ -45,19 +46,22 @@ describe('BrowseResponseParsingService', () => { it('should deserialize flatBrowses correctly', () => { let deserialized = service.deserialize(mockFlatBrowse); - expect(deserialized.type).toBe(FLAT_BROWSE_DEFINITION); + expect(deserialized.type).toBe(BROWSE_DEFINITION); + expect(deserialized.browseType).toBe(FLAT_BROWSE_DEFINITION); expect(deserialized.id).toBe(mockFlatBrowse.id); }); it('should deserialize valueList browses correctly', () => { let deserialized = service.deserialize(mockValueList); - expect(deserialized.type).toBe(VALUE_LIST_BROWSE_DEFINITION); + expect(deserialized.type).toBe(BROWSE_DEFINITION); + expect(deserialized.browseType).toBe(VALUE_LIST_BROWSE_DEFINITION); expect(deserialized.id).toBe(mockValueList.id); }); it('should deserialize hierarchicalBrowses correctly', () => { let deserialized = service.deserialize(mockHierarchicalBrowse); - expect(deserialized.type).toBe(HIERARCHICAL_BROWSE_DEFINITION); + expect(deserialized.type).toBe(BROWSE_DEFINITION); + expect(deserialized.browseType).toBe(HIERARCHICAL_BROWSE_DEFINITION); expect(deserialized.id).toBe(mockHierarchicalBrowse.id); }); }); From 5ba59b66900dcab01eadfa7131ea1340bf59fb2c Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Wed, 12 Mar 2025 13:32:38 +0100 Subject: [PATCH 16/21] 124369: Update prefetch config comments --- config/config.example.yml | 3 +++ src/config/prefetch-config.ts | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/config.example.yml b/config/config.example.yml index d40d18ea8ee..3bfac817f45 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -381,8 +381,11 @@ comcolSelectionSort: sortField: 'dc.title' sortDirection: 'ASC' +# REST response prefetching configuration prefetch: + # The URLs for which the response will be prefetched urls: - /api - /api/discover + # How often the responses are refreshed in milliseconds refreshInterval: 60000 diff --git a/src/config/prefetch-config.ts b/src/config/prefetch-config.ts index e456f538e49..85015cd79ee 100644 --- a/src/config/prefetch-config.ts +++ b/src/config/prefetch-config.ts @@ -12,7 +12,7 @@ export interface PrefetchConfig extends Config { */ bootstrap: Record; /** - * How often the HAL endpoints should be refreshed + * How often the responses should be refreshed in milliseconds */ refreshInterval: number; } From d52051120de39ff15fddc1e97e5dc5b30f20efff Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Wed, 12 Mar 2025 15:01:14 +0100 Subject: [PATCH 17/21] 124369: Fix prefetching info message --- src/config/config.server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.server.ts b/src/config/config.server.ts index 0d306dd0880..8a533e77904 100644 --- a/src/config/config.server.ts +++ b/src/config/config.server.ts @@ -259,7 +259,7 @@ export const setupEndpointPrefetching = async (appConfig: AppConfig, destConfigP }; export const prefetchResponses = async (appConfig: AppConfig, destConfigPath: string, env: any, hfm: ServerHashedFileMapping): Promise => { - console.info('Prefetching endpoint maps'); + console.info('Prefetching REST responses'); const restConfig = appConfig.rest; const prefetchConfig = appConfig.prefetch; From 4dfc9eaf29b8058194897ba4f5b9585fd706449b Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Thu, 13 Mar 2025 16:03:04 +0100 Subject: [PATCH 18/21] 124369: Apply 8_x specific fixes after merge --- src/app/app.config.ts | 6 ++++++ src/app/core/provide-core.ts | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/app/app.config.ts b/src/app/app.config.ts index 77b29206cb5..5cf201bb761 100644 --- a/src/app/app.config.ts +++ b/src/app/app.config.ts @@ -36,6 +36,8 @@ import { } from '../config/app-config.interface'; import { StoreDevModules } from '../config/store/devtools'; import { environment } from '../environments/environment'; +import { HashedFileMapping } from '../modules/dynamic-hash/hashed-file-mapping'; +import { BrowserHashedFileMapping } from '../modules/dynamic-hash/hashed-file-mapping.browser'; import { EagerThemesModule } from '../themes/eager-themes.module'; import { appEffects } from './app.effects'; import { @@ -154,6 +156,10 @@ export const commonAppConfig: ApplicationConfig = { useClass: DspaceRestInterceptor, multi: true, }, + { + provide: HashedFileMapping, + useClass: BrowserHashedFileMapping, + }, // register the dynamic matcher used by form. MUST be provided by the app module ...DYNAMIC_MATCHER_PROVIDERS, provideCore(), diff --git a/src/app/core/provide-core.ts b/src/app/core/provide-core.ts index 7b8d4e0574c..09b7d2ebb77 100644 --- a/src/app/core/provide-core.ts +++ b/src/app/core/provide-core.ts @@ -19,6 +19,7 @@ import { import { AccessStatusObject } from '../shared/object-collection/shared/badges/access-status-badge/access-status.model'; import { IdentifierData } from '../shared/object-list/identifier-data/identifier-data.model'; import { Subscription } from '../shared/subscriptions/models/subscription.model'; +import { StatisticsEndpoint } from '../statistics/statistics-endpoint.model'; import { SubmissionCoarNotifyConfig } from '../submission/sections/section-coar-notify/submission-coar-notify.config'; import { SystemWideAlert } from '../system-wide-alert/system-wide-alert.model'; import { AuthStatus } from './auth/models/auth-status.model'; @@ -192,4 +193,5 @@ export const models = SubmissionCoarNotifyConfig, NotifyRequestsStatus, SystemWideAlert, + StatisticsEndpoint, ]; From c015ed47ff95458637d3639269be0b88953ad430 Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Thu, 13 Mar 2025 11:56:31 +0100 Subject: [PATCH 19/21] 124369: Stop prefetching responses when process is terminated --- server.ts | 32 ++++++++++++++++++-------------- src/config/config.server.ts | 4 ++-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/server.ts b/server.ts index e500bbb2f58..3ae9a3d54b2 100644 --- a/server.ts +++ b/server.ts @@ -542,7 +542,7 @@ function serverStarted() { * Create an HTTPS server with the configured port and host * @param keys SSL credentials */ -function createHttpsServer(keys) { +function createHttpsServer(prefetchRefreshTimeout: NodeJS.Timeout, keys) { const listener = createServer({ key: keys.serviceKey, cert: keys.certificate, @@ -555,7 +555,7 @@ function createHttpsServer(keys) { process.on('SIGINT', () => { void (async ()=> { console.debug('Closing HTTPS server on signal'); - await terminator.terminate().catch(e => { console.error(e); }); + clearTimeout(prefetchRefreshTimeout);await terminator.terminate().catch(e => { console.error(e); }); console.debug('HTTPS server closed'); })(); }); @@ -564,7 +564,7 @@ function createHttpsServer(keys) { /** * Create an HTTP server with the configured port and host. */ -function run() { +function run(prefetchRefreshTimeout: NodeJS.Timeout) { const port = environment.ui.port || 4000; const host = environment.ui.host || '/'; @@ -579,13 +579,16 @@ function run() { process.on('SIGINT', () => { void (async () => { console.debug('Closing HTTP server on signal'); - await terminator.terminate().catch(e => { console.error(e); }); - console.debug('HTTP server closed.');return undefined; + clearTimeout(prefetchRefreshTimeout); + await terminator.terminate().catch(e => { + console.error(e); + }); + console.debug('HTTP server closed.'); })(); }); } -function start() { +function start(prefetchRefreshTimeout: NodeJS.Timeout) { logStartupMessage(environment); /* @@ -611,10 +614,11 @@ function start() { } if (serviceKey && certificate) { - createHttpsServer({ - serviceKey: serviceKey, - certificate: certificate, - }); + createHttpsServer(prefetchRefreshTimeout, + { + serviceKey: serviceKey, + certificate: certificate, + }); } else { console.warn('Disabling certificate validation and proceeding with a self-signed certificate. If this is a production server, it is recommended that you configure a valid certificate instead.'); @@ -624,11 +628,11 @@ function start() { days: 1, selfSigned: true, }, (error, keys) => { - createHttpsServer(keys); + createHttpsServer(prefetchRefreshTimeout, keys); }); } } else { - run(); + run(prefetchRefreshTimeout); } } @@ -653,9 +657,9 @@ function healthCheck(req, res) { declare const __non_webpack_require__: NodeRequire; const mainModule = __non_webpack_require__.main; const moduleFilename = (mainModule && mainModule.filename) || ''; -setupEndpointPrefetching(appConfig, destConfigPath, environment, hashedFileMapping).then(() => { +setupEndpointPrefetching(appConfig, destConfigPath, environment, hashedFileMapping).then(prefetchRefreshTimeout => { if (moduleFilename === __filename || moduleFilename.includes('iisnode')) { - start(); + start(prefetchRefreshTimeout); } }).catch((error) => { console.error('Errored while prefetching Endpoint Maps', error); diff --git a/src/config/config.server.ts b/src/config/config.server.ts index 6891eae4184..302e15f26e9 100644 --- a/src/config/config.server.ts +++ b/src/config/config.server.ts @@ -265,10 +265,10 @@ export const buildAppConfig = (destConfigPath?: string, mapping?: ServerHashedFi return appConfig; }; -export const setupEndpointPrefetching = async (appConfig: AppConfig, destConfigPath: string, env: any, hfm: ServerHashedFileMapping): Promise => { +export const setupEndpointPrefetching = async (appConfig: AppConfig, destConfigPath: string, env: any, hfm: ServerHashedFileMapping): Promise => { await prefetchResponses(appConfig, destConfigPath, env, hfm); - setInterval(() => void prefetchResponses(appConfig, destConfigPath, env, hfm), appConfig.prefetch.refreshInterval); + return setInterval(() => void prefetchResponses(appConfig, destConfigPath, env, hfm), appConfig.prefetch.refreshInterval); }; export const prefetchResponses = async (appConfig: AppConfig, destConfigPath: string, env: any, hfm: ServerHashedFileMapping): Promise => { From b353bf99ccebc16d1684300c93ace6d238620569 Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Fri, 14 Mar 2025 08:49:48 +0100 Subject: [PATCH 20/21] 124369: Catch errors during fetching of responses --- src/config/config.server.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/config/config.server.ts b/src/config/config.server.ts index 8a533e77904..71a0ffd12ec 100644 --- a/src/config/config.server.ts +++ b/src/config/config.server.ts @@ -269,7 +269,13 @@ export const prefetchResponses = async (appConfig: AppConfig, destConfigPath: st for (const relativeUrl of prefetchConfig.urls) { const url = baseUrl + relativeUrl; - const response = await fetch(url); + let response: Response; + try { + response = await fetch(url); + } catch (e) { + console.warn(`Failed to prefetch REST response for url "${url}". Aborting prefetching. Is the REST server offline?`); + return; + } const headers: Record = {}; response.headers.forEach((value, header) => { From 95b58d2f70d97115969e0722f0386bdf6d56215e Mon Sep 17 00:00:00 2001 From: Andreas Awouters Date: Tue, 18 Mar 2025 12:00:26 +0100 Subject: [PATCH 21/21] 124369: Wait until responses have been used before clearing prefetched responses --- src/app/init.service.ts | 17 +++++++++++++---- src/modules/app/browser-init.service.ts | 19 ++++++------------- src/modules/app/server-init.service.ts | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/app/init.service.ts b/src/app/init.service.ts index 253dc0190aa..6ed33e1cffd 100644 --- a/src/app/init.service.ts +++ b/src/app/init.service.ts @@ -21,8 +21,8 @@ import { MetadataService } from './core/metadata/metadata.service'; import { BreadcrumbsService } from './breadcrumbs/breadcrumbs.service'; import { ThemeService } from './shared/theme-support/theme.service'; import { isAuthenticationBlocking } from './core/auth/selectors'; -import { distinctUntilChanged, find, take } from 'rxjs/operators'; -import { Observable } from 'rxjs'; +import { distinctUntilChanged, find, map, take } from 'rxjs/operators'; +import { combineLatest, Observable, of } from 'rxjs'; import { MenuService } from './shared/menu/menu.service'; import { hasValue } from './shared/empty.util'; import { HrefOnlyDataService } from './core/data/href-only-data.service'; @@ -207,11 +207,20 @@ export abstract class InitService { /** * Use the bootstrapped requests to prefill the cache */ - protected initBootstrapEndpoints() { + protected initBootstrapEndpoints$(): Observable { if (hasValue(this.appConfig?.prefetch?.bootstrap)) { + const observables = {}; + for (let url of Object.getOwnPropertyNames(this.appConfig.prefetch.bootstrap)) { - this.hrefOnlyDataService.findByHref(url, false).pipe(take(1)).subscribe(); + observables[url] = this.hrefOnlyDataService.findByHref(url, false); } + + return combineLatest(observables).pipe( + take(1), + map(_ => undefined), + ); + } else { + return of(undefined); } } } diff --git a/src/modules/app/browser-init.service.ts b/src/modules/app/browser-init.service.ts index 60295a99399..d011625e03b 100644 --- a/src/modules/app/browser-init.service.ts +++ b/src/modules/app/browser-init.service.ts @@ -104,7 +104,12 @@ export class BrowserInitService extends InitService { this.initKlaro(); - this.initBootstrapEndpoints(); + this.initBootstrapEndpoints$().subscribe(_ => { + if (hasValue(this.appConfig?.prefetch?.bootstrap)) { + // Clear bootstrap once finished so the dspace-rest.service does not keep using the bootstrapped responses + this.appConfig.prefetch.bootstrap = undefined; + } + }); await this.authenticationReady$().toPromise(); @@ -177,16 +182,4 @@ export class BrowserInitService extends InitService { }); } - /** - * Use the bootstrapped requests from the server to prefill the cache on the client - */ - override initBootstrapEndpoints() { - super.initBootstrapEndpoints(); - - if (hasValue(this.appConfig?.prefetch?.bootstrap)) { - // Clear bootstrap once finished so the dspace-rest.service does not keep using the bootstrap - this.appConfig.prefetch.bootstrap = undefined; - } - } - } diff --git a/src/modules/app/server-init.service.ts b/src/modules/app/server-init.service.ts index d87275b5af2..3f8b3f501c3 100644 --- a/src/modules/app/server-init.service.ts +++ b/src/modules/app/server-init.service.ts @@ -70,7 +70,7 @@ export class ServerInitService extends InitService { this.initRouteListeners(); this.themeService.listenForThemeChanges(false); - this.initBootstrapEndpoints(); + this.initBootstrapEndpoints$().subscribe(); await this.authenticationReady$().toPromise();