From fe8c85bd86ed16c883f5502eedeb217edb799603 Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 13:54:42 +0100 Subject: [PATCH 01/19] draft separate data model --- .../visualization/barchart/results-count.ts | 527 ++++++++++++++++++ 1 file changed, 527 insertions(+) create mode 100644 frontend/src/app/visualization/barchart/results-count.ts diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts new file mode 100644 index 000000000..bb269d685 --- /dev/null +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -0,0 +1,527 @@ +import { BarchartSeries, CorpusField, DateFilterData, HistogramDataPoint, HistogramSeries, MultipleChoiceFilterOptions, QueryModel, RangeFilterOptions, TaskResult, TermFrequencyResult, TimeCategory, TimelineDataPoint, TimelineSeries } from '@models'; +import { DateHistogramAggregator, DateHistogramResult, TermsAggregator, TermsResult } from '@models/aggregation'; + +import { ComparedQueries } from '@models/compared-queries'; +import { ApiService, SearchService, VisualizationService } from '@services'; +import _ from 'lodash'; +import { hasPrefixTerm } from './query-utils'; +import { BehaviorSubject, Observable, Subject, takeUntil } from 'rxjs'; +import { showLoading } from '@utils/utils'; +import { differenceInDays } from 'date-fns'; + +export type FrequencyMeasure = 'documents' | 'tokens'; + +const documentLimitBase = 10000; +const documentLimitPrefixQueries = 1000; + + +export abstract class BarchartData< + AggregateResult extends TermsResult | DateHistogramResult, + DataPoint extends TimelineDataPoint | HistogramDataPoint +> { + // rawData: a list of series + rawData: BarchartSeries[]; + documentLimitExceeded = false; // whether the results include documents than the limit + totalTokenCountAvailable: boolean; // whether the data includes token count totals + dataHasLoaded: boolean; + + tasksToCancel: string[]; + + loading$ = new BehaviorSubject(false); + error$ = new Subject(); + stopPolling$ = new Subject(); + + + constructor( + protected queryModel: QueryModel, + private comparedQueries: ComparedQueries, + public frequencyMeasure: FrequencyMeasure, + public visualizedField: CorpusField, + protected searchService: SearchService, + protected apiService: ApiService, + protected visualizationService: VisualizationService, + private destroy$: Observable, + ) { + this.refresh(); + this.queryModel.update.pipe( + takeUntil(this.destroy$) + ).subscribe(this.loadData.bind(this)); + this.comparedQueries.allQueries$.subscribe(this.updateQueries.bind(this)); + this.destroy$.subscribe(() => this.onDestroy()); + } + + get documentLimit(): number { + return hasPrefixTerm(this.queryModel.queryText) ? documentLimitPrefixQueries : documentLimitBase; + } + + protected refresh() { + this.initQueries(); + this.loadData(); + } + + /** Request and fill in doc counts for a series */ + protected getSeriesDocumentData( + series: BarchartSeries, + queryModel: QueryModel = this.queryModel, + setSearchRatio = true + ): Promise> { + const queryModelCopy = this.queryModelForSeries(series, queryModel); + + return this.requestSeriesDocCounts(queryModelCopy).then((result) => + this.docCountResultIntoSeries(result, series, setSearchRatio) + ); + } + + protected getTermFrequencies( + series: BarchartSeries, + queryModel: QueryModel + ): Promise> { + this.dataHasLoaded = false; + const queryModelCopy = this.queryModelForSeries(series, queryModel); + return new Promise((resolve, reject) => { + this.requestSeriesTermFrequency(series, queryModelCopy).then(response => { + this.tasksToCancel = response.task_ids; + const poller$ = this.apiService.pollTasks(this.tasksToCancel, this.stopPolling$); + poller$.subscribe({ + error: (error) => { + this.onFailure(error); + reject(error); + }, + next: (result) => { + resolve(this.processSeriesTermFrequency(result['results'], series)); + }, + complete: () => { + // abort tasks if the Observable is completed via takeUntil + if (!this.dataHasLoaded) { + this.apiService.abortTasks({ task_ids: this.tasksToCancel }); + } + } + }); + }); + }); + } + + /** + * calculate the maximum number of documents to read through in a bin + * when determining term frequency. + */ + protected documentLimitForCategory( + cat: DataPoint, + series: BarchartSeries + ): number { + return _.min([ + this.documentLimit, + _.ceil(cat.doc_count * series.searchRatio), + ]); + } + + private initQueries(): void { + this.rawData = [ + this.newSeries(this.queryModel.queryText), + ...this.comparedQueries.state$.value.compare.map(this.newSeries) + ]; + } + + /** update the queries in the graph to the input array. Preserve results if possible, + * and start loading for the rest. + */ + private updateQueries(queries: string[]) { + if (this.rawData) { + this.rawData = queries.map((queryText) => { + const existingSeries = this.rawData.find( + (series) => series.queryText === queryText + ); + return existingSeries || this.newSeries(queryText); + }); + this.loadData(); + } + } + + /** make a blank series object */ + private newSeries(queryText: string): BarchartSeries { + return { + queryText, + data: [], + total_doc_count: 0, + searchRatio: 1.0, + }; + } + + private loadData(): Promise { + const dataPromises = this.requestDocumentData().then( + this.frequencyMeasure === 'tokens' + ? this.requestTermFrequencyData.bind(this) + : _.identity + ) + .then((rawData) => { + this.rawData = rawData; + + if (!this.rawData.length) { + this.error$.next('No results'); + }; + }); + return showLoading(this.loading$, dataPromises); + } + + private requestDocumentData(): Promise[]> { + const dataPromises = this.rawData.map((series) => { + if (!series.data.length) { + // retrieve data if it was not already loaded + return this.getSeriesDocumentData(series); + } else { + return series; + } + }); + return Promise.all(dataPromises).then( + this.checkDocumentLimitExceeded.bind(this) + ); + } + + /** Retrieve all term frequencies and store in `rawData`. + * Term frequencies are only loaded if they were not already there. + */ + private requestTermFrequencyData(rawData: typeof this.rawData): Promise[]> { + // cancel and stop polling running tasks + this.stopPolling$.next(); + const dataPromises = rawData.map(series => { + if (series.queryText && series.data.length && series.data[0].match_count === undefined) { + // retrieve data if it was not already loaded + return this.getTermFrequencies(series, this.queryModel); + } else { + return series; + } + }); + + return Promise.all(dataPromises); + } + + + + /** adapt query model to fit series: use correct search fields and query text */ + private queryModelForSeries( + series: BarchartSeries, + queryModel: QueryModel + ) { + return this.selectSearchFields( + this.setQueryText(queryModel, series.queryText) + ); + } + + + /** + * Check whether any series found more documents than the document limit. + * This means that not all documents will be read when counting term frequency. + */ + private checkDocumentLimitExceeded( + rawData: typeof this.rawData + ): typeof this.rawData { + this.documentLimitExceeded = + rawData.find((series) => series.searchRatio < 1) !== undefined; + return rawData; + } + + private selectSearchFields(queryModel: QueryModel) { + if (this.frequencyMeasure === 'documents') { + return queryModel; + } else { + const mainContentFields = this.queryModel.corpus.fields.filter( + (field) => + field.searchable && field.displayType === 'text_content' + ); + const queryModelCopy = queryModel.clone(); + queryModelCopy.setParams({searchFields: mainContentFields}); + return queryModelCopy; + } + } + + + /** return a copy of a query model with the query text set to the given value */ + private setQueryText(query: QueryModel, queryText: string): QueryModel { + const queryModelCopy = query.clone(); + queryModelCopy.setQueryText(queryText); + return queryModelCopy; + } + + /** + * Convert the result from an document search into a data series object. + * - Converts to the relevant DataPoint type using `aggregateResultToDataPoint` + * - Adds the total document count and search ratio for retrieving term frequencies. + * - Adds the relative document count. + * + * @param result result from aggregation search + * @param series series object that this data belongs to + * @param setSearchRatio whether the search ratio should be reset. Defaults to `true`, + * may be set to `false` when loading a portion of the series during zoom. + * @returns a copy of the series with the document counts included. + */ + private docCountResultIntoSeries( + result: AggregateResult[], + series: BarchartSeries, + setSearchRatio = true + ): BarchartSeries { + let data = result.map(this.aggregateResultToDataPoint); + const totalDocCount = this.totalDocCount(result); + const searchRatio = setSearchRatio + ? this.documentLimit / totalDocCount + : series.searchRatio; + data = this.includeRelativeDocCount(data, totalDocCount); + return { + data, + total_doc_count: totalDocCount, + searchRatio, + queryText: series.queryText, + }; + } + + /** total document count for a data array */ + private totalDocCount(data: AggregateResult[]) { + return _.sumBy(data, (item) => item.doc_count); + } + + /** fill in the `relative_doc_count` property for an array of datapoints. + */ + private includeRelativeDocCount(data: DataPoint[], total: number): DataPoint[] { + return data.map((item) => { + const result = _.clone(item); + result.relative_doc_count = result.doc_count / total; + return result; + }); + } + + private onFailure(error) { + this.error$.next(`could not load results: ${error.message}`); + } + + private processSeriesTermFrequency(results: TermFrequencyResult[], series: BarchartSeries): BarchartSeries { + this.dataHasLoaded = true; + series.data = _.zip(series.data, results).map(pair => { + const [bin, res] = pair; + return this.addTermFrequencyToCategory(res, bin); + }); + return series; + }; + + /** + * add term frequency data to a DataPoint object + * + * @param result output from request for term frequencies + * @param cat DataPoint object where the data should be added + */ + private addTermFrequencyToCategory( + data: TermFrequencyResult, + cat: DataPoint + ): DataPoint { + cat.match_count = data.match_count; + cat.total_doc_count = data.total_doc_count; + cat.token_count = data.token_count; + cat.matches_by_doc_count = data.match_count / data.total_doc_count; + cat.matches_by_token_count = data.token_count + ? data.match_count / data.token_count + : undefined; + return cat; + } + + private onDestroy() { + this.stopPolling$.next(); + this.stopPolling$.complete(); + this.error$.complete(); + } + + /** Request doc counts for a series */ + protected abstract requestSeriesDocCounts( + queryModel: QueryModel + ): Promise; + + + /** convert the output of an aggregation search to the relevant result type */ + protected abstract aggregateResultToDataPoint(cat: AggregateResult): DataPoint; + + protected abstract requestSeriesTermFrequency(series: BarchartSeries, queryModel: QueryModel): Promise; + +} + +export class HistogramData extends BarchartData { + /** specify aggregator object based on visualised field; + * used in document requests. + */ + protected getAggregator(): TermsAggregator { + let size = 100; + + const filterOptions = this.visualizedField.filterOptions; + if (filterOptions.name === 'MultipleChoiceFilter') { + size = (filterOptions as MultipleChoiceFilterOptions).option_count; + } else if (filterOptions.name === 'RangeFilter') { + const filterRange = + (filterOptions as RangeFilterOptions).upper - + (filterOptions as RangeFilterOptions).lower; + size = _.max([size, filterRange]) + } + return new TermsAggregator(this.visualizedField, size); + } + + protected requestSeriesDocCounts(queryModel: QueryModel) { + const aggregator = this.getAggregator(); + return this.searchService.aggregateSearch(this.queryModel.corpus, queryModel, aggregator); + } + + protected aggregateResultToDataPoint(cat: TermsResult) { + return cat; + } + + protected requestSeriesTermFrequency( + series: HistogramSeries, + queryModel: QueryModel + ) { + const bins = this.makeTermFrequencyBins(series); + return this.visualizationService.aggregateTermFrequencySearch( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins + ); + } + + private makeTermFrequencyBins(series: HistogramSeries) { + return series.data.map((bin) => ({ + fieldValue: bin.key, + size: this.documentLimitForCategory(bin, series), + })); + } + +} + +export class TimelineData extends BarchartData { + currentTimeCategory: TimeCategory = 'year'; + + /** threshold for scaling down a unit on the time scale */ + private scaleDownThreshold = 10; + + zoomedInData (min: Date, max: Date) { + // const series = this.rawData[seriesIndex]; + this.rawData.map(series => { + const queryModelCopy = this.addQueryDateFilter( + this.queryModel, + min, + max + ); + return this.getSeriesDocumentData( + series, + queryModelCopy, + false + ).then((result) => { + if (this.frequencyMeasure === 'tokens') { + return this.getTermFrequencies(result, queryModelCopy); + } else { + return result; + } + }); + }); + } + + protected refresh(): void { + this.setTimeDomain(); + super.refresh(); + } + + protected aggregateResultToDataPoint(cat: DateHistogramResult): TimelineDataPoint { + /* date fields are returned with keys containing identifiers by elasticsearch + replace with string representation, contained in 'key_as_string' field + */ + return { + date: new Date(cat.key_as_string), + doc_count: cat.doc_count, + }; + } + + /** Retrieve doc counts for a series. + * + * @param series series object + * @param setSearchRatio whether the `searchRatio` property of the series should be updated. + * True when retrieving results for the entire series, false when retrieving a window. + */ + protected requestSeriesDocCounts(queryModel: QueryModel) { + const aggregation = new DateHistogramAggregator( + this.visualizedField, + this.currentTimeCategory + ); + return this.searchService.aggregateSearch(this.queryModel.corpus, queryModel, aggregation); + } + + protected requestSeriesTermFrequency(series: TimelineSeries, queryModel: QueryModel) { + const bins = this.makeTermFrequencyBins(series); + return this.visualizationService.dateTermFrequencySearch( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins, + this.currentTimeCategory + ); + } + + + /** + * Add a date filter to a query model restricting it to the provided min and max values. + */ + private addQueryDateFilter(query: QueryModel, min: Date, max: Date): QueryModel { + const queryModelCopy = query.clone(); + // download zoomed in results + const filter = this.visualizedField.makeSearchFilter(); + filter.set({ min, max }); + queryModelCopy.addFilter(filter); + return queryModelCopy; + } + + private makeTermFrequencyBins(series: TimelineSeries) { + return series.data.map((bin, index) => { + const [minDate, maxDate] = this.categoryTimeDomain( + bin, + index, + series + ); + return { + start_date: minDate, + end_date: maxDate, + size: this.documentLimitForCategory(bin, series), + }; + }); + } + + /** get min/max date for the entire graph and set domain and time category */ + private setTimeDomain() { + const filter = this.queryModel.filterForField(this.visualizedField) + || this.visualizedField.makeSearchFilter(); + const currentDomain = filter.currentData as DateFilterData; + const min = new Date(currentDomain.min); + const max = new Date(currentDomain.max); + // this.xDomain = [min, max]; TODO: remove this or keep it? + this.currentTimeCategory = this.calculateTimeCategory(min, max); + } + + /** time domain for a bin */ + private categoryTimeDomain(cat, catIndex, series): [Date, Date] { + const startDate = cat.date; + const endDate = + catIndex < series.data.length - 1 + ? series.data[catIndex + 1].date + : undefined; + return [startDate, endDate]; + } + + /** + * Get the time category (year/month/week/day) that should be used in the graph, + * based on minimum and maximum dates on the x axis. + */ + private calculateTimeCategory(min: Date, max: Date): TimeCategory { + const diff = differenceInDays(max, min); + if (diff <= this.scaleDownThreshold) { + return 'day'; + } else if (diff <= this.scaleDownThreshold * 7) { + return 'week'; + } else if (diff <= this.scaleDownThreshold * 30) { + return 'month'; + } else { + return 'year'; + } + } + +} From ff8914dc1adba9dd83994b8bc287fdbf9e15ec08 Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 13:59:55 +0100 Subject: [PATCH 02/19] use behaviorsubject for rawData --- .../visualization/barchart/results-count.ts | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts index bb269d685..fc6927e23 100644 --- a/frontend/src/app/visualization/barchart/results-count.ts +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -20,7 +20,7 @@ export abstract class BarchartData< DataPoint extends TimelineDataPoint | HistogramDataPoint > { // rawData: a list of series - rawData: BarchartSeries[]; + rawData$: BehaviorSubject[]>; documentLimitExceeded = false; // whether the results include documents than the limit totalTokenCountAvailable: boolean; // whether the data includes token count totals dataHasLoaded: boolean; @@ -116,23 +116,25 @@ export abstract class BarchartData< } private initQueries(): void { - this.rawData = [ + const series = [ this.newSeries(this.queryModel.queryText), ...this.comparedQueries.state$.value.compare.map(this.newSeries) ]; + this.rawData$ = new BehaviorSubject(series); } /** update the queries in the graph to the input array. Preserve results if possible, * and start loading for the rest. */ private updateQueries(queries: string[]) { - if (this.rawData) { - this.rawData = queries.map((queryText) => { - const existingSeries = this.rawData.find( + if (this.rawData$) { + const data = queries.map((queryText) => { + const existingSeries = this.rawData$.value.find( (series) => series.queryText === queryText ); return existingSeries || this.newSeries(queryText); }); + this.rawData$.next(data); this.loadData(); } } @@ -154,9 +156,9 @@ export abstract class BarchartData< : _.identity ) .then((rawData) => { - this.rawData = rawData; + this.rawData$.next(rawData); - if (!this.rawData.length) { + if (!rawData.length) { this.error$.next('No results'); }; }); @@ -164,7 +166,7 @@ export abstract class BarchartData< } private requestDocumentData(): Promise[]> { - const dataPromises = this.rawData.map((series) => { + const dataPromises = this.rawData$.value.map((series) => { if (!series.data.length) { // retrieve data if it was not already loaded return this.getSeriesDocumentData(series); @@ -180,7 +182,7 @@ export abstract class BarchartData< /** Retrieve all term frequencies and store in `rawData`. * Term frequencies are only loaded if they were not already there. */ - private requestTermFrequencyData(rawData: typeof this.rawData): Promise[]> { + private requestTermFrequencyData(rawData: typeof this.rawData$.value): Promise[]> { // cancel and stop polling running tasks this.stopPolling$.next(); const dataPromises = rawData.map(series => { @@ -213,8 +215,8 @@ export abstract class BarchartData< * This means that not all documents will be read when counting term frequency. */ private checkDocumentLimitExceeded( - rawData: typeof this.rawData - ): typeof this.rawData { + rawData: typeof this.rawData$.value + ): typeof this.rawData$.value { this.documentLimitExceeded = rawData.find((series) => series.searchRatio < 1) !== undefined; return rawData; @@ -398,7 +400,7 @@ export class TimelineData extends BarchartData { + this.rawData$.value.map(series => { const queryModelCopy = this.addQueryDateFilter( this.queryModel, min, From 445cb6b5ead5a97609ede2c0291a67698fb2b457 Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 14:42:40 +0100 Subject: [PATCH 03/19] use data model in barchart components --- .../barchart/barchart.directive.ts | 389 +++--------------- .../barchart/histogram.component.html | 6 +- .../barchart/histogram.component.ts | 97 +---- .../visualization/barchart/results-count.ts | 115 ++++-- .../barchart/timeline.component.html | 6 +- .../barchart/timeline.component.ts | 165 ++------ .../visualization.component.html | 8 +- 7 files changed, 181 insertions(+), 605 deletions(-) diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index f75a2ec47..c741e54cc 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/member-ordering */ -import { Directive, EventEmitter, Host, HostBinding, Input, OnChanges, OnDestroy, OnInit, Output, SimpleChanges } from '@angular/core'; +import { Directive, EventEmitter, HostBinding, Input, OnChanges, OnDestroy, OnInit, Output, SimpleChanges } from '@angular/core'; import * as _ from 'lodash'; @@ -10,27 +10,19 @@ import { } from '@services/index'; import { Chart, ChartOptions } from 'chart.js'; import { - Corpus, FreqTableHeaders, QueryModel, CorpusField, - TaskResult, - BarchartSeries, - TimelineDataPoint, - HistogramDataPoint, - TermFrequencyResult, ChartParameters, } from '@models'; import Zoom from 'chartjs-plugin-zoom'; import { BehaviorSubject, Subject } from 'rxjs'; import { selectColor } from '@utils/select-color'; import { VisualizationService } from '@services/visualization.service'; -import { showLoading } from '@utils/utils'; -import { takeUntil } from 'rxjs/operators'; -import { DateHistogramResult, TermsResult } from '@models/aggregation'; import { ComparedQueries } from '@models/compared-queries'; import { RouterStoreService } from '../../store/router-store.service'; import { hasPrefixTerm } from './query-utils'; +import { BarchartData } from './results-count'; const hintSeenSessionStorageKey = 'hasSeenTimelineZoomingHint'; const hintHidingMinDelay = 500; // milliseconds @@ -47,20 +39,15 @@ const documentLimitPrefixQueries = 1000; /** The barchartComponent is used to define shared functionality between the * histogram and timeline components. It does not function as a stand-alone component. */ export abstract class BarchartDirective< - AggregateResult extends TermsResult | DateHistogramResult, - DataPoint extends TimelineDataPoint | HistogramDataPoint + Data extends BarchartData > implements OnChanges, OnInit, OnDestroy { @HostBinding('style.display') display = 'block'; // needed for loading spinner positioning public showHint: boolean; - // rawData: a list of series - rawData: BarchartSeries[]; - // chart object chart: Chart; - @Input() corpus: Corpus; @Input() queryModel: QueryModel; @Input() visualizedField: CorpusField; @Input() asTable: boolean; @@ -72,8 +59,8 @@ export abstract class BarchartDirective< chartType: 'bar' | 'line' | 'scatter' = 'bar'; comparedQueries: ComparedQueries; - documentLimitExceeded = false; // whether the results include documents than the limit - totalTokenCountAvailable: boolean; // whether the data includes token count totals + // documentLimitExceeded = false; // whether the results include documents than the limit + // totalTokenCountAvailable: boolean; // whether the data includes token count totals // table data tableHeaders: FreqTableHeaders; @@ -103,11 +90,12 @@ export abstract class BarchartDirective< @Output() error = new EventEmitter(); destroy$ = new Subject(); - stopPolling$ = new Subject(); + // stopPolling$ = new Subject(); + // tasksToCancel: string[]; - tasksToCancel: string[]; + // dataHasLoaded: boolean; - dataHasLoaded: boolean; + data: Data; basicChartOptions: ChartOptions = { // chart options not suitable for Chart.defaults.global scales: { @@ -173,11 +161,10 @@ export abstract class BarchartDirective< chartDefault.plugins.tooltip.displayColors = false; chartDefault.plugins.tooltip.intersect = false; this.comparedQueries = new ComparedQueries(this.routerStoreService); - this.comparedQueries.allQueries$.subscribe(this.updateQueries.bind(this)); } get isLoading() { - return this.isLoading$.value; + return this.data.loading$.value; } get documentLimit(): number { @@ -189,25 +176,33 @@ export abstract class BarchartDirective< } ngOnChanges(changes: SimpleChanges) { - if (changes.queryModel) { - this.queryModel.update - .pipe(takeUntil(this.destroy$)) - .subscribe(this.refreshChart.bind(this)); - } - // new doc counts should be requested if query has changed if (this.changesRequireRefresh(changes)) { this.refreshChart(); } else if (changes.palette) { - this.prepareChart(); + this.setChart(); } } ngOnDestroy(): void { - this.stopPolling$.next(); this.destroy$.next(undefined); + this.destroy$.complete(); this.comparedQueries.complete(); } + onDataLoaded(data: typeof this.data.rawData$.value) { + // initialise or update chart + this.setChart(); + + // update freqtable + this.setTableHeaders(); + this.setTableData(); + + // load zoomed-in data if needed + if (this.isZoomedIn) { + this.onZoomIn(this.chart, true); + } + } + /** check whether input changes should force reloading the data */ changesRequireRefresh(changes: SimpleChanges): boolean { const relevantChanges = [ @@ -227,8 +222,8 @@ export abstract class BarchartDirective< onOptionChange(chartParameters: ChartParameters) { this.normalizer = chartParameters.normalizer; this.chartType = chartParameters.chartType; - if (this.rawData && this.chart) { - this.prepareChart(); + if (this.data.rawData$.value && this.chart) { + this.onDataLoaded(this.data.rawData$.value); } } @@ -236,16 +231,12 @@ export abstract class BarchartDirective< * clear data and update chart */ refreshChart(): void { - this.initQueries(); + this.data = this.initData(); + this.data.rawData$.subscribe(data => this.onDataLoaded(data)); + this.data.error$.subscribe(message => this.onDataError(message)); + // this.initQueries(); this.clearCanvas(); - this.prepareChart(); - } - - initQueries(): void { - this.rawData = [ - this.newSeries(this.queryText), - ...this.comparedQueries.state$.value.compare.map(this.newSeries) - ]; + // this.prepareChart(); } /** if a chart is active, clear canvas and reset chart object */ @@ -257,88 +248,11 @@ export abstract class BarchartDirective< } } - /** update the queries in the graph to the input array. Preserve results if possible, and kick off loading the rest. */ - updateQueries(queries: string[]) { - if (this.rawData) { - this.rawData = queries.map((queryText) => { - const existingSeries = this.rawData.find( - (series) => series.queryText === queryText - ); - return existingSeries || this.newSeries(queryText); - }); - this.prepareChart(); - } - } - - /** make a blank series object */ - newSeries(queryText: string): BarchartSeries { - return { - queryText, - data: [], - total_doc_count: 0, - searchRatio: 1.0, - }; - } - - /** Show a loading spinner and load data for the graph. - * This function should be called after (potential) changes to parameters. - */ - prepareChart() { - showLoading(this.isLoading$, this.loadData()); - } - - /** load data for the graph (if needed), update the graph and freqtable. */ - loadData(): Promise { - // load data if needed - return this.requestDocumentData() - .then( - this.frequencyMeasure === 'tokens' - ? this.requestTermFrequencyData.bind(this) - : _.identity - ) - .then((rawData) => { - this.rawData = rawData; - - if (!this.rawData.length) { - this.error.emit({ message: 'No results' }); - } - - // initialise or update chart - this.setChart(); - - // update freqtable - this.setTableHeaders(); - this.setTableData(); - - // load zoomed-in data if needed - if (this.isZoomedIn) { - this.onZoomIn(this.chart, true); - } - }); - } - - /** Retrieve all document frequencies and store in `rawData`. - * Document frequencies are only loaded if they are not already in the data. */ - requestDocumentData(): Promise { - const dataPromises = this.rawData.map((series) => { - if (!series.data.length) { - // retrieve data if it was not already loaded - return this.getSeriesDocumentData(series); - } else { - return series; - } - }); - - return Promise.all(dataPromises).then( - this.checkDocumentLimitExceeded.bind(this) - ); - } - selectSearchFields(queryModel: QueryModel) { if (this.frequencyMeasure === 'documents') { return queryModel; } else { - const mainContentFields = this.corpus.fields.filter( + const mainContentFields = this.queryModel.corpus.fields.filter( (field) => field.searchable && field.displayType === 'text_content' ); @@ -348,215 +262,13 @@ export abstract class BarchartDirective< } } - /** - * Convert the result from an document search into a data series object. - * - Converts to the relevant DataPoint type using `aggregateResultToDataPoint` - * - Adds the total document count and search ratio for retrieving term frequencies. - * - Adds the relative document count. - * - * @param result result from aggregation search - * @param series series object that this data belongs to - * @param setSearchRatio whether the search ratio should be reset. Defaults to `true`, - * may be set to `false` when loading a portion of the series during zoom. - * @returns a copy of the series with the document counts included. - */ - docCountResultIntoSeries( - result: AggregateResult[], - series: BarchartSeries, - setSearchRatio = true - ): BarchartSeries { - let data = result.map(this.aggregateResultToDataPoint); - const totalDocCount = this.totalDocCount(result); - const searchRatio = setSearchRatio - ? this.documentLimit / totalDocCount - : series.searchRatio; - data = this.includeRelativeDocCount(data, totalDocCount); - return { - data, - total_doc_count: totalDocCount, - searchRatio, - queryText: series.queryText, - }; - } - - /** convert the output of an aggregation search to the relevant result type */ - abstract aggregateResultToDataPoint(cat: AggregateResult): DataPoint; - - /** fill in the `relative_doc_count` property for an array of datapoints. - */ - includeRelativeDocCount(data: DataPoint[], total: number): DataPoint[] { - return data.map((item) => { - const result = _.clone(item); - result.relative_doc_count = result.doc_count / total; - return result; - }); - } - - /** - * Check whether any series found more documents than the document limit. - * This means that not all documents will be read when counting term frequency. - */ - checkDocumentLimitExceeded( - rawData: typeof this.rawData - ): typeof this.rawData { - this.documentLimitExceeded = - rawData.find((series) => series.searchRatio < 1) !== undefined; - return rawData; + onDataError(message: string) { + console.error(message); + this.error.emit(`could not load results: ${message}`); } - /** Retrieve all term frequencies and store in `rawData`. - * Term frequencies are only loaded if they were not already there. - */ - requestTermFrequencyData(rawData: typeof this.rawData): Promise[]> { - // cancel and stop polling running tasks - this.stopPolling$.next(); - const dataPromises = rawData.map(series => { - if (series.queryText && series.data.length && series.data[0].match_count === undefined) { - // retrieve data if it was not already loaded - return this.getTermFrequencies(series, this.queryModel); - } else { - return series; - } - }); - - return Promise.all(dataPromises).then( - this.checkTotalTokenCount.bind(this) - ); - } - - checkTotalTokenCount(rawData: typeof this.rawData): typeof this.rawData { - const totalTokenCountAvailable = - rawData.find((series) => - series.data.find((cat) => cat.token_count) - ) !== undefined; - if ( - this.frequencyMeasure === 'tokens' && - totalTokenCountAvailable && - !this.totalTokenCountAvailable - ) { - this.normalizer = 'terms'; - } - this.totalTokenCountAvailable = totalTokenCountAvailable; - return rawData; - } - - getTermFrequencies( - series: BarchartSeries, - queryModel: QueryModel - ): Promise> { - this.dataHasLoaded = false; - const queryModelCopy = this.queryModelForSeries(series, queryModel); - return new Promise((resolve, reject) => { - this.requestSeriesTermFrequency(series, queryModelCopy).then(response => { - this.tasksToCancel = response.task_ids; - const poller$ = this.apiService.pollTasks(this.tasksToCancel, this.stopPolling$); - poller$.subscribe({ - error: (error) => { - this.onFailure(error); - reject(error); - }, - next: (result) => { - resolve(this.processSeriesTermFrequency(result['results'], series)); - }, - complete: () => { - // abort tasks if the Observable is completed via takeUntil - if (!this.dataHasLoaded) { - this.apiService.abortTasks({ task_ids: this.tasksToCancel }); - } - } - }); - }); - }); - } - - onFailure(error) { - console.error(error); - this.error.emit(`could not load results: ${error.message}`); - } - - abstract requestSeriesTermFrequency(series: BarchartSeries, queryModel: QueryModel): Promise; - - abstract makeTermFrequencyBins(series: BarchartSeries); - - processSeriesTermFrequency(results: TermFrequencyResult[], series: BarchartSeries): BarchartSeries { - this.dataHasLoaded = true; - series.data = _.zip(series.data, results).map(pair => { - const [bin, res] = pair; - return this.addTermFrequencyToCategory(res, bin); - }); - return series; - }; - - - /** total document count for a data array */ - totalDocCount(data: AggregateResult[]) { - return _.sumBy(data, (item) => item.doc_count); - } - - /** - * calculate the maximum number of documents to read through in a bin - * when determining term frequency. - */ - documentLimitForCategory( - cat: DataPoint, - series: BarchartSeries - ): number { - return _.min([ - this.documentLimit, - _.ceil(cat.doc_count * series.searchRatio), - ]); - } - - /** - * add term frequency data to a DataPoint object - * - * @param result output from request for term frequencies - * @param cat DataPoint object where the data should be added - */ - addTermFrequencyToCategory( - data: TermFrequencyResult, - cat: DataPoint - ): DataPoint { - cat.match_count = data.match_count; - cat.total_doc_count = data.total_doc_count; - cat.token_count = data.token_count; - cat.matches_by_doc_count = data.match_count / data.total_doc_count; - cat.matches_by_token_count = data.token_count - ? data.match_count / data.token_count - : undefined; - return cat; - } - - /** Request and fill in doc counts for a series */ - getSeriesDocumentData( - series: BarchartSeries, - queryModel: QueryModel = this.queryModel, - setSearchRatio = true - ): Promise> { - const queryModelCopy = this.queryModelForSeries(series, queryModel); - - return this.requestSeriesDocCounts(queryModelCopy).then((result) => - this.docCountResultIntoSeries(result, series, setSearchRatio) - ); - } - - /** adapt query model to fit series: use correct search fields and query text */ - queryModelForSeries( - series: BarchartSeries, - queryModel: QueryModel - ) { - return this.selectSearchFields( - this.setQueryText(queryModel, series.queryText) - ); - } - - /** Request doc counts for a series */ - abstract requestSeriesDocCounts( - queryModel: QueryModel - ): Promise; - requestFullData() { - this.fullDataRequest() + this.data.fullDataRequest() .then(() => this.notificationService.showMessage( 'Full data requested! You will receive an email when your download is ready.', @@ -576,8 +288,6 @@ export abstract class BarchartDirective< }); } - abstract fullDataRequest(): Promise; - /** update or initialise chart (should be ran after updates to `rawData`) */ setChart(): void { if (this.chart) { @@ -693,17 +403,10 @@ export abstract class BarchartDirective< } } - /** return a copy of a query model with the query text set to the given value */ - setQueryText(query: QueryModel, queryText: string): QueryModel { - const queryModelCopy = query.clone(); - queryModelCopy.setQueryText(queryText); - return queryModelCopy; - } - /** assemble the array of table data */ setTableData() { - if (this.rawData && this.rawData.length) { - this.tableData = _.flatMap(this.rawData, (series) => + if (this.data.rawData$.value && this.data.rawData$.value.length) { + this.tableData = _.flatMap(this.data.rawData$.value, (series) => series.data.map((item) => { const result = _.cloneDeep(item) as any; result.queryText = series.queryText; @@ -723,9 +426,9 @@ export abstract class BarchartDirective< * This differs per series, but user message only gives one value, currently the minimum. */ get percentageDocumentsSearched() { - if (this.rawData) { + if (this.data.rawData$.value) { return _.round( - 100 * _.min(this.rawData.map((series) => series.searchRatio)), + 100 * _.min(this.data.rawData$.value.map((series) => series.searchRatio)), 1 ); } @@ -740,7 +443,7 @@ export abstract class BarchartDirective< } get searchFields(): string { - if (this.corpus && this.queryModel) { + if (this.queryModel) { const searchFields = this.selectSearchFields( this.queryModel ).searchFields; @@ -754,8 +457,8 @@ export abstract class BarchartDirective< } chartTitle() { - const queryTexts = this.rawData.map((series) => series.queryText); - if (this.queryText == null && this.rawData.length === 1) { + const queryTexts = this.data.rawData$.value.map((series) => series.queryText); + if (this.queryText == null && this.data.rawData$.value.length === 1) { return `Frequency of documents by ${this.visualizedField.displayName} (n of ${this.frequencyMeasure}, ${this.normalizer})`; } else { const normalizationText = ['raw', 'percent'].includes( @@ -774,4 +477,6 @@ export abstract class BarchartDirective< return this.queryModel.queryText; } } + + abstract initData(): Data; } diff --git a/frontend/src/app/visualization/barchart/histogram.component.html b/frontend/src/app/visualization/barchart/histogram.component.html index 90f17cf1b..25054a79d 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.html +++ b/frontend/src/app/visualization/barchart/histogram.component.html @@ -6,10 +6,10 @@
+ (chartParameters)="onOptionChange($event)">
@@ -23,7 +23,7 @@ -
+
The results for your search include too many documents to count all matches. diff --git a/frontend/src/app/visualization/barchart/histogram.component.ts b/frontend/src/app/visualization/barchart/histogram.component.ts index a1fdac1a5..0ea385cee 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.ts +++ b/frontend/src/app/visualization/barchart/histogram.component.ts @@ -1,16 +1,9 @@ import { Component, OnChanges, OnInit } from '@angular/core'; import * as _ from 'lodash'; -import { - HistogramDataPoint, - HistogramSeries, - MultipleChoiceFilterOptions, - QueryModel, - RangeFilterOptions, -} from '@models/index'; import { selectColor } from '@utils/select-color'; import { BarchartDirective } from './barchart.directive'; -import { TermsAggregator, TermsResult } from '@models/aggregation'; +import { HistogramData } from './results-count'; function formatXAxisLabel(value): string { const label = this.getLabelForValue(value); // from chartJS api @@ -28,7 +21,7 @@ function formatXAxisLabel(value): string { standalone: false }) export class HistogramComponent - extends BarchartDirective + extends BarchartDirective implements OnInit, OnChanges { /** On what property should the data be sorted? */ get defaultSort(): string { @@ -38,80 +31,24 @@ export class HistogramComponent return this.currentValueKey; } - /** specify aggregator object based on visualised field; - * used in document requests. - */ - getAggregator(): TermsAggregator { - let size = 100; - - const filterOptions = this.visualizedField.filterOptions; - if (filterOptions?.name === 'MultipleChoiceFilter') { - size = (filterOptions as MultipleChoiceFilterOptions).option_count; - } else if (filterOptions?.name === 'RangeFilter') { - const filterRange = - (filterOptions as RangeFilterOptions).upper - - (filterOptions as RangeFilterOptions).lower; - size = _.max([size, filterRange]) - } - return new TermsAggregator(this.visualizedField, size); - } - - requestSeriesDocCounts(queryModel: QueryModel) { - const aggregator = this.getAggregator(); - - return this.searchService.aggregateSearch(this.corpus, queryModel, aggregator); - } - - aggregateResultToDataPoint(cat: TermsResult) { - return cat; - } - - requestSeriesTermFrequency( - series: HistogramSeries, - queryModel: QueryModel - ) { - const bins = this.makeTermFrequencyBins(series); - return this.visualizationService.aggregateTermFrequencySearch( - this.corpus, - queryModel, - this.visualizedField.name, - bins + initData(): HistogramData { + return new HistogramData( + this.queryModel, + this.comparedQueries, + this.frequencyMeasure, + this.visualizedField, + this.searchService, + this.apiService, + this.visualizationService, + this.destroy$, ); } - makeTermFrequencyBins(series: HistogramSeries) { - return series.data.map((bin) => ({ - fieldValue: bin.key, - size: this.documentLimitForCategory(bin, series), - })); - } - - fullDataRequest() { - const paramsPerSeries = this.rawData.map((series) => { - const queryModel = this.queryModelForSeries( - series, - this.queryModel - ); - const bins = this.makeTermFrequencyBins(series); - return this.visualizationService.makeAggregateTermFrequencyParameters( - this.corpus, - queryModel, - this.visualizedField.name, - bins - ); - }); - return this.apiService.requestFullData({ - visualization: 'aggregate_term_frequency', - parameters: paramsPerSeries, - corpus_name: this.corpus.name, - }); - } - getLabels(): string[] { // make an array of all unique labels and sort - if (this.rawData) { - const all_labels = _.flatMap(this.rawData, (series) => + if (this.data.rawData$.value) { + const all_labels = _.flatMap(this.data.rawData$.value, (series) => series.data.map((item) => item.key) ); const labels = all_labels.filter( @@ -123,7 +60,7 @@ export class HistogramComponent } else { const valueKey = this.currentValueKey; sorted_labels = _.sortBy(labels, (label) => - _.sumBy(this.rawData, (series) => { + _.sumBy(this.data.rawData$.value, (series) => { const item = series.data.find((i) => i.key === label); return -1 * (item ? item[valueKey] : 0); }) @@ -137,7 +74,7 @@ export class HistogramComponent const labels = this.getLabels(); const valueKey = this.currentValueKey; - return this.rawData.map((series, seriesIndex) => ({ + return this.data.rawData$.value.map((series, seriesIndex) => ({ type: this.chartType, label: series.queryText ? series.queryText : '(no query)', data: labels.map((key) => { @@ -183,7 +120,7 @@ export class HistogramComponent this.normalizer === 'raw' ? 'Frequency' : 'Relative frequency'; const valueKey = this.currentValueKey; - if (this.rawData.length > 1) { + if (this.data.rawData$.value.length > 1) { // if there are several queries, fulltable is disabled this.tableHeaders = [ { key: 'key', label, isSecondaryFactor: true }, diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts index fc6927e23..e40b22fa0 100644 --- a/frontend/src/app/visualization/barchart/results-count.ts +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -115,6 +115,16 @@ export abstract class BarchartData< ]); } + /** adapt query model to fit series: use correct search fields and query text */ + protected queryModelForSeries( + series: BarchartSeries, + queryModel: QueryModel + ) { + return this.selectSearchFields( + this.setQueryText(queryModel, series.queryText) + ); + } + private initQueries(): void { const series = [ this.newSeries(this.queryModel.queryText), @@ -198,18 +208,6 @@ export abstract class BarchartData< } - - /** adapt query model to fit series: use correct search fields and query text */ - private queryModelForSeries( - series: BarchartSeries, - queryModel: QueryModel - ) { - return this.selectSearchFields( - this.setQueryText(queryModel, series.queryText) - ); - } - - /** * Check whether any series found more documents than the document limit. * This means that not all documents will be read when counting term frequency. @@ -329,6 +327,8 @@ export abstract class BarchartData< this.error$.complete(); } + abstract fullDataRequest(): Promise; + /** Request doc counts for a series */ protected abstract requestSeriesDocCounts( queryModel: QueryModel @@ -343,6 +343,27 @@ export abstract class BarchartData< } export class HistogramData extends BarchartData { + fullDataRequest(): Promise { + const paramsPerSeries = this.rawData$.value.map((series) => { + const queryModel = this.queryModelForSeries( + series, + this.queryModel + ); + const bins = this.makeTermFrequencyBins(series); + return this.visualizationService.makeAggregateTermFrequencyParameters( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins + ); + }); + return this.apiService.requestFullData({ + visualization: 'aggregate_term_frequency', + parameters: paramsPerSeries, + corpus_name: this.queryModel.corpus.name, + }); + } + /** specify aggregator object based on visualised field; * used in document requests. */ @@ -350,9 +371,9 @@ export class HistogramData extends BarchartData let size = 100; const filterOptions = this.visualizedField.filterOptions; - if (filterOptions.name === 'MultipleChoiceFilter') { + if (filterOptions?.name === 'MultipleChoiceFilter') { size = (filterOptions as MultipleChoiceFilterOptions).option_count; - } else if (filterOptions.name === 'RangeFilter') { + } else if (filterOptions?.name === 'RangeFilter') { const filterRange = (filterOptions as RangeFilterOptions).upper - (filterOptions as RangeFilterOptions).lower; @@ -389,7 +410,6 @@ export class HistogramData extends BarchartData size: this.documentLimitForCategory(bin, series), })); } - } export class TimelineData extends BarchartData { @@ -398,9 +418,8 @@ export class TimelineData extends BarchartData { + zoomedInData (min: Date, max: Date): Promise[]> { + const promises = this.rawData$.value.map(series => { const queryModelCopy = this.addQueryDateFilter( this.queryModel, min, @@ -418,8 +437,51 @@ export class TimelineData extends BarchartData { + const queryModel = this.queryModelForSeries( + series, + this.queryModel + ); + const bins = this.makeTermFrequencyBins(series); + const unit = this.currentTimeCategory; + return this.visualizationService.makeDateTermFrequencyParameters( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins, + unit + ); + }); + return this.apiService.requestFullData({ + visualization: 'date_term_frequency', + parameters: paramsPerSeries, + corpus_name: this.queryModel.corpus.name, + }); + } + + + /** + * Get the time category (year/month/week/day) that should be used in the graph, + * based on minimum and maximum dates on the x axis. + */ + calculateTimeCategory(min: Date, max: Date): TimeCategory { + const diff = differenceInDays(max, min); + if (diff <= this.scaleDownThreshold) { + return 'day'; + } else if (diff <= this.scaleDownThreshold * 7) { + return 'week'; + } else if (diff <= this.scaleDownThreshold * 30) { + return 'month'; + } else { + return 'year'; + } } + protected refresh(): void { this.setTimeDomain(); super.refresh(); @@ -509,21 +571,4 @@ export class TimelineData extends BarchartData + (chartParameters)="onOptionChange($event)">
@@ -27,7 +27,7 @@
-
+
The results for your search include too many documents to count all matches. diff --git a/frontend/src/app/visualization/barchart/timeline.component.ts b/frontend/src/app/visualization/barchart/timeline.component.ts index 2069b1218..26869ad9c 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.ts +++ b/frontend/src/app/visualization/barchart/timeline.component.ts @@ -1,13 +1,12 @@ import { Component, OnChanges, OnInit } from '@angular/core'; import * as _ from 'lodash'; -import { addDays, addMonths, addWeeks, addYears, differenceInDays, format } from 'date-fns'; +import { addDays, addMonths, addWeeks, addYears, format } from 'date-fns'; import 'chartjs-adapter-moment'; import { QueryModel, TimelineSeries, - TimelineDataPoint, TimeCategory, DateFilterData, } from '@models/index'; @@ -15,10 +14,7 @@ import { BarchartDirective } from './barchart.directive'; import { selectColor } from '@utils/select-color'; import { showLoading } from '@utils/utils'; -import { - DateHistogramAggregator, - DateHistogramResult, -} from '@models/aggregation'; +import { TimelineData } from './results-count'; @Component({ @@ -28,20 +24,24 @@ import { standalone: false }) export class TimelineComponent - extends BarchartDirective + extends BarchartDirective implements OnChanges, OnInit { /** domain on the axis */ public xDomain: [Date, Date]; /** time unit on the x-axis */ private currentTimeCategory: TimeCategory; - /** threshold for scaling down a unit on the time scale */ - private scaleDownThreshold = 10; - - refreshChart(): void { - this.initQueries(); - this.clearCanvas(); - this.setTimeDomain(); - this.prepareChart(); + + initData(): TimelineData { + return new TimelineData( + this.queryModel, + this.comparedQueries, + this.frequencyMeasure, + this.visualizedField, + this.searchService, + this.apiService, + this.visualizationService, + this.destroy$, + ); } /** get min/max date for the entire graph and set domain and time category */ @@ -52,57 +52,7 @@ export class TimelineComponent const min = new Date(currentDomain.min); const max = new Date(currentDomain.max); this.xDomain = [min, max]; - this.currentTimeCategory = this.calculateTimeCategory(min, max); - } - - aggregateResultToDataPoint(cat: DateHistogramResult): TimelineDataPoint { - /* date fields are returned with keys containing identifiers by elasticsearch - replace with string representation, contained in 'key_as_string' field - */ - return { - date: new Date(cat.key_as_string), - doc_count: cat.doc_count, - }; - } - - /** Retrieve doc counts for a series. - * - * @param series series object - * @param setSearchRatio whether the `searchRatio` property of the series should be updated. - * True when retrieving results for the entire series, false when retrieving a window. - */ - requestSeriesDocCounts(queryModel: QueryModel) { - const aggregation = new DateHistogramAggregator( - this.visualizedField, - this.currentTimeCategory - ); - return this.searchService.aggregateSearch(this.corpus, queryModel, aggregation); - } - - requestSeriesTermFrequency(series: TimelineSeries, queryModel: QueryModel) { - const bins = this.makeTermFrequencyBins(series); - return this.visualizationService.dateTermFrequencySearch( - this.corpus, - queryModel, - this.visualizedField.name, - bins, - this.currentTimeCategory - ); - } - - makeTermFrequencyBins(series: TimelineSeries) { - return series.data.map((bin, index) => { - const [minDate, maxDate] = this.categoryTimeDomain( - bin, - index, - series - ); - return { - start_date: minDate, - end_date: maxDate, - size: this.documentLimitForCategory(bin, series), - }; - }); + this.currentTimeCategory = this.data.calculateTimeCategory(min, max); } /** time domain for a bin */ @@ -115,29 +65,6 @@ export class TimelineComponent return [startDate, endDate]; } - fullDataRequest() { - const paramsPerSeries = this.rawData.map((series) => { - const queryModel = this.queryModelForSeries( - series, - this.queryModel - ); - const bins = this.makeTermFrequencyBins(series); - const unit = this.calculateTimeCategory(...this.xDomain); // use initial unit, not zoomed-in-status - return this.visualizationService.makeDateTermFrequencyParameters( - this.corpus, - queryModel, - this.visualizedField.name, - bins, - unit - ); - }); - return this.apiService.requestFullData({ - visualization: 'date_term_frequency', - parameters: paramsPerSeries, - corpus_name: this.corpus.name, - }); - } - setChart() { if (this.chart) { // reset time unit to the one set in the chart @@ -153,7 +80,7 @@ export class TimelineComponent } getDatasets() { - return this.rawData.map((series, seriesIndex) => { + return this.data.rawData$.value.map((series, seriesIndex) => { const data = this.chartDataFromSeries(series); return { type: this.chartType, @@ -179,7 +106,7 @@ export class TimelineComponent } callibratexAxis(date: Date, margin: number = 1) { - switch(this.currentTimeCategory) { + switch (this.data.currentTimeCategory) { case 'day': return addDays(date, margin); case 'week': @@ -206,7 +133,7 @@ export class TimelineComponent x.type = 'time'; (x as any).time = { minUnit: 'day', - unit: this.currentTimeCategory, + unit: this.data.currentTimeCategory, }; x.min = xMin.toISOString(); x.max = xMax.toISOString(); @@ -246,16 +173,16 @@ export class TimelineComponent * underlying data. */ onZoomIn(chart, triggeredByDataUpdate = false) { - const initialTimeCategory = this.calculateTimeCategory(...this.xDomain); - const previousTimeCategory = this.currentTimeCategory; + const initialTimeCategory = this.data.calculateTimeCategory(...this.xDomain); + const previousTimeCategory = this.data.currentTimeCategory; const min = new Date(chart.scales.x.min); const max = new Date(chart.scales.x.max); - this.currentTimeCategory = this.calculateTimeCategory(min, max); + this.data.currentTimeCategory = this.data.calculateTimeCategory(min, max); if ( - this.currentTimeCategory !== previousTimeCategory || + this.data.currentTimeCategory !== previousTimeCategory || (triggeredByDataUpdate && - this.currentTimeCategory !== initialTimeCategory) + this.data.currentTimeCategory !== initialTimeCategory) ) { showLoading( this.isLoading$, @@ -288,29 +215,8 @@ export class TimelineComponent // when zooming, hide data for smooth transition chart.update(triggeredByDataUpdate ? 'none' : 'hide'); - const dataPromises: Promise[] = chart.data.datasets.map( - (dataset, seriesIndex) => { - const series = this.rawData[seriesIndex]; - const queryModelCopy = this.addQueryDateFilter( - this.queryModel, - min, - max - ); - return this.getSeriesDocumentData( - series, - queryModelCopy, - false - ).then((result) => { - if (this.frequencyMeasure === 'tokens') { - return this.getTermFrequencies(result, queryModelCopy); - } else { - return result; - } - }); - } - ); - const zoomedInResults = await Promise.all(dataPromises); + const zoomedInResults = await this.data.zoomedInData(min, max); zoomedInResults.forEach((data, seriesIndex) => { chart.data.datasets[seriesIndex].data = @@ -336,7 +242,7 @@ export class TimelineComponent /** trigger zoom out, update chart data accordingly */ zoomOut(): void { this.chart.resetZoom(); - this.currentTimeCategory = this.calculateTimeCategory(...this.xDomain); + this.currentTimeCategory = this.data.calculateTimeCategory(...this.xDomain); (this.chart.options.scales.x as any).time.unit = this.currentTimeCategory; this.chart.update(); @@ -344,29 +250,12 @@ export class TimelineComponent this.setChart(); } - /** - * Get the time category (year/month/week/day) that should be used in the graph, - * based on minimum and maximum dates on the x axis. - */ - public calculateTimeCategory(min: Date, max: Date): TimeCategory { - const diff = differenceInDays(max, min); - if (diff <= this.scaleDownThreshold) { - return 'day'; - } else if (diff <= this.scaleDownThreshold * 7) { - return 'week'; - } else if (diff <= this.scaleDownThreshold * 30) { - return 'month'; - } else { - return 'year'; - } - } - setTableHeaders() { const rightColumnName = this.normalizer === 'raw' ? 'Frequency' : 'Relative frequency'; const valueKey = this.currentValueKey; - if (this.rawData.length > 1) { + if (this.data.rawData$.value.length > 1) { this.tableHeaders = [ { key: 'date', diff --git a/frontend/src/app/visualization/visualization.component.html b/frontend/src/app/visualization/visualization.component.html index 9623b1084..7f697059a 100644 --- a/frontend/src/app/visualization/visualization.component.html +++ b/frontend/src/app/visualization/visualization.component.html @@ -91,23 +91,23 @@
- - - - Date: Wed, 29 Oct 2025 14:55:07 +0100 Subject: [PATCH 04/19] time category fixes --- .../barchart/barchart.directive.ts | 4 +- .../visualization/barchart/results-count.ts | 13 ++-- .../barchart/timeline.component.spec.ts | 4 +- .../barchart/timeline.component.ts | 68 +++++++------------ 4 files changed, 35 insertions(+), 54 deletions(-) diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index c741e54cc..3e996e1b2 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -16,7 +16,7 @@ import { ChartParameters, } from '@models'; import Zoom from 'chartjs-plugin-zoom'; -import { BehaviorSubject, Subject } from 'rxjs'; +import { Subject } from 'rxjs'; import { selectColor } from '@utils/select-color'; import { VisualizationService } from '@services/visualization.service'; import { ComparedQueries } from '@models/compared-queries'; @@ -84,8 +84,6 @@ export abstract class BarchartDirective< }, }; - isLoading$ = new BehaviorSubject(false); - // eslint-disable-next-line @angular-eslint/no-output-native @Output() error = new EventEmitter(); diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts index e40b22fa0..e5563fd27 100644 --- a/frontend/src/app/visualization/barchart/results-count.ts +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -413,7 +413,8 @@ export class HistogramData extends BarchartData } export class TimelineData extends BarchartData { - currentTimeCategory: TimeCategory = 'year'; + xDomain: [Date, Date]; + timeCategory: TimeCategory = 'year'; /** threshold for scaling down a unit on the time scale */ private scaleDownThreshold = 10; @@ -447,7 +448,7 @@ export class TimelineData extends BarchartData { component.setTimeDomain(); const min = new Date('1802-01-01'); const max = new Date('1804-01-01'); - expect(component.callibratexAxis(min, -1)).toEqual(new Date('1801-01-01')); - expect(component.callibratexAxis(max)).toEqual(new Date('1805-01-01')); + expect(component.addDateMargin(min, -1)).toEqual(new Date('1801-01-01')); + expect(component.addDateMargin(max)).toEqual(new Date('1805-01-01')); }) }); diff --git a/frontend/src/app/visualization/barchart/timeline.component.ts b/frontend/src/app/visualization/barchart/timeline.component.ts index 26869ad9c..8f591cdfe 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.ts +++ b/frontend/src/app/visualization/barchart/timeline.component.ts @@ -8,13 +8,13 @@ import { QueryModel, TimelineSeries, TimeCategory, - DateFilterData, } from '@models/index'; import { BarchartDirective } from './barchart.directive'; import { selectColor } from '@utils/select-color'; import { showLoading } from '@utils/utils'; import { TimelineData } from './results-count'; +import { BehaviorSubject } from 'rxjs'; @Component({ @@ -26,10 +26,14 @@ import { TimelineData } from './results-count'; export class TimelineComponent extends BarchartDirective implements OnChanges, OnInit { - /** domain on the axis */ - public xDomain: [Date, Date]; - /** time unit on the x-axis */ - private currentTimeCategory: TimeCategory; + /** time unit on the x-axis; may be the category of zoomed-in data */ + private viewTimeCategory: TimeCategory; + + private zoomedDataLoading$ = new BehaviorSubject(false); + + get isLoading(): boolean { + return this.data.loading$.value || this.zoomedDataLoading$.value; + } initData(): TimelineData { return new TimelineData( @@ -44,34 +48,13 @@ export class TimelineComponent ); } - /** get min/max date for the entire graph and set domain and time category */ - setTimeDomain() { - const filter = this.queryModel.filterForField(this.visualizedField) - || this.visualizedField.makeSearchFilter(); - const currentDomain = filter.currentData as DateFilterData; - const min = new Date(currentDomain.min); - const max = new Date(currentDomain.max); - this.xDomain = [min, max]; - this.currentTimeCategory = this.data.calculateTimeCategory(min, max); - } - - /** time domain for a bin */ - categoryTimeDomain(cat, catIndex, series): [Date, Date] { - const startDate = cat.date; - const endDate = - catIndex < series.data.length - 1 - ? series.data[catIndex + 1].date - : undefined; - return [startDate, endDate]; - } - setChart() { if (this.chart) { // reset time unit to the one set in the chart const unit = (this.chart.options.scales.x as any).time .unit as TimeCategory; if (unit) { - this.currentTimeCategory = unit; + this.viewTimeCategory = unit; } this.updateChartData(); } else { @@ -105,8 +88,8 @@ export class TimelineComponent })); } - callibratexAxis(date: Date, margin: number = 1) { - switch (this.data.currentTimeCategory) { + addDateMargin(date: Date, margin: number = 1): Date { + switch (this.data.timeCategory) { case 'day': return addDays(date, margin); case 'week': @@ -115,7 +98,6 @@ export class TimelineComponent return addMonths(date, margin); case 'year': return addYears(date, margin); - } } @@ -123,8 +105,8 @@ export class TimelineComponent const xLabel = this.visualizedField.displayName ? this.visualizedField.displayName : this.visualizedField.name; - const xMin = this.callibratexAxis(this.xDomain[0], -1); - const xMax = this.callibratexAxis(this.xDomain[1]); + const xMin = this.addDateMargin(this.data.xDomain[0], -1); + const xMax = this.addDateMargin(this.data.xDomain[1]); const options = this.basicChartOptions; options.plugins.title.text = this.chartTitle(); @@ -133,7 +115,7 @@ export class TimelineComponent x.type = 'time'; (x as any).time = { minUnit: 'day', - unit: this.data.currentTimeCategory, + unit: this.data.timeCategory, }; x.min = xMin.toISOString(); x.max = xMax.toISOString(); @@ -173,19 +155,19 @@ export class TimelineComponent * underlying data. */ onZoomIn(chart, triggeredByDataUpdate = false) { - const initialTimeCategory = this.data.calculateTimeCategory(...this.xDomain); - const previousTimeCategory = this.data.currentTimeCategory; + const initialTimeCategory = this.data.timeCategory; + const previousTimeCategory = this.viewTimeCategory; const min = new Date(chart.scales.x.min); const max = new Date(chart.scales.x.max); - this.data.currentTimeCategory = this.data.calculateTimeCategory(min, max); + this.viewTimeCategory = this.data.calculateTimeCategory(min, max); if ( - this.data.currentTimeCategory !== previousTimeCategory || + this.viewTimeCategory !== previousTimeCategory || (triggeredByDataUpdate && - this.data.currentTimeCategory !== initialTimeCategory) + this.viewTimeCategory !== initialTimeCategory) ) { showLoading( - this.isLoading$, + this.zoomedDataLoading$, this.loadZoomedInData( chart, min, @@ -223,7 +205,7 @@ export class TimelineComponent this.chartDataFromSeries(data); }); - chart.options.scales.x.time.unit = this.currentTimeCategory; + chart.options.scales.x.time.unit = this.viewTimeCategory; chart.update('show'); // fade into view } @@ -242,9 +224,9 @@ export class TimelineComponent /** trigger zoom out, update chart data accordingly */ zoomOut(): void { this.chart.resetZoom(); - this.currentTimeCategory = this.data.calculateTimeCategory(...this.xDomain); + this.viewTimeCategory = this.data.timeCategory; (this.chart.options.scales.x as any).time.unit = - this.currentTimeCategory; + this.viewTimeCategory; this.chart.update(); this.setChart(); @@ -320,7 +302,7 @@ export class TimelineComponent // eslint-disable-next-line @typescript-eslint/member-ordering get formatDate(): (date) => string { let dateFormat: string; - switch (this.currentTimeCategory) { + switch (this.viewTimeCategory) { case 'year': dateFormat = 'yyyy'; break; From a581463cebfb33dfdb5f7581c967d81a35971074 Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 15:07:53 +0100 Subject: [PATCH 05/19] close subscriptions on refresh --- .../barchart/barchart.directive.ts | 29 +++++++++++++------ .../visualization/barchart/results-count.ts | 16 +++++----- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index 3e996e1b2..b37623e0c 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -16,7 +16,7 @@ import { ChartParameters, } from '@models'; import Zoom from 'chartjs-plugin-zoom'; -import { Subject } from 'rxjs'; +import { Subject, takeUntil } from 'rxjs'; import { selectColor } from '@utils/select-color'; import { VisualizationService } from '@services/visualization.service'; import { ComparedQueries } from '@models/compared-queries'; @@ -87,11 +87,10 @@ export abstract class BarchartDirective< // eslint-disable-next-line @angular-eslint/no-output-native @Output() error = new EventEmitter(); - destroy$ = new Subject(); - // stopPolling$ = new Subject(); - // tasksToCancel: string[]; + /** indicates that the data model should be re-instantiated */ + refresh$ = new Subject(); - // dataHasLoaded: boolean; + destroy$ = new Subject(); data: Data; @@ -175,6 +174,7 @@ export abstract class BarchartDirective< ngOnChanges(changes: SimpleChanges) { if (this.changesRequireRefresh(changes)) { + this.refresh$.next(); this.refreshChart(); } else if (changes.palette) { this.setChart(); @@ -183,6 +183,7 @@ export abstract class BarchartDirective< ngOnDestroy(): void { this.destroy$.next(undefined); + this.refresh$.complete(); this.destroy$.complete(); this.comparedQueries.complete(); } @@ -229,12 +230,22 @@ export abstract class BarchartDirective< * clear data and update chart */ refreshChart(): void { + // complete existing data model + this.data?.complete(); + + // create new data model and subscribe this.data = this.initData(); - this.data.rawData$.subscribe(data => this.onDataLoaded(data)); - this.data.error$.subscribe(message => this.onDataError(message)); - // this.initQueries(); + this.data.rawData$.pipe( + takeUntil(this.refresh$), + takeUntil(this.destroy$), + ).subscribe(data => this.onDataLoaded(data)); + this.data.error$.pipe( + takeUntil(this.refresh$), + takeUntil(this.destroy$), + ).subscribe(message => this.onDataError(message)); + + // clear canvas this.clearCanvas(); - // this.prepareChart(); } /** if a chart is active, clear canvas and reset chart object */ diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts index e5563fd27..895585d70 100644 --- a/frontend/src/app/visualization/barchart/results-count.ts +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -47,13 +47,21 @@ export abstract class BarchartData< takeUntil(this.destroy$) ).subscribe(this.loadData.bind(this)); this.comparedQueries.allQueries$.subscribe(this.updateQueries.bind(this)); - this.destroy$.subscribe(() => this.onDestroy()); + this.destroy$.subscribe(() => this.complete()); } get documentLimit(): number { return hasPrefixTerm(this.queryModel.queryText) ? documentLimitPrefixQueries : documentLimitBase; } + complete() { + this.stopPolling$.next(); + this.stopPolling$.complete(); + this.error$.complete(); + } + + + protected refresh() { this.initQueries(); this.loadData(); @@ -321,12 +329,6 @@ export abstract class BarchartData< return cat; } - private onDestroy() { - this.stopPolling$.next(); - this.stopPolling$.complete(); - this.error$.complete(); - } - abstract fullDataRequest(): Promise; /** Request doc counts for a series */ From 391762bd4576261530243efe8d4b344e38a64d1e Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 15:14:13 +0100 Subject: [PATCH 06/19] split data classes into separate files --- .../barchart/histogram.component.ts | 2 +- .../barchart/results-count-histogram.ts | 74 ++++++ .../barchart/results-count-timeline.ts | 166 ++++++++++++ .../visualization/barchart/results-count.ts | 237 +----------------- .../barchart/timeline.component.ts | 2 +- 5 files changed, 244 insertions(+), 237 deletions(-) create mode 100644 frontend/src/app/visualization/barchart/results-count-histogram.ts create mode 100644 frontend/src/app/visualization/barchart/results-count-timeline.ts diff --git a/frontend/src/app/visualization/barchart/histogram.component.ts b/frontend/src/app/visualization/barchart/histogram.component.ts index 0ea385cee..54a297597 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.ts +++ b/frontend/src/app/visualization/barchart/histogram.component.ts @@ -3,7 +3,7 @@ import * as _ from 'lodash'; import { selectColor } from '@utils/select-color'; import { BarchartDirective } from './barchart.directive'; -import { HistogramData } from './results-count'; +import { HistogramData } from './results-count-histogram'; function formatXAxisLabel(value): string { const label = this.getLabelForValue(value); // from chartJS api diff --git a/frontend/src/app/visualization/barchart/results-count-histogram.ts b/frontend/src/app/visualization/barchart/results-count-histogram.ts new file mode 100644 index 000000000..003cb18d5 --- /dev/null +++ b/frontend/src/app/visualization/barchart/results-count-histogram.ts @@ -0,0 +1,74 @@ +import { TermsAggregator, TermsResult } from '@models/aggregation'; +import { BarchartData } from './results-count'; +import { HistogramDataPoint, HistogramSeries, MultipleChoiceFilterOptions, QueryModel, RangeFilterOptions, TaskResult } from '@models'; +import _ from 'lodash'; + +export class HistogramData extends BarchartData { + fullDataRequest(): Promise { + const paramsPerSeries = this.rawData$.value.map((series) => { + const queryModel = this.queryModelForSeries( + series, + this.queryModel + ); + const bins = this.makeTermFrequencyBins(series); + return this.visualizationService.makeAggregateTermFrequencyParameters( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins + ); + }); + return this.apiService.requestFullData({ + visualization: 'aggregate_term_frequency', + parameters: paramsPerSeries, + corpus_name: this.queryModel.corpus.name, + }); + } + + /** specify aggregator object based on visualised field; + * used in document requests. + */ + protected getAggregator(): TermsAggregator { + let size = 100; + + const filterOptions = this.visualizedField.filterOptions; + if (filterOptions?.name === 'MultipleChoiceFilter') { + size = (filterOptions as MultipleChoiceFilterOptions).option_count; + } else if (filterOptions?.name === 'RangeFilter') { + const filterRange = + (filterOptions as RangeFilterOptions).upper - + (filterOptions as RangeFilterOptions).lower; + size = _.max([size, filterRange]) + } + return new TermsAggregator(this.visualizedField, size); + } + + protected requestSeriesDocCounts(queryModel: QueryModel) { + const aggregator = this.getAggregator(); + return this.searchService.aggregateSearch(this.queryModel.corpus, queryModel, aggregator); + } + + protected aggregateResultToDataPoint(cat: TermsResult) { + return cat; + } + + protected requestSeriesTermFrequency( + series: HistogramSeries, + queryModel: QueryModel + ) { + const bins = this.makeTermFrequencyBins(series); + return this.visualizationService.aggregateTermFrequencySearch( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins + ); + } + + private makeTermFrequencyBins(series: HistogramSeries) { + return series.data.map((bin) => ({ + fieldValue: bin.key, + size: this.documentLimitForCategory(bin, series), + })); + } +} diff --git a/frontend/src/app/visualization/barchart/results-count-timeline.ts b/frontend/src/app/visualization/barchart/results-count-timeline.ts new file mode 100644 index 000000000..9f3f5e636 --- /dev/null +++ b/frontend/src/app/visualization/barchart/results-count-timeline.ts @@ -0,0 +1,166 @@ +import { BarchartData } from './results-count'; +import { BarchartSeries, DateFilterData, QueryModel, TimeCategory, TimelineDataPoint, TimelineSeries } from '@models'; +import { DateHistogramAggregator, DateHistogramResult } from '@models/aggregation'; +import { differenceInDays } from 'date-fns'; + +export class TimelineData extends BarchartData { + xDomain: [Date, Date]; + timeCategory: TimeCategory = 'year'; + + /** threshold for scaling down a unit on the time scale */ + private scaleDownThreshold = 10; + + zoomedInData (min: Date, max: Date): Promise[]> { + const promises = this.rawData$.value.map(series => { + const queryModelCopy = this.addQueryDateFilter( + this.queryModel, + min, + max + ); + return this.getSeriesDocumentData( + series, + queryModelCopy, + false + ).then((result) => { + if (this.frequencyMeasure === 'tokens') { + return this.getTermFrequencies(result, queryModelCopy); + } else { + return result; + } + }); + }); + return Promise.all(promises); + } + + fullDataRequest() { + const paramsPerSeries = this.rawData$.value.map((series) => { + const queryModel = this.queryModelForSeries( + series, + this.queryModel + ); + const bins = this.makeTermFrequencyBins(series); + const unit = this.timeCategory; + return this.visualizationService.makeDateTermFrequencyParameters( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins, + unit + ); + }); + return this.apiService.requestFullData({ + visualization: 'date_term_frequency', + parameters: paramsPerSeries, + corpus_name: this.queryModel.corpus.name, + }); + } + + + /** + * Get the time category (year/month/week/day) that should be used in the graph, + * based on minimum and maximum dates on the x axis. + */ + calculateTimeCategory(min: Date, max: Date): TimeCategory { + const diff = differenceInDays(max, min); + if (diff <= this.scaleDownThreshold) { + return 'day'; + } else if (diff <= this.scaleDownThreshold * 7) { + return 'week'; + } else if (diff <= this.scaleDownThreshold * 30) { + return 'month'; + } else { + return 'year'; + } + } + + + protected refresh(): void { + this.setTimeDomain(); + super.refresh(); + } + + protected aggregateResultToDataPoint(cat: DateHistogramResult): TimelineDataPoint { + /* date fields are returned with keys containing identifiers by elasticsearch + replace with string representation, contained in 'key_as_string' field + */ + return { + date: new Date(cat.key_as_string), + doc_count: cat.doc_count, + }; + } + + /** Retrieve doc counts for a series. + * + * @param series series object + * @param setSearchRatio whether the `searchRatio` property of the series should be updated. + * True when retrieving results for the entire series, false when retrieving a window. + */ + protected requestSeriesDocCounts(queryModel: QueryModel) { + const aggregation = new DateHistogramAggregator( + this.visualizedField, + this.timeCategory + ); + return this.searchService.aggregateSearch(this.queryModel.corpus, queryModel, aggregation); + } + + protected requestSeriesTermFrequency(series: TimelineSeries, queryModel: QueryModel) { + const bins = this.makeTermFrequencyBins(series); + return this.visualizationService.dateTermFrequencySearch( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins, + this.timeCategory + ); + } + + + /** + * Add a date filter to a query model restricting it to the provided min and max values. + */ + private addQueryDateFilter(query: QueryModel, min: Date, max: Date): QueryModel { + const queryModelCopy = query.clone(); + // download zoomed in results + const filter = this.visualizedField.makeSearchFilter(); + filter.set({ min, max }); + queryModelCopy.addFilter(filter); + return queryModelCopy; + } + + private makeTermFrequencyBins(series: TimelineSeries) { + return series.data.map((bin, index) => { + const [minDate, maxDate] = this.categoryTimeDomain( + bin, + index, + series + ); + return { + start_date: minDate, + end_date: maxDate, + size: this.documentLimitForCategory(bin, series), + }; + }); + } + + /** get min/max date for the entire graph and set domain and time category */ + private setTimeDomain() { + const filter = this.queryModel.filterForField(this.visualizedField) + || this.visualizedField.makeSearchFilter(); + const currentDomain = filter.currentData as DateFilterData; + const min = new Date(currentDomain.min); + const max = new Date(currentDomain.max); + this.xDomain = [min, max]; + this.timeCategory = this.calculateTimeCategory(min, max); + } + + /** time domain for a bin */ + private categoryTimeDomain(cat, catIndex, series): [Date, Date] { + const startDate = cat.date; + const endDate = + catIndex < series.data.length - 1 + ? series.data[catIndex + 1].date + : undefined; + return [startDate, endDate]; + } + +} diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts index 895585d70..c04beb980 100644 --- a/frontend/src/app/visualization/barchart/results-count.ts +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -1,5 +1,5 @@ -import { BarchartSeries, CorpusField, DateFilterData, HistogramDataPoint, HistogramSeries, MultipleChoiceFilterOptions, QueryModel, RangeFilterOptions, TaskResult, TermFrequencyResult, TimeCategory, TimelineDataPoint, TimelineSeries } from '@models'; -import { DateHistogramAggregator, DateHistogramResult, TermsAggregator, TermsResult } from '@models/aggregation'; +import { BarchartSeries, CorpusField, HistogramDataPoint, QueryModel, TaskResult, TermFrequencyResult, TimelineDataPoint } from '@models'; +import { DateHistogramResult, TermsResult } from '@models/aggregation'; import { ComparedQueries } from '@models/compared-queries'; import { ApiService, SearchService, VisualizationService } from '@services'; @@ -7,7 +7,6 @@ import _ from 'lodash'; import { hasPrefixTerm } from './query-utils'; import { BehaviorSubject, Observable, Subject, takeUntil } from 'rxjs'; import { showLoading } from '@utils/utils'; -import { differenceInDays } from 'date-fns'; export type FrequencyMeasure = 'documents' | 'tokens'; @@ -343,235 +342,3 @@ export abstract class BarchartData< protected abstract requestSeriesTermFrequency(series: BarchartSeries, queryModel: QueryModel): Promise; } - -export class HistogramData extends BarchartData { - fullDataRequest(): Promise { - const paramsPerSeries = this.rawData$.value.map((series) => { - const queryModel = this.queryModelForSeries( - series, - this.queryModel - ); - const bins = this.makeTermFrequencyBins(series); - return this.visualizationService.makeAggregateTermFrequencyParameters( - this.queryModel.corpus, - queryModel, - this.visualizedField.name, - bins - ); - }); - return this.apiService.requestFullData({ - visualization: 'aggregate_term_frequency', - parameters: paramsPerSeries, - corpus_name: this.queryModel.corpus.name, - }); - } - - /** specify aggregator object based on visualised field; - * used in document requests. - */ - protected getAggregator(): TermsAggregator { - let size = 100; - - const filterOptions = this.visualizedField.filterOptions; - if (filterOptions?.name === 'MultipleChoiceFilter') { - size = (filterOptions as MultipleChoiceFilterOptions).option_count; - } else if (filterOptions?.name === 'RangeFilter') { - const filterRange = - (filterOptions as RangeFilterOptions).upper - - (filterOptions as RangeFilterOptions).lower; - size = _.max([size, filterRange]) - } - return new TermsAggregator(this.visualizedField, size); - } - - protected requestSeriesDocCounts(queryModel: QueryModel) { - const aggregator = this.getAggregator(); - return this.searchService.aggregateSearch(this.queryModel.corpus, queryModel, aggregator); - } - - protected aggregateResultToDataPoint(cat: TermsResult) { - return cat; - } - - protected requestSeriesTermFrequency( - series: HistogramSeries, - queryModel: QueryModel - ) { - const bins = this.makeTermFrequencyBins(series); - return this.visualizationService.aggregateTermFrequencySearch( - this.queryModel.corpus, - queryModel, - this.visualizedField.name, - bins - ); - } - - private makeTermFrequencyBins(series: HistogramSeries) { - return series.data.map((bin) => ({ - fieldValue: bin.key, - size: this.documentLimitForCategory(bin, series), - })); - } -} - -export class TimelineData extends BarchartData { - xDomain: [Date, Date]; - timeCategory: TimeCategory = 'year'; - - /** threshold for scaling down a unit on the time scale */ - private scaleDownThreshold = 10; - - zoomedInData (min: Date, max: Date): Promise[]> { - const promises = this.rawData$.value.map(series => { - const queryModelCopy = this.addQueryDateFilter( - this.queryModel, - min, - max - ); - return this.getSeriesDocumentData( - series, - queryModelCopy, - false - ).then((result) => { - if (this.frequencyMeasure === 'tokens') { - return this.getTermFrequencies(result, queryModelCopy); - } else { - return result; - } - }); - }); - return Promise.all(promises); - } - - fullDataRequest() { - const paramsPerSeries = this.rawData$.value.map((series) => { - const queryModel = this.queryModelForSeries( - series, - this.queryModel - ); - const bins = this.makeTermFrequencyBins(series); - const unit = this.timeCategory; - return this.visualizationService.makeDateTermFrequencyParameters( - this.queryModel.corpus, - queryModel, - this.visualizedField.name, - bins, - unit - ); - }); - return this.apiService.requestFullData({ - visualization: 'date_term_frequency', - parameters: paramsPerSeries, - corpus_name: this.queryModel.corpus.name, - }); - } - - - /** - * Get the time category (year/month/week/day) that should be used in the graph, - * based on minimum and maximum dates on the x axis. - */ - calculateTimeCategory(min: Date, max: Date): TimeCategory { - const diff = differenceInDays(max, min); - if (diff <= this.scaleDownThreshold) { - return 'day'; - } else if (diff <= this.scaleDownThreshold * 7) { - return 'week'; - } else if (diff <= this.scaleDownThreshold * 30) { - return 'month'; - } else { - return 'year'; - } - } - - - protected refresh(): void { - this.setTimeDomain(); - super.refresh(); - } - - protected aggregateResultToDataPoint(cat: DateHistogramResult): TimelineDataPoint { - /* date fields are returned with keys containing identifiers by elasticsearch - replace with string representation, contained in 'key_as_string' field - */ - return { - date: new Date(cat.key_as_string), - doc_count: cat.doc_count, - }; - } - - /** Retrieve doc counts for a series. - * - * @param series series object - * @param setSearchRatio whether the `searchRatio` property of the series should be updated. - * True when retrieving results for the entire series, false when retrieving a window. - */ - protected requestSeriesDocCounts(queryModel: QueryModel) { - const aggregation = new DateHistogramAggregator( - this.visualizedField, - this.timeCategory - ); - return this.searchService.aggregateSearch(this.queryModel.corpus, queryModel, aggregation); - } - - protected requestSeriesTermFrequency(series: TimelineSeries, queryModel: QueryModel) { - const bins = this.makeTermFrequencyBins(series); - return this.visualizationService.dateTermFrequencySearch( - this.queryModel.corpus, - queryModel, - this.visualizedField.name, - bins, - this.timeCategory - ); - } - - - /** - * Add a date filter to a query model restricting it to the provided min and max values. - */ - private addQueryDateFilter(query: QueryModel, min: Date, max: Date): QueryModel { - const queryModelCopy = query.clone(); - // download zoomed in results - const filter = this.visualizedField.makeSearchFilter(); - filter.set({ min, max }); - queryModelCopy.addFilter(filter); - return queryModelCopy; - } - - private makeTermFrequencyBins(series: TimelineSeries) { - return series.data.map((bin, index) => { - const [minDate, maxDate] = this.categoryTimeDomain( - bin, - index, - series - ); - return { - start_date: minDate, - end_date: maxDate, - size: this.documentLimitForCategory(bin, series), - }; - }); - } - - /** get min/max date for the entire graph and set domain and time category */ - private setTimeDomain() { - const filter = this.queryModel.filterForField(this.visualizedField) - || this.visualizedField.makeSearchFilter(); - const currentDomain = filter.currentData as DateFilterData; - const min = new Date(currentDomain.min); - const max = new Date(currentDomain.max); - this.xDomain = [min, max]; - this.timeCategory = this.calculateTimeCategory(min, max); - } - - /** time domain for a bin */ - private categoryTimeDomain(cat, catIndex, series): [Date, Date] { - const startDate = cat.date; - const endDate = - catIndex < series.data.length - 1 - ? series.data[catIndex + 1].date - : undefined; - return [startDate, endDate]; - } - -} diff --git a/frontend/src/app/visualization/barchart/timeline.component.ts b/frontend/src/app/visualization/barchart/timeline.component.ts index 8f591cdfe..45e45c264 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.ts +++ b/frontend/src/app/visualization/barchart/timeline.component.ts @@ -13,7 +13,7 @@ import { BarchartDirective } from './barchart.directive'; import { selectColor } from '@utils/select-color'; import { showLoading } from '@utils/utils'; -import { TimelineData } from './results-count'; +import { TimelineData } from './results-count-timeline'; import { BehaviorSubject } from 'rxjs'; From 4c4f339101fab9c7dd3b47261155fcb62d94bff0 Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 15:24:26 +0100 Subject: [PATCH 07/19] formatting --- .../barchart/results-count-histogram.ts | 5 +--- .../barchart/results-count-timeline.ts | 23 +++-------------- .../visualization/barchart/results-count.ts | 25 ++++++++----------- 3 files changed, 15 insertions(+), 38 deletions(-) diff --git a/frontend/src/app/visualization/barchart/results-count-histogram.ts b/frontend/src/app/visualization/barchart/results-count-histogram.ts index 003cb18d5..4179593af 100644 --- a/frontend/src/app/visualization/barchart/results-count-histogram.ts +++ b/frontend/src/app/visualization/barchart/results-count-histogram.ts @@ -6,10 +6,7 @@ import _ from 'lodash'; export class HistogramData extends BarchartData { fullDataRequest(): Promise { const paramsPerSeries = this.rawData$.value.map((series) => { - const queryModel = this.queryModelForSeries( - series, - this.queryModel - ); + const queryModel = this.queryModelForSeries(series, this.queryModel); const bins = this.makeTermFrequencyBins(series); return this.visualizationService.makeAggregateTermFrequencyParameters( this.queryModel.corpus, diff --git a/frontend/src/app/visualization/barchart/results-count-timeline.ts b/frontend/src/app/visualization/barchart/results-count-timeline.ts index 9f3f5e636..dc6854e51 100644 --- a/frontend/src/app/visualization/barchart/results-count-timeline.ts +++ b/frontend/src/app/visualization/barchart/results-count-timeline.ts @@ -12,15 +12,9 @@ export class TimelineData extends BarchartData[]> { const promises = this.rawData$.value.map(series => { - const queryModelCopy = this.addQueryDateFilter( - this.queryModel, - min, - max - ); + const queryModelCopy = this.addQueryDateFilter(this.queryModel, min, max); return this.getSeriesDocumentData( - series, - queryModelCopy, - false + series, queryModelCopy, false ).then((result) => { if (this.frequencyMeasure === 'tokens') { return this.getTermFrequencies(result, queryModelCopy); @@ -34,10 +28,7 @@ export class TimelineData extends BarchartData { - const queryModel = this.queryModelForSeries( - series, - this.queryModel - ); + const queryModel = this.queryModelForSeries(series, this.queryModel); const bins = this.makeTermFrequencyBins(series); const unit = this.timeCategory; return this.visualizationService.makeDateTermFrequencyParameters( @@ -55,7 +46,6 @@ export class TimelineData extends BarchartData { const [minDate, maxDate] = this.categoryTimeDomain( - bin, - index, - series + bin, index, series ); return { start_date: minDate, @@ -162,5 +148,4 @@ export class TimelineData extends BarchartData> { const queryModelCopy = this.queryModelForSeries(series, queryModel); - return this.requestSeriesDocCounts(queryModelCopy).then((result) => this.docCountResultIntoSeries(result, series, setSearchRatio) ); @@ -168,11 +165,10 @@ export abstract class BarchartData< private loadData(): Promise { const dataPromises = this.requestDocumentData().then( - this.frequencyMeasure === 'tokens' - ? this.requestTermFrequencyData.bind(this) - : _.identity - ) - .then((rawData) => { + this.frequencyMeasure === 'tokens' + ? this.requestTermFrequencyData.bind(this) + : _.identity + ).then((rawData) => { this.rawData$.next(rawData); if (!rawData.length) { @@ -184,8 +180,8 @@ export abstract class BarchartData< private requestDocumentData(): Promise[]> { const dataPromises = this.rawData$.value.map((series) => { + // retrieve data if it was not already loaded if (!series.data.length) { - // retrieve data if it was not already loaded return this.getSeriesDocumentData(series); } else { return series; @@ -236,7 +232,7 @@ export abstract class BarchartData< field.searchable && field.displayType === 'text_content' ); const queryModelCopy = queryModel.clone(); - queryModelCopy.setParams({searchFields: mainContentFields}); + queryModelCopy.setParams({ searchFields: mainContentFields }); return queryModelCopy; } } @@ -288,11 +284,10 @@ export abstract class BarchartData< /** fill in the `relative_doc_count` property for an array of datapoints. */ private includeRelativeDocCount(data: DataPoint[], total: number): DataPoint[] { - return data.map((item) => { - const result = _.clone(item); - result.relative_doc_count = result.doc_count / total; - return result; - }); + return data.map((item) => ({ + ...item, + relative_doc_count: item.doc_count / total, + })); } private onFailure(error) { From 5eb6a116a01049be11be987eabf11bced7437347 Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 15:25:53 +0100 Subject: [PATCH 08/19] complete more subjects --- frontend/src/app/visualization/barchart/results-count.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts index 6caecb4bf..f98667bbc 100644 --- a/frontend/src/app/visualization/barchart/results-count.ts +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -54,9 +54,11 @@ export abstract class BarchartData< } complete() { + this.rawData$.complete(); this.stopPolling$.next(); this.stopPolling$.complete(); this.error$.complete(); + this.loading$.complete(); } protected refresh() { From 3da8a261fff6125f5f5f6acdf2302903ebc1bce8 Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 15:41:24 +0100 Subject: [PATCH 09/19] fix query update refresh --- .../app/visualization/barchart/results-count.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts index f98667bbc..4f86d56e8 100644 --- a/frontend/src/app/visualization/barchart/results-count.ts +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -19,7 +19,7 @@ export abstract class BarchartData< DataPoint extends TimelineDataPoint | HistogramDataPoint > { // rawData: a list of series - rawData$: BehaviorSubject[]>; + rawData$ = new BehaviorSubject[]>([]); documentLimitExceeded = false; // whether the results include documents than the limit totalTokenCountAvailable: boolean; // whether the data includes token count totals dataHasLoaded: boolean; @@ -30,7 +30,6 @@ export abstract class BarchartData< error$ = new Subject(); stopPolling$ = new Subject(); - constructor( protected queryModel: QueryModel, private comparedQueries: ComparedQueries, @@ -44,7 +43,7 @@ export abstract class BarchartData< this.refresh(); this.queryModel.update.pipe( takeUntil(this.destroy$) - ).subscribe(this.loadData.bind(this)); + ).subscribe(this.refresh.bind(this)); this.comparedQueries.allQueries$.subscribe(this.updateQueries.bind(this)); this.destroy$.subscribe(() => this.complete()); } @@ -136,7 +135,7 @@ export abstract class BarchartData< this.newSeries(this.queryModel.queryText), ...this.comparedQueries.state$.value.compare.map(this.newSeries) ]; - this.rawData$ = new BehaviorSubject(series); + this.rawData$.next(series) } /** update the queries in the graph to the input array. Preserve results if possible, @@ -293,10 +292,12 @@ export abstract class BarchartData< } private onFailure(error) { - this.error$.next(`could not load results: ${error.message}`); + this.error$.next(error.message); } - private processSeriesTermFrequency(results: TermFrequencyResult[], series: BarchartSeries): BarchartSeries { + private processSeriesTermFrequency( + results: TermFrequencyResult[], series: BarchartSeries + ): BarchartSeries { this.dataHasLoaded = true; series.data = _.zip(series.data, results).map(pair => { const [bin, res] = pair; @@ -312,8 +313,7 @@ export abstract class BarchartData< * @param cat DataPoint object where the data should be added */ private addTermFrequencyToCategory( - data: TermFrequencyResult, - cat: DataPoint + data: TermFrequencyResult, cat: DataPoint ): DataPoint { cat.match_count = data.match_count; cat.total_doc_count = data.total_doc_count; From 0425a36a0f1ee897adfccea9e18c49558aa8cecb Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 15:57:31 +0100 Subject: [PATCH 10/19] fix loading being triggered twice --- frontend/src/app/models/compared-queries.ts | 9 ++++++++- frontend/src/app/visualization/barchart/results-count.ts | 9 +++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/models/compared-queries.ts b/frontend/src/app/models/compared-queries.ts index 79d6f6059..454364d75 100644 --- a/frontend/src/app/models/compared-queries.ts +++ b/frontend/src/app/models/compared-queries.ts @@ -4,7 +4,7 @@ import * as _ from 'lodash'; import { Store } from '../store/types'; import { Observable } from 'rxjs'; import { queryFromParams, queryToParams } from '../utils/params'; -import { filter, map, takeUntil } from 'rxjs/operators'; +import { distinctUntilChanged, distinctUntilKeyChanged, filter, map, takeUntil } from 'rxjs/operators'; interface CompareQueryState { primary: string|undefined; @@ -18,6 +18,9 @@ export class ComparedQueries extends StoreSync { /** all queries, including the primary query text */ allQueries$: Observable; + /** compared queries, excluding the primary query text */ + comparedQueries$: Observable; + protected keysInStore = ['query', COMPARE_TERMS_KEY]; constructor(store: Store) { @@ -29,6 +32,10 @@ export class ComparedQueries extends StoreSync { this.allQueries$ = this.state$.pipe( map(state => [state.primary, ...state.compare]) ); + this.comparedQueries$ = this.state$.pipe( + distinctUntilKeyChanged('compare', _.isEqual), + map(state => state.compare), + ); } reset() { diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts index 4f86d56e8..4776c5d0c 100644 --- a/frontend/src/app/visualization/barchart/results-count.ts +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -5,7 +5,7 @@ import { ComparedQueries } from '@models/compared-queries'; import { ApiService, SearchService, VisualizationService } from '@services'; import _ from 'lodash'; import { hasPrefixTerm } from './query-utils'; -import { BehaviorSubject, Observable, Subject, takeUntil } from 'rxjs'; +import { BehaviorSubject, map, Observable, skip, Subject, takeUntil, withLatestFrom } from 'rxjs'; import { showLoading } from '@utils/utils'; export type FrequencyMeasure = 'documents' | 'tokens'; @@ -44,7 +44,12 @@ export abstract class BarchartData< this.queryModel.update.pipe( takeUntil(this.destroy$) ).subscribe(this.refresh.bind(this)); - this.comparedQueries.allQueries$.subscribe(this.updateQueries.bind(this)); + this.comparedQueries.comparedQueries$.pipe( + skip(1), + withLatestFrom(this.comparedQueries.allQueries$), + map(([compared, all]) => all), + takeUntil(this.destroy$), + ).subscribe(this.updateQueries.bind(this)); this.destroy$.subscribe(() => this.complete()); } From 16389fd7c65859a1264ba4fb097d37c4b828029f Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 16:11:27 +0100 Subject: [PATCH 11/19] use observable for total token count available --- .../app/visualization/barchart/barchart.directive.ts | 7 ++----- .../visualization/barchart/histogram.component.html | 4 +++- .../src/app/visualization/barchart/results-count.ts | 11 ++++++++++- .../visualization/barchart/timeline.component.html | 4 +++- .../app/visualization/visualization.component.html | 8 ++++---- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index b37623e0c..a950c02ed 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -22,7 +22,7 @@ import { VisualizationService } from '@services/visualization.service'; import { ComparedQueries } from '@models/compared-queries'; import { RouterStoreService } from '../../store/router-store.service'; import { hasPrefixTerm } from './query-utils'; -import { BarchartData } from './results-count'; +import { BarchartData, FrequencyMeasure } from './results-count'; const hintSeenSessionStorageKey = 'hasSeenTimelineZoomingHint'; const hintHidingMinDelay = 500; // milliseconds @@ -53,15 +53,12 @@ export abstract class BarchartDirective< @Input() asTable: boolean; @Input() palette: string[]; - @Input() frequencyMeasure: 'documents' | 'tokens' = 'documents'; + @Input() frequencyMeasure: FrequencyMeasure = 'documents'; normalizer: 'raw' | 'percent' | 'documents' | 'terms' = 'raw'; chartType: 'bar' | 'line' | 'scatter' = 'bar'; comparedQueries: ComparedQueries; - // documentLimitExceeded = false; // whether the results include documents than the limit - // totalTokenCountAvailable: boolean; // whether the data includes token count totals - // table data tableHeaders: FreqTableHeaders; tableData: any[]; diff --git a/frontend/src/app/visualization/barchart/histogram.component.html b/frontend/src/app/visualization/barchart/histogram.component.html index 25054a79d..a53b4454a 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.html +++ b/frontend/src/app/visualization/barchart/histogram.component.html @@ -6,7 +6,9 @@
diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/results-count.ts index 4776c5d0c..581566748 100644 --- a/frontend/src/app/visualization/barchart/results-count.ts +++ b/frontend/src/app/visualization/barchart/results-count.ts @@ -21,7 +21,6 @@ export abstract class BarchartData< // rawData: a list of series rawData$ = new BehaviorSubject[]>([]); documentLimitExceeded = false; // whether the results include documents than the limit - totalTokenCountAvailable: boolean; // whether the data includes token count totals dataHasLoaded: boolean; tasksToCancel: string[]; @@ -30,6 +29,10 @@ export abstract class BarchartData< error$ = new Subject(); stopPolling$ = new Subject(); + totalTokenCountAvailable$ = this.rawData$.pipe( + map(this.checkTotalTokenCount) + ); + constructor( protected queryModel: QueryModel, private comparedQueries: ComparedQueries, @@ -330,6 +333,12 @@ export abstract class BarchartData< return cat; } + private checkTotalTokenCount(data: BarchartSeries[]): boolean { + return data.find((series) => + series.data.find((cat) => cat.token_count !== undefined) + ) !== undefined; + } + abstract fullDataRequest(): Promise; /** Request doc counts for a series */ diff --git a/frontend/src/app/visualization/barchart/timeline.component.html b/frontend/src/app/visualization/barchart/timeline.component.html index d158323d0..049bd15fd 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.html +++ b/frontend/src/app/visualization/barchart/timeline.component.html @@ -10,7 +10,9 @@
diff --git a/frontend/src/app/visualization/visualization.component.html b/frontend/src/app/visualization/visualization.component.html index 7f697059a..708528253 100644 --- a/frontend/src/app/visualization/visualization.component.html +++ b/frontend/src/app/visualization/visualization.component.html @@ -95,23 +95,23 @@ [visualizedField]="visualizedField" [frequencyMeasure]="'documents'" [asTable]="freqtable" [palette]="palette" (error)="setErrorMessage($event)" - (totalTokenCountAvailable)="showTokenCountOption = $event"> + /> + /> + /> + /> Date: Wed, 29 Oct 2025 16:42:10 +0100 Subject: [PATCH 12/19] pass data object to methods --- .../barchart/barchart.directive.ts | 46 ++++++++++--------- .../barchart/histogram.component.ts | 18 ++++---- .../barchart/timeline.component.ts | 27 ++++------- 3 files changed, 41 insertions(+), 50 deletions(-) diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index a950c02ed..51b7df8f2 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -16,7 +16,7 @@ import { ChartParameters, } from '@models'; import Zoom from 'chartjs-plugin-zoom'; -import { Subject, takeUntil } from 'rxjs'; +import { skip, Subject, takeUntil } from 'rxjs'; import { selectColor } from '@utils/select-color'; import { VisualizationService } from '@services/visualization.service'; import { ComparedQueries } from '@models/compared-queries'; @@ -174,11 +174,12 @@ export abstract class BarchartDirective< this.refresh$.next(); this.refreshChart(); } else if (changes.palette) { - this.setChart(); + this.setChart(this.data.rawData$.value); } } ngOnDestroy(): void { + this.clearCanvas(); this.destroy$.next(undefined); this.refresh$.complete(); this.destroy$.complete(); @@ -187,11 +188,11 @@ export abstract class BarchartDirective< onDataLoaded(data: typeof this.data.rawData$.value) { // initialise or update chart - this.setChart(); + this.setChart(data); // update freqtable - this.setTableHeaders(); - this.setTableData(); + this.setTableHeaders(data); + this.setTableData(data); // load zoomed-in data if needed if (this.isZoomedIn) { @@ -227,6 +228,9 @@ export abstract class BarchartDirective< * clear data and update chart */ refreshChart(): void { + // clear canvas + this.clearCanvas(); + // complete existing data model this.data?.complete(); @@ -241,8 +245,6 @@ export abstract class BarchartDirective< takeUntil(this.destroy$), ).subscribe(message => this.onDataError(message)); - // clear canvas - this.clearCanvas(); } /** if a chart is active, clear canvas and reset chart object */ @@ -295,15 +297,15 @@ export abstract class BarchartDirective< } /** update or initialise chart (should be ran after updates to `rawData`) */ - setChart(): void { + setChart(data: typeof this.data.rawData$.value): void { if (this.chart) { - this.updateChartData(); + this.updateChartData(data); } else { - this.initChart(); + this.initChart(data); } } /** select the columns/headers for the frequency table */ - abstract setTableHeaders(): void; + abstract setTableHeaders(data: typeof this.data.rawData$.value): void; /** code to be executed when zooming in, or when parameters are updated while zoomed in */ onZoomIn(chart, triggeredByDataUpdate = false) {} /** options for the chart. @@ -313,9 +315,9 @@ export abstract class BarchartDirective< abstract chartOptions(datasets); /** initalise a new chart */ - initChart() { - const labels = this.getLabels(); - const datasets = this.getDatasets(); + initChart(data: typeof this.data.rawData$.value) { + const labels = this.getLabels(data); + const datasets = this.getDatasets(data); const options = this.chartOptions(datasets); this.chart = new Chart(barchartID, { @@ -337,9 +339,9 @@ export abstract class BarchartDirective< } /** After updating `rawData`, this executes the update in the chart. */ - updateChartData() { - const labels = this.getLabels(); - const datasets = this.getDatasets(); + updateChartData(data: typeof this.data.rawData$.value) { + const labels = this.getLabels(data); + const datasets = this.getDatasets(data); this.chart.options = this.chartOptions(datasets); this.chart.data.labels = labels; this.chart.data.datasets = datasets; @@ -350,12 +352,12 @@ export abstract class BarchartDirective< /** Return x-axis labels for the chartJS dataset. * Can be left undefined depending on data format. */ - getLabels(): string[] { + getLabels(data: typeof this.data.rawData$.value): string[] { return undefined; } /** return chartJS dataset objects based on rawData */ - abstract getDatasets(): any[]; + abstract getDatasets(data: typeof this.data.rawData$.value): any[]; /** * Show the zooming hint once per session, hide automatically with a delay @@ -410,9 +412,9 @@ export abstract class BarchartDirective< } /** assemble the array of table data */ - setTableData() { - if (this.data.rawData$.value && this.data.rawData$.value.length) { - this.tableData = _.flatMap(this.data.rawData$.value, (series) => + setTableData(data: typeof this.data.rawData$.value) { + if (data && data.length) { + this.tableData = _.flatMap(data, (series) => series.data.map((item) => { const result = _.cloneDeep(item) as any; result.queryText = series.queryText; diff --git a/frontend/src/app/visualization/barchart/histogram.component.ts b/frontend/src/app/visualization/barchart/histogram.component.ts index 54a297597..7bf5757e0 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.ts +++ b/frontend/src/app/visualization/barchart/histogram.component.ts @@ -44,11 +44,11 @@ export class HistogramComponent ); } - getLabels(): string[] { + getLabels(data: typeof this.data.rawData$.value): string[] { // make an array of all unique labels and sort - if (this.data.rawData$.value) { - const all_labels = _.flatMap(this.data.rawData$.value, (series) => + if (data) { + const all_labels = _.flatMap(data, (series) => series.data.map((item) => item.key) ); const labels = all_labels.filter( @@ -60,7 +60,7 @@ export class HistogramComponent } else { const valueKey = this.currentValueKey; sorted_labels = _.sortBy(labels, (label) => - _.sumBy(this.data.rawData$.value, (series) => { + _.sumBy(data, (series) => { const item = series.data.find((i) => i.key === label); return -1 * (item ? item[valueKey] : 0); }) @@ -70,11 +70,11 @@ export class HistogramComponent } } - getDatasets() { - const labels = this.getLabels(); + getDatasets(data: typeof this.data.rawData$.value) { + const labels = this.getLabels(data); const valueKey = this.currentValueKey; - return this.data.rawData$.value.map((series, seriesIndex) => ({ + return data.map((series, seriesIndex) => ({ type: this.chartType, label: series.queryText ? series.queryText : '(no query)', data: labels.map((key) => { @@ -109,7 +109,7 @@ export class HistogramComponent return options; } - setTableHeaders() { + setTableHeaders(data: typeof this.data.rawData$.value) { /* Provides the table headers to the freqTable component. Determines optional headers. */ @@ -120,7 +120,7 @@ export class HistogramComponent this.normalizer === 'raw' ? 'Frequency' : 'Relative frequency'; const valueKey = this.currentValueKey; - if (this.data.rawData$.value.length > 1) { + if (data.length > 1) { // if there are several queries, fulltable is disabled this.tableHeaders = [ { key: 'key', label, isSecondaryFactor: true }, diff --git a/frontend/src/app/visualization/barchart/timeline.component.ts b/frontend/src/app/visualization/barchart/timeline.component.ts index 45e45c264..6701e95fe 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.ts +++ b/frontend/src/app/visualization/barchart/timeline.component.ts @@ -48,22 +48,13 @@ export class TimelineComponent ); } - setChart() { - if (this.chart) { - // reset time unit to the one set in the chart - const unit = (this.chart.options.scales.x as any).time - .unit as TimeCategory; - if (unit) { - this.viewTimeCategory = unit; - } - this.updateChartData(); - } else { - this.initChart(); - } + setChart(data: typeof this.data.rawData$.value) { + this.viewTimeCategory = this.data.timeCategory; + super.setChart(data); } - getDatasets() { - return this.data.rawData$.value.map((series, seriesIndex) => { + getDatasets(data: typeof this.data.rawData$.value) { + return data.map((series, seriesIndex) => { const data = this.chartDataFromSeries(series); return { type: this.chartType, @@ -197,9 +188,7 @@ export class TimelineComponent // when zooming, hide data for smooth transition chart.update(triggeredByDataUpdate ? 'none' : 'hide'); - const zoomedInResults = await this.data.zoomedInData(min, max); - zoomedInResults.forEach((data, seriesIndex) => { chart.data.datasets[seriesIndex].data = this.chartDataFromSeries(data); @@ -229,15 +218,15 @@ export class TimelineComponent this.viewTimeCategory; this.chart.update(); - this.setChart(); + this.setChart(this.data.rawData$.value); } - setTableHeaders() { + setTableHeaders(data: typeof this.data.rawData$.value) { const rightColumnName = this.normalizer === 'raw' ? 'Frequency' : 'Relative frequency'; const valueKey = this.currentValueKey; - if (this.data.rawData$.value.length > 1) { + if (data.length > 1) { this.tableHeaders = [ { key: 'date', From 432f2afe0b8390c6c8dde4fc1eaa419ea1e75ba7 Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 17:00:31 +0100 Subject: [PATCH 13/19] skip first data event --- frontend/src/app/visualization/barchart/barchart.directive.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index 51b7df8f2..40637d22e 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -237,6 +237,7 @@ export abstract class BarchartDirective< // create new data model and subscribe this.data = this.initData(); this.data.rawData$.pipe( + skip(1), // skip the first (empty) event takeUntil(this.refresh$), takeUntil(this.destroy$), ).subscribe(data => this.onDataLoaded(data)); @@ -304,6 +305,7 @@ export abstract class BarchartDirective< this.initChart(data); } } + /** select the columns/headers for the frequency table */ abstract setTableHeaders(data: typeof this.data.rawData$.value): void; /** code to be executed when zooming in, or when parameters are updated while zoomed in */ From 7bcc4a00c6a6a0420ed75095efee42f8b9c02dda Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Wed, 29 Oct 2025 17:03:08 +0100 Subject: [PATCH 14/19] show loading spinner for freq table --- .../src/app/visualization/barchart/histogram.component.html | 2 +- frontend/src/app/visualization/barchart/timeline.component.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/visualization/barchart/histogram.component.html b/frontend/src/app/visualization/barchart/histogram.component.html index a53b4454a..8cfc016b9 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.html +++ b/frontend/src/app/visualization/barchart/histogram.component.html @@ -19,7 +19,7 @@
-
+
diff --git a/frontend/src/app/visualization/barchart/timeline.component.html b/frontend/src/app/visualization/barchart/timeline.component.html index 049bd15fd..98cb59bb0 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.html +++ b/frontend/src/app/visualization/barchart/timeline.component.html @@ -23,7 +23,7 @@
-
+
From 56099afb9e0fe4ca217301b3851c7bfd0cdc1d6b Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Thu, 30 Oct 2025 12:58:21 +0100 Subject: [PATCH 15/19] rename modules --- .../{results-count-histogram.ts => barchart-data-histogram.ts} | 2 +- .../{results-count-timeline.ts => barchart-data-timeline.ts} | 2 +- .../barchart/{results-count.ts => barchart-data.ts} | 0 frontend/src/app/visualization/barchart/barchart.directive.ts | 2 +- frontend/src/app/visualization/barchart/histogram.component.ts | 2 +- frontend/src/app/visualization/barchart/timeline.component.ts | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename frontend/src/app/visualization/barchart/{results-count-histogram.ts => barchart-data-histogram.ts} (98%) rename frontend/src/app/visualization/barchart/{results-count-timeline.ts => barchart-data-timeline.ts} (99%) rename frontend/src/app/visualization/barchart/{results-count.ts => barchart-data.ts} (100%) diff --git a/frontend/src/app/visualization/barchart/results-count-histogram.ts b/frontend/src/app/visualization/barchart/barchart-data-histogram.ts similarity index 98% rename from frontend/src/app/visualization/barchart/results-count-histogram.ts rename to frontend/src/app/visualization/barchart/barchart-data-histogram.ts index 4179593af..6a725d060 100644 --- a/frontend/src/app/visualization/barchart/results-count-histogram.ts +++ b/frontend/src/app/visualization/barchart/barchart-data-histogram.ts @@ -1,5 +1,5 @@ import { TermsAggregator, TermsResult } from '@models/aggregation'; -import { BarchartData } from './results-count'; +import { BarchartData } from './barchart-data'; import { HistogramDataPoint, HistogramSeries, MultipleChoiceFilterOptions, QueryModel, RangeFilterOptions, TaskResult } from '@models'; import _ from 'lodash'; diff --git a/frontend/src/app/visualization/barchart/results-count-timeline.ts b/frontend/src/app/visualization/barchart/barchart-data-timeline.ts similarity index 99% rename from frontend/src/app/visualization/barchart/results-count-timeline.ts rename to frontend/src/app/visualization/barchart/barchart-data-timeline.ts index dc6854e51..b10b085b1 100644 --- a/frontend/src/app/visualization/barchart/results-count-timeline.ts +++ b/frontend/src/app/visualization/barchart/barchart-data-timeline.ts @@ -1,4 +1,4 @@ -import { BarchartData } from './results-count'; +import { BarchartData } from './barchart-data'; import { BarchartSeries, DateFilterData, QueryModel, TimeCategory, TimelineDataPoint, TimelineSeries } from '@models'; import { DateHistogramAggregator, DateHistogramResult } from '@models/aggregation'; import { differenceInDays } from 'date-fns'; diff --git a/frontend/src/app/visualization/barchart/results-count.ts b/frontend/src/app/visualization/barchart/barchart-data.ts similarity index 100% rename from frontend/src/app/visualization/barchart/results-count.ts rename to frontend/src/app/visualization/barchart/barchart-data.ts diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index 40637d22e..d3456d634 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -22,7 +22,7 @@ import { VisualizationService } from '@services/visualization.service'; import { ComparedQueries } from '@models/compared-queries'; import { RouterStoreService } from '../../store/router-store.service'; import { hasPrefixTerm } from './query-utils'; -import { BarchartData, FrequencyMeasure } from './results-count'; +import { BarchartData, FrequencyMeasure } from './barchart-data'; const hintSeenSessionStorageKey = 'hasSeenTimelineZoomingHint'; const hintHidingMinDelay = 500; // milliseconds diff --git a/frontend/src/app/visualization/barchart/histogram.component.ts b/frontend/src/app/visualization/barchart/histogram.component.ts index 7bf5757e0..724e7d9f6 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.ts +++ b/frontend/src/app/visualization/barchart/histogram.component.ts @@ -3,7 +3,7 @@ import * as _ from 'lodash'; import { selectColor } from '@utils/select-color'; import { BarchartDirective } from './barchart.directive'; -import { HistogramData } from './results-count-histogram'; +import { HistogramData } from './barchart-data-histogram'; function formatXAxisLabel(value): string { const label = this.getLabelForValue(value); // from chartJS api diff --git a/frontend/src/app/visualization/barchart/timeline.component.ts b/frontend/src/app/visualization/barchart/timeline.component.ts index 6701e95fe..9c162d5ec 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.ts +++ b/frontend/src/app/visualization/barchart/timeline.component.ts @@ -13,7 +13,7 @@ import { BarchartDirective } from './barchart.directive'; import { selectColor } from '@utils/select-color'; import { showLoading } from '@utils/utils'; -import { TimelineData } from './results-count-timeline'; +import { TimelineData } from './barchart-data-timeline'; import { BehaviorSubject } from 'rxjs'; From 26d82854f669800f143ed60ff2328d331b8a79ca Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Thu, 30 Oct 2025 13:18:39 +0100 Subject: [PATCH 16/19] adjust tests --- .../barchart/histogram.component.spec.ts | 20 +++++++++++++------ .../barchart/timeline.component.spec.ts | 18 ++++++++++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/frontend/src/app/visualization/barchart/histogram.component.spec.ts b/frontend/src/app/visualization/barchart/histogram.component.spec.ts index a7fffb3e6..63bab550a 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.spec.ts +++ b/frontend/src/app/visualization/barchart/histogram.component.spec.ts @@ -5,6 +5,7 @@ import { commonTestBed } from '../../common-test-bed'; import { HistogramComponent } from './histogram.component'; import { corpusFactory, keywordFieldFactory } from '../../../mock-data/corpus'; +import { SimpleChange } from '@angular/core'; describe('HistogramCompoment', () => { let component: HistogramComponent; @@ -17,6 +18,13 @@ describe('HistogramCompoment', () => { beforeEach(() => { fixture = TestBed.createComponent(HistogramComponent); component = fixture.componentInstance; + + const corpus = corpusFactory(); + component.queryModel = new QueryModel(corpus); + component.visualizedField = corpus.fields.find(field => field.mappingType == 'keyword'); + component.ngOnChanges({ + queryModel: new SimpleChange(undefined, component.queryModel, true) + }); fixture.detectChanges(); }); @@ -26,18 +34,18 @@ describe('HistogramCompoment', () => { it('should filter text fields', () => { - component.corpus = corpusFactory(); - component.corpus.fields[0] = keywordFieldFactory(true); + const corpus = corpusFactory(); + corpus.fields[0] = keywordFieldFactory(true); component.frequencyMeasure = 'documents'; - const query1 = new QueryModel(component.corpus); + const query1 = new QueryModel(corpus); query1.setQueryText('test'); - const query2 = new QueryModel(component.corpus); + const query2 = new QueryModel(corpus); query2.setParams({ queryText: 'test', - searchFields: component.corpus.fields.slice(0, 2), + searchFields: corpus.fields.slice(0, 2), }); const queries = [ query1, query2 ]; @@ -51,7 +59,7 @@ describe('HistogramCompoment', () => { queries.forEach(query => { const newQuery = component.selectSearchFields(query); - expect(newQuery.searchFields).toEqual([component.corpus.fields[1]]); + expect(newQuery.searchFields).toEqual([corpus.fields[1]]); }); }); diff --git a/frontend/src/app/visualization/barchart/timeline.component.spec.ts b/frontend/src/app/visualization/barchart/timeline.component.spec.ts index 97e4eaf65..172a28bb1 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.spec.ts +++ b/frontend/src/app/visualization/barchart/timeline.component.spec.ts @@ -5,6 +5,7 @@ import { commonTestBed } from '../../common-test-bed'; import { TimelineComponent } from './timeline.component'; import { QueryModel } from '@models'; import { corpusFactory } from 'mock-data/corpus'; +import { SimpleChange } from '@angular/core'; describe('TimelineComponent', () => { let component: TimelineComponent; @@ -17,8 +18,12 @@ describe('TimelineComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(TimelineComponent); component = fixture.componentInstance; - component.queryModel = new QueryModel(corpusFactory()); - component.visualizedField = component.queryModel.corpus.fields[2]; + const corpus = corpusFactory(); + component.queryModel = new QueryModel(corpus); + component.visualizedField = corpus.fields.find(field => field.mappingType == 'date'); + component.ngOnChanges({ + queryModel: new SimpleChange(undefined, component.queryModel, true) + }); fixture.detectChanges(); }); @@ -27,14 +32,13 @@ describe('TimelineComponent', () => { }); it('should calculate correct time categories', () => { - expect(component.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-01-05'))).toEqual('day'); - expect(component.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-02-01'))).toEqual('week'); - expect(component.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-06-01'))).toEqual('month'); - expect(component.calculateTimeCategory(new Date('1950-01-01'), new Date('1951-01-01'))).toEqual('year'); + expect(component.data.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-01-05'))).toEqual('day'); + expect(component.data.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-02-01'))).toEqual('week'); + expect(component.data.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-06-01'))).toEqual('month'); + expect(component.data.calculateTimeCategory(new Date('1950-01-01'), new Date('1953-01-01'))).toEqual('year'); }) it('should calibrate the x axis with a margin', () => { - component.setTimeDomain(); const min = new Date('1802-01-01'); const max = new Date('1804-01-01'); expect(component.addDateMargin(min, -1)).toEqual(new Date('1801-01-01')); From 4fe4534036c809fd327dd50b1c3aff6fc50dfc9e Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Thu, 30 Oct 2025 13:20:31 +0100 Subject: [PATCH 17/19] remove unused function remove trivial get method --- .../app/visualization/barchart/barchart.directive.ts | 12 +++--------- .../visualization/barchart/histogram.component.html | 2 +- .../visualization/barchart/timeline.component.html | 2 +- .../app/visualization/barchart/timeline.component.ts | 12 ------------ 4 files changed, 5 insertions(+), 23 deletions(-) diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index d3456d634..6e7e739af 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -48,8 +48,8 @@ export abstract class BarchartDirective< // chart object chart: Chart; - @Input() queryModel: QueryModel; - @Input() visualizedField: CorpusField; + @Input({ required: true }) queryModel!: QueryModel; + @Input({ required: true }) visualizedField!: CorpusField; @Input() asTable: boolean; @Input() palette: string[]; @@ -468,7 +468,7 @@ export abstract class BarchartDirective< chartTitle() { const queryTexts = this.data.rawData$.value.map((series) => series.queryText); - if (this.queryText == null && this.data.rawData$.value.length === 1) { + if (!this.queryModel.queryText && this.data.rawData$.value.length === 1) { return `Frequency of documents by ${this.visualizedField.displayName} (n of ${this.frequencyMeasure}, ${this.normalizer})`; } else { const normalizationText = ['raw', 'percent'].includes( @@ -482,11 +482,5 @@ export abstract class BarchartDirective< } } - get queryText(): string { - if (this.queryModel) { - return this.queryModel.queryText; - } - } - abstract initData(): Data; } diff --git a/frontend/src/app/visualization/barchart/histogram.component.html b/frontend/src/app/visualization/barchart/histogram.component.html index 8cfc016b9..a5a20be5e 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.html +++ b/frontend/src/app/visualization/barchart/histogram.component.html @@ -6,7 +6,7 @@
Date: Thu, 30 Oct 2025 13:30:29 +0100 Subject: [PATCH 18/19] outfactor common function --- frontend/src/app/visualization/barchart/barchart-data.ts | 9 +++------ .../src/app/visualization/barchart/barchart.directive.ts | 9 ++------- frontend/src/app/visualization/barchart/query-utils.ts | 8 ++++++++ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/frontend/src/app/visualization/barchart/barchart-data.ts b/frontend/src/app/visualization/barchart/barchart-data.ts index 581566748..b36419360 100644 --- a/frontend/src/app/visualization/barchart/barchart-data.ts +++ b/frontend/src/app/visualization/barchart/barchart-data.ts @@ -4,7 +4,7 @@ import { DateHistogramResult, TermsResult } from '@models/aggregation'; import { ComparedQueries } from '@models/compared-queries'; import { ApiService, SearchService, VisualizationService } from '@services'; import _ from 'lodash'; -import { hasPrefixTerm } from './query-utils'; +import { hasPrefixTerm, mainContentFields } from './query-utils'; import { BehaviorSubject, map, Observable, skip, Subject, takeUntil, withLatestFrom } from 'rxjs'; import { showLoading } from '@utils/utils'; @@ -236,12 +236,9 @@ export abstract class BarchartData< if (this.frequencyMeasure === 'documents') { return queryModel; } else { - const mainContentFields = this.queryModel.corpus.fields.filter( - (field) => - field.searchable && field.displayType === 'text_content' - ); + const searchFields = mainContentFields(queryModel); const queryModelCopy = queryModel.clone(); - queryModelCopy.setParams({ searchFields: mainContentFields }); + queryModelCopy.setParams({ searchFields }); return queryModelCopy; } } diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index 6e7e739af..790d65389 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -21,7 +21,7 @@ import { selectColor } from '@utils/select-color'; import { VisualizationService } from '@services/visualization.service'; import { ComparedQueries } from '@models/compared-queries'; import { RouterStoreService } from '../../store/router-store.service'; -import { hasPrefixTerm } from './query-utils'; +import { hasPrefixTerm, mainContentFields } from './query-utils'; import { BarchartData, FrequencyMeasure } from './barchart-data'; const hintSeenSessionStorageKey = 'hasSeenTimelineZoomingHint'; @@ -454,15 +454,10 @@ export abstract class BarchartDirective< get searchFields(): string { if (this.queryModel) { - const searchFields = this.selectSearchFields( - this.queryModel - ).searchFields; - + const searchFields = mainContentFields(this.queryModel); const displayNames = searchFields.map((field) => field.displayName); - return displayNames.join(', '); } - return 'all fields'; } diff --git a/frontend/src/app/visualization/barchart/query-utils.ts b/frontend/src/app/visualization/barchart/query-utils.ts index cee72a8d9..6a740bc7d 100644 --- a/frontend/src/app/visualization/barchart/query-utils.ts +++ b/frontend/src/app/visualization/barchart/query-utils.ts @@ -1,3 +1,5 @@ +import { CorpusField, QueryModel } from '@models'; + /** Whether the query contains a prefix term * * The term frequency graph requires more resources for prefix terms so these should use @@ -7,3 +9,9 @@ */ export const hasPrefixTerm = (queryText: string): boolean => /\*/.test(queryText); + + +export const mainContentFields = (query: QueryModel): CorpusField[] => + query.corpus.fields.filter( + (field) => field.searchable && field.displayType === 'text_content' + ); From 0c814b0b64c2e0a2699bceb6ee65290df4fd2b61 Mon Sep 17 00:00:00 2001 From: Luka van der Plas Date: Thu, 30 Oct 2025 13:58:41 +0100 Subject: [PATCH 19/19] docstrings --- .../visualization/barchart/barchart-data.ts | 79 ++++++++++++------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/frontend/src/app/visualization/barchart/barchart-data.ts b/frontend/src/app/visualization/barchart/barchart-data.ts index b36419360..4e48b24a1 100644 --- a/frontend/src/app/visualization/barchart/barchart-data.ts +++ b/frontend/src/app/visualization/barchart/barchart-data.ts @@ -60,6 +60,7 @@ export abstract class BarchartData< return hasPrefixTerm(this.queryModel.queryText) ? documentLimitPrefixQueries : documentLimitBase; } + /** Complete observables */ complete() { this.rawData$.complete(); this.stopPolling$.next(); @@ -68,8 +69,9 @@ export abstract class BarchartData< this.loading$.complete(); } + /** Create a new data object and fetch results */ protected refresh() { - this.initQueries(); + this.initSeries(); this.loadData(); } @@ -85,6 +87,16 @@ export abstract class BarchartData< ); } + /** + * Get term frequency data for a data series + * + * Makes initial request for the series, polls result, merges frequency data with the + * series object. + + * @param series data series with document counts already filled in + * @param queryModel user query + * @returns promise of the series with term frequencies added + */ protected getTermFrequencies( series: BarchartSeries, queryModel: QueryModel @@ -115,7 +127,7 @@ export abstract class BarchartData< } /** - * calculate the maximum number of documents to read through in a bin + * Calculate the maximum number of documents to read through in a bin * when determining term frequency. */ protected documentLimitForCategory( @@ -128,7 +140,7 @@ export abstract class BarchartData< ]); } - /** adapt query model to fit series: use correct search fields and query text */ + /** Adapt query model to fit series: use correct search fields and query text */ protected queryModelForSeries( series: BarchartSeries, queryModel: QueryModel @@ -138,7 +150,8 @@ export abstract class BarchartData< ); } - private initQueries(): void { + /** Create new data object with empty frequencies */ + private initSeries(): void { const series = [ this.newSeries(this.queryModel.queryText), ...this.comparedQueries.state$.value.compare.map(this.newSeries) @@ -146,23 +159,21 @@ export abstract class BarchartData< this.rawData$.next(series) } - /** update the queries in the graph to the input array. Preserve results if possible, - * and start loading for the rest. + /** Update the data object to add/remove data series. If a query was already + * present, results are preserved. Starts loading results for new series. */ private updateQueries(queries: string[]) { - if (this.rawData$) { - const data = queries.map((queryText) => { - const existingSeries = this.rawData$.value.find( - (series) => series.queryText === queryText - ); - return existingSeries || this.newSeries(queryText); - }); - this.rawData$.next(data); - this.loadData(); - } + const data = queries.map((queryText) => { + const existingSeries = this.rawData$.value.find( + (series) => series.queryText === queryText + ); + return existingSeries || this.newSeries(queryText); + }); + this.rawData$.next(data); + this.loadData(); } - /** make a blank series object */ + /** Make a blank series object, without frequencies */ private newSeries(queryText: string): BarchartSeries { return { queryText, @@ -172,6 +183,7 @@ export abstract class BarchartData< }; } + /** Overarching function to load frequency data */ private loadData(): Promise { const dataPromises = this.requestDocumentData().then( this.frequencyMeasure === 'tokens' @@ -187,6 +199,9 @@ export abstract class BarchartData< return showLoading(this.loading$, dataPromises); } + /** Request document frequencies for all series, if they are not + * already filled in. + */ private requestDocumentData(): Promise[]> { const dataPromises = this.rawData$.value.map((series) => { // retrieve data if it was not already loaded @@ -201,8 +216,8 @@ export abstract class BarchartData< ); } - /** Retrieve all term frequencies and store in `rawData`. - * Term frequencies are only loaded if they were not already there. + /** Fetch term frequencies for all data series, if they are not already filled + * in. */ private requestTermFrequencyData(rawData: typeof this.rawData$.value): Promise[]> { // cancel and stop polling running tasks @@ -232,7 +247,10 @@ export abstract class BarchartData< return rawData; } - private selectSearchFields(queryModel: QueryModel) { + /** + * Return a copy of the query model with the right search fields selected. + */ + private selectSearchFields(queryModel: QueryModel): QueryModel { if (this.frequencyMeasure === 'documents') { return queryModel; } else { @@ -243,8 +261,7 @@ export abstract class BarchartData< } } - - /** return a copy of a query model with the query text set to the given value */ + /** Return a copy of a query model with the query text set to the given value */ private setQueryText(query: QueryModel, queryText: string): QueryModel { const queryModelCopy = query.clone(); queryModelCopy.setQueryText(queryText); @@ -282,12 +299,12 @@ export abstract class BarchartData< }; } - /** total document count for a data array */ + /** Total document count for a data array */ private totalDocCount(data: AggregateResult[]) { return _.sumBy(data, (item) => item.doc_count); } - /** fill in the `relative_doc_count` property for an array of datapoints. + /** Fill in the `relative_doc_count` property for an array of datapoints. */ private includeRelativeDocCount(data: DataPoint[], total: number): DataPoint[] { return data.map((item) => ({ @@ -296,10 +313,13 @@ export abstract class BarchartData< })); } - private onFailure(error) { + private onFailure(error: { message: string }) { this.error$.next(error.message); } + /** Update a data series in place with term frequency results. Also set + * `dataHasLoaded` to true. + */ private processSeriesTermFrequency( results: TermFrequencyResult[], series: BarchartSeries ): BarchartSeries { @@ -312,7 +332,7 @@ export abstract class BarchartData< }; /** - * add term frequency data to a DataPoint object + * Add term frequency data to a DataPoint object * * @param result output from request for term frequencies * @param cat DataPoint object where the data should be added @@ -330,12 +350,16 @@ export abstract class BarchartData< return cat; } + /** Whether total token counts are included in the data (which would enable + * normalisation) + */ private checkTotalTokenCount(data: BarchartSeries[]): boolean { return data.find((series) => series.data.find((cat) => cat.token_count !== undefined) ) !== undefined; } + /** Make request for the full term frequency data, rather than a sample */ abstract fullDataRequest(): Promise; /** Request doc counts for a series */ @@ -343,10 +367,11 @@ export abstract class BarchartData< queryModel: QueryModel ): Promise; + /** Request term frequency counts for a series */ + protected abstract requestSeriesTermFrequency(series: BarchartSeries, queryModel: QueryModel): Promise; /** convert the output of an aggregation search to the relevant result type */ protected abstract aggregateResultToDataPoint(cat: AggregateResult): DataPoint; - protected abstract requestSeriesTermFrequency(series: BarchartSeries, queryModel: QueryModel): Promise; }