diff --git a/frontend/src/app/models/compared-queries.ts b/frontend/src/app/models/compared-queries.ts index 79d6f6059..454364d75 100644 --- a/frontend/src/app/models/compared-queries.ts +++ b/frontend/src/app/models/compared-queries.ts @@ -4,7 +4,7 @@ import * as _ from 'lodash'; import { Store } from '../store/types'; import { Observable } from 'rxjs'; import { queryFromParams, queryToParams } from '../utils/params'; -import { filter, map, takeUntil } from 'rxjs/operators'; +import { distinctUntilChanged, distinctUntilKeyChanged, filter, map, takeUntil } from 'rxjs/operators'; interface CompareQueryState { primary: string|undefined; @@ -18,6 +18,9 @@ export class ComparedQueries extends StoreSync { /** all queries, including the primary query text */ allQueries$: Observable; + /** compared queries, excluding the primary query text */ + comparedQueries$: Observable; + protected keysInStore = ['query', COMPARE_TERMS_KEY]; constructor(store: Store) { @@ -29,6 +32,10 @@ export class ComparedQueries extends StoreSync { this.allQueries$ = this.state$.pipe( map(state => [state.primary, ...state.compare]) ); + this.comparedQueries$ = this.state$.pipe( + distinctUntilKeyChanged('compare', _.isEqual), + map(state => state.compare), + ); } reset() { diff --git a/frontend/src/app/visualization/barchart/barchart-data-histogram.ts b/frontend/src/app/visualization/barchart/barchart-data-histogram.ts new file mode 100644 index 000000000..6a725d060 --- /dev/null +++ b/frontend/src/app/visualization/barchart/barchart-data-histogram.ts @@ -0,0 +1,71 @@ +import { TermsAggregator, TermsResult } from '@models/aggregation'; +import { BarchartData } from './barchart-data'; +import { HistogramDataPoint, HistogramSeries, MultipleChoiceFilterOptions, QueryModel, RangeFilterOptions, TaskResult } from '@models'; +import _ from 'lodash'; + +export class HistogramData extends BarchartData { + fullDataRequest(): Promise { + const paramsPerSeries = this.rawData$.value.map((series) => { + const queryModel = this.queryModelForSeries(series, this.queryModel); + const bins = this.makeTermFrequencyBins(series); + return this.visualizationService.makeAggregateTermFrequencyParameters( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins + ); + }); + return this.apiService.requestFullData({ + visualization: 'aggregate_term_frequency', + parameters: paramsPerSeries, + corpus_name: this.queryModel.corpus.name, + }); + } + + /** specify aggregator object based on visualised field; + * used in document requests. + */ + protected getAggregator(): TermsAggregator { + let size = 100; + + const filterOptions = this.visualizedField.filterOptions; + if (filterOptions?.name === 'MultipleChoiceFilter') { + size = (filterOptions as MultipleChoiceFilterOptions).option_count; + } else if (filterOptions?.name === 'RangeFilter') { + const filterRange = + (filterOptions as RangeFilterOptions).upper - + (filterOptions as RangeFilterOptions).lower; + size = _.max([size, filterRange]) + } + return new TermsAggregator(this.visualizedField, size); + } + + protected requestSeriesDocCounts(queryModel: QueryModel) { + const aggregator = this.getAggregator(); + return this.searchService.aggregateSearch(this.queryModel.corpus, queryModel, aggregator); + } + + protected aggregateResultToDataPoint(cat: TermsResult) { + return cat; + } + + protected requestSeriesTermFrequency( + series: HistogramSeries, + queryModel: QueryModel + ) { + const bins = this.makeTermFrequencyBins(series); + return this.visualizationService.aggregateTermFrequencySearch( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins + ); + } + + private makeTermFrequencyBins(series: HistogramSeries) { + return series.data.map((bin) => ({ + fieldValue: bin.key, + size: this.documentLimitForCategory(bin, series), + })); + } +} diff --git a/frontend/src/app/visualization/barchart/barchart-data-timeline.ts b/frontend/src/app/visualization/barchart/barchart-data-timeline.ts new file mode 100644 index 000000000..b10b085b1 --- /dev/null +++ b/frontend/src/app/visualization/barchart/barchart-data-timeline.ts @@ -0,0 +1,151 @@ +import { BarchartData } from './barchart-data'; +import { BarchartSeries, DateFilterData, QueryModel, TimeCategory, TimelineDataPoint, TimelineSeries } from '@models'; +import { DateHistogramAggregator, DateHistogramResult } from '@models/aggregation'; +import { differenceInDays } from 'date-fns'; + +export class TimelineData extends BarchartData { + xDomain: [Date, Date]; + timeCategory: TimeCategory = 'year'; + + /** threshold for scaling down a unit on the time scale */ + private scaleDownThreshold = 10; + + zoomedInData (min: Date, max: Date): Promise[]> { + const promises = this.rawData$.value.map(series => { + const queryModelCopy = this.addQueryDateFilter(this.queryModel, min, max); + return this.getSeriesDocumentData( + series, queryModelCopy, false + ).then((result) => { + if (this.frequencyMeasure === 'tokens') { + return this.getTermFrequencies(result, queryModelCopy); + } else { + return result; + } + }); + }); + return Promise.all(promises); + } + + fullDataRequest() { + const paramsPerSeries = this.rawData$.value.map((series) => { + const queryModel = this.queryModelForSeries(series, this.queryModel); + const bins = this.makeTermFrequencyBins(series); + const unit = this.timeCategory; + return this.visualizationService.makeDateTermFrequencyParameters( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins, + unit + ); + }); + return this.apiService.requestFullData({ + visualization: 'date_term_frequency', + parameters: paramsPerSeries, + corpus_name: this.queryModel.corpus.name, + }); + } + + /** + * Get the time category (year/month/week/day) that should be used in the graph, + * based on minimum and maximum dates on the x axis. + */ + calculateTimeCategory(min: Date, max: Date): TimeCategory { + const diff = differenceInDays(max, min); + if (diff <= this.scaleDownThreshold) { + return 'day'; + } else if (diff <= this.scaleDownThreshold * 7) { + return 'week'; + } else if (diff <= this.scaleDownThreshold * 30) { + return 'month'; + } else { + return 'year'; + } + } + + protected refresh(): void { + this.setTimeDomain(); + super.refresh(); + } + + protected aggregateResultToDataPoint(cat: DateHistogramResult): TimelineDataPoint { + /* date fields are returned with keys containing identifiers by elasticsearch + replace with string representation, contained in 'key_as_string' field + */ + return { + date: new Date(cat.key_as_string), + doc_count: cat.doc_count, + }; + } + + /** Retrieve doc counts for a series. + * + * @param series series object + * @param setSearchRatio whether the `searchRatio` property of the series should be updated. + * True when retrieving results for the entire series, false when retrieving a window. + */ + protected requestSeriesDocCounts(queryModel: QueryModel) { + const aggregation = new DateHistogramAggregator( + this.visualizedField, + this.timeCategory + ); + return this.searchService.aggregateSearch(this.queryModel.corpus, queryModel, aggregation); + } + + protected requestSeriesTermFrequency(series: TimelineSeries, queryModel: QueryModel) { + const bins = this.makeTermFrequencyBins(series); + return this.visualizationService.dateTermFrequencySearch( + this.queryModel.corpus, + queryModel, + this.visualizedField.name, + bins, + this.timeCategory + ); + } + + /** + * Add a date filter to a query model restricting it to the provided min and max values. + */ + private addQueryDateFilter(query: QueryModel, min: Date, max: Date): QueryModel { + const queryModelCopy = query.clone(); + // download zoomed in results + const filter = this.visualizedField.makeSearchFilter(); + filter.set({ min, max }); + queryModelCopy.addFilter(filter); + return queryModelCopy; + } + + private makeTermFrequencyBins(series: TimelineSeries) { + return series.data.map((bin, index) => { + const [minDate, maxDate] = this.categoryTimeDomain( + bin, index, series + ); + return { + start_date: minDate, + end_date: maxDate, + size: this.documentLimitForCategory(bin, series), + }; + }); + } + + /** get min/max date for the entire graph and set domain and time category */ + private setTimeDomain() { + const filter = this.queryModel.filterForField(this.visualizedField) + || this.visualizedField.makeSearchFilter(); + const currentDomain = filter.currentData as DateFilterData; + const min = new Date(currentDomain.min); + const max = new Date(currentDomain.max); + this.xDomain = [min, max]; + this.timeCategory = this.calculateTimeCategory(min, max); + } + + /** time domain for a bin */ + private categoryTimeDomain(cat, catIndex, series): [Date, Date] { + const startDate = cat.date; + const endDate = + catIndex < series.data.length - 1 + ? series.data[catIndex + 1].date + : undefined; + return [startDate, endDate]; + } +} diff --git a/frontend/src/app/visualization/barchart/barchart-data.ts b/frontend/src/app/visualization/barchart/barchart-data.ts new file mode 100644 index 000000000..4e48b24a1 --- /dev/null +++ b/frontend/src/app/visualization/barchart/barchart-data.ts @@ -0,0 +1,377 @@ +import { BarchartSeries, CorpusField, HistogramDataPoint, QueryModel, TaskResult, TermFrequencyResult, TimelineDataPoint } from '@models'; +import { DateHistogramResult, TermsResult } from '@models/aggregation'; + +import { ComparedQueries } from '@models/compared-queries'; +import { ApiService, SearchService, VisualizationService } from '@services'; +import _ from 'lodash'; +import { hasPrefixTerm, mainContentFields } from './query-utils'; +import { BehaviorSubject, map, Observable, skip, Subject, takeUntil, withLatestFrom } from 'rxjs'; +import { showLoading } from '@utils/utils'; + +export type FrequencyMeasure = 'documents' | 'tokens'; + +const documentLimitBase = 10000; +const documentLimitPrefixQueries = 1000; + + +export abstract class BarchartData< + AggregateResult extends TermsResult | DateHistogramResult, + DataPoint extends TimelineDataPoint | HistogramDataPoint +> { + // rawData: a list of series + rawData$ = new BehaviorSubject[]>([]); + documentLimitExceeded = false; // whether the results include documents than the limit + dataHasLoaded: boolean; + + tasksToCancel: string[]; + + loading$ = new BehaviorSubject(false); + error$ = new Subject(); + stopPolling$ = new Subject(); + + totalTokenCountAvailable$ = this.rawData$.pipe( + map(this.checkTotalTokenCount) + ); + + constructor( + protected queryModel: QueryModel, + private comparedQueries: ComparedQueries, + public frequencyMeasure: FrequencyMeasure, + public visualizedField: CorpusField, + protected searchService: SearchService, + protected apiService: ApiService, + protected visualizationService: VisualizationService, + private destroy$: Observable, + ) { + this.refresh(); + this.queryModel.update.pipe( + takeUntil(this.destroy$) + ).subscribe(this.refresh.bind(this)); + this.comparedQueries.comparedQueries$.pipe( + skip(1), + withLatestFrom(this.comparedQueries.allQueries$), + map(([compared, all]) => all), + takeUntil(this.destroy$), + ).subscribe(this.updateQueries.bind(this)); + this.destroy$.subscribe(() => this.complete()); + } + + get documentLimit(): number { + return hasPrefixTerm(this.queryModel.queryText) ? documentLimitPrefixQueries : documentLimitBase; + } + + /** Complete observables */ + complete() { + this.rawData$.complete(); + this.stopPolling$.next(); + this.stopPolling$.complete(); + this.error$.complete(); + this.loading$.complete(); + } + + /** Create a new data object and fetch results */ + protected refresh() { + this.initSeries(); + this.loadData(); + } + + /** Request and fill in doc counts for a series */ + protected getSeriesDocumentData( + series: BarchartSeries, + queryModel: QueryModel = this.queryModel, + setSearchRatio = true + ): Promise> { + const queryModelCopy = this.queryModelForSeries(series, queryModel); + return this.requestSeriesDocCounts(queryModelCopy).then((result) => + this.docCountResultIntoSeries(result, series, setSearchRatio) + ); + } + + /** + * Get term frequency data for a data series + * + * Makes initial request for the series, polls result, merges frequency data with the + * series object. + + * @param series data series with document counts already filled in + * @param queryModel user query + * @returns promise of the series with term frequencies added + */ + protected getTermFrequencies( + series: BarchartSeries, + queryModel: QueryModel + ): Promise> { + this.dataHasLoaded = false; + const queryModelCopy = this.queryModelForSeries(series, queryModel); + return new Promise((resolve, reject) => { + this.requestSeriesTermFrequency(series, queryModelCopy).then(response => { + this.tasksToCancel = response.task_ids; + const poller$ = this.apiService.pollTasks(this.tasksToCancel, this.stopPolling$); + poller$.subscribe({ + error: (error) => { + this.onFailure(error); + reject(error); + }, + next: (result) => { + resolve(this.processSeriesTermFrequency(result['results'], series)); + }, + complete: () => { + // abort tasks if the Observable is completed via takeUntil + if (!this.dataHasLoaded) { + this.apiService.abortTasks({ task_ids: this.tasksToCancel }); + } + } + }); + }); + }); + } + + /** + * Calculate the maximum number of documents to read through in a bin + * when determining term frequency. + */ + protected documentLimitForCategory( + cat: DataPoint, + series: BarchartSeries + ): number { + return _.min([ + this.documentLimit, + _.ceil(cat.doc_count * series.searchRatio), + ]); + } + + /** Adapt query model to fit series: use correct search fields and query text */ + protected queryModelForSeries( + series: BarchartSeries, + queryModel: QueryModel + ) { + return this.selectSearchFields( + this.setQueryText(queryModel, series.queryText) + ); + } + + /** Create new data object with empty frequencies */ + private initSeries(): void { + const series = [ + this.newSeries(this.queryModel.queryText), + ...this.comparedQueries.state$.value.compare.map(this.newSeries) + ]; + this.rawData$.next(series) + } + + /** Update the data object to add/remove data series. If a query was already + * present, results are preserved. Starts loading results for new series. + */ + private updateQueries(queries: string[]) { + const data = queries.map((queryText) => { + const existingSeries = this.rawData$.value.find( + (series) => series.queryText === queryText + ); + return existingSeries || this.newSeries(queryText); + }); + this.rawData$.next(data); + this.loadData(); + } + + /** Make a blank series object, without frequencies */ + private newSeries(queryText: string): BarchartSeries { + return { + queryText, + data: [], + total_doc_count: 0, + searchRatio: 1.0, + }; + } + + /** Overarching function to load frequency data */ + private loadData(): Promise { + const dataPromises = this.requestDocumentData().then( + this.frequencyMeasure === 'tokens' + ? this.requestTermFrequencyData.bind(this) + : _.identity + ).then((rawData) => { + this.rawData$.next(rawData); + + if (!rawData.length) { + this.error$.next('No results'); + }; + }); + return showLoading(this.loading$, dataPromises); + } + + /** Request document frequencies for all series, if they are not + * already filled in. + */ + private requestDocumentData(): Promise[]> { + const dataPromises = this.rawData$.value.map((series) => { + // retrieve data if it was not already loaded + if (!series.data.length) { + return this.getSeriesDocumentData(series); + } else { + return series; + } + }); + return Promise.all(dataPromises).then( + this.checkDocumentLimitExceeded.bind(this) + ); + } + + /** Fetch term frequencies for all data series, if they are not already filled + * in. + */ + private requestTermFrequencyData(rawData: typeof this.rawData$.value): Promise[]> { + // cancel and stop polling running tasks + this.stopPolling$.next(); + const dataPromises = rawData.map(series => { + if (series.queryText && series.data.length && series.data[0].match_count === undefined) { + // retrieve data if it was not already loaded + return this.getTermFrequencies(series, this.queryModel); + } else { + return series; + } + }); + + return Promise.all(dataPromises); + } + + + /** + * Check whether any series found more documents than the document limit. + * This means that not all documents will be read when counting term frequency. + */ + private checkDocumentLimitExceeded( + rawData: typeof this.rawData$.value + ): typeof this.rawData$.value { + this.documentLimitExceeded = + rawData.find((series) => series.searchRatio < 1) !== undefined; + return rawData; + } + + /** + * Return a copy of the query model with the right search fields selected. + */ + private selectSearchFields(queryModel: QueryModel): QueryModel { + if (this.frequencyMeasure === 'documents') { + return queryModel; + } else { + const searchFields = mainContentFields(queryModel); + const queryModelCopy = queryModel.clone(); + queryModelCopy.setParams({ searchFields }); + return queryModelCopy; + } + } + + /** Return a copy of a query model with the query text set to the given value */ + private setQueryText(query: QueryModel, queryText: string): QueryModel { + const queryModelCopy = query.clone(); + queryModelCopy.setQueryText(queryText); + return queryModelCopy; + } + + /** + * Convert the result from an document search into a data series object. + * - Converts to the relevant DataPoint type using `aggregateResultToDataPoint` + * - Adds the total document count and search ratio for retrieving term frequencies. + * - Adds the relative document count. + * + * @param result result from aggregation search + * @param series series object that this data belongs to + * @param setSearchRatio whether the search ratio should be reset. Defaults to `true`, + * may be set to `false` when loading a portion of the series during zoom. + * @returns a copy of the series with the document counts included. + */ + private docCountResultIntoSeries( + result: AggregateResult[], + series: BarchartSeries, + setSearchRatio = true + ): BarchartSeries { + let data = result.map(this.aggregateResultToDataPoint); + const totalDocCount = this.totalDocCount(result); + const searchRatio = setSearchRatio + ? this.documentLimit / totalDocCount + : series.searchRatio; + data = this.includeRelativeDocCount(data, totalDocCount); + return { + data, + total_doc_count: totalDocCount, + searchRatio, + queryText: series.queryText, + }; + } + + /** Total document count for a data array */ + private totalDocCount(data: AggregateResult[]) { + return _.sumBy(data, (item) => item.doc_count); + } + + /** Fill in the `relative_doc_count` property for an array of datapoints. + */ + private includeRelativeDocCount(data: DataPoint[], total: number): DataPoint[] { + return data.map((item) => ({ + ...item, + relative_doc_count: item.doc_count / total, + })); + } + + private onFailure(error: { message: string }) { + this.error$.next(error.message); + } + + /** Update a data series in place with term frequency results. Also set + * `dataHasLoaded` to true. + */ + private processSeriesTermFrequency( + results: TermFrequencyResult[], series: BarchartSeries + ): BarchartSeries { + this.dataHasLoaded = true; + series.data = _.zip(series.data, results).map(pair => { + const [bin, res] = pair; + return this.addTermFrequencyToCategory(res, bin); + }); + return series; + }; + + /** + * Add term frequency data to a DataPoint object + * + * @param result output from request for term frequencies + * @param cat DataPoint object where the data should be added + */ + private addTermFrequencyToCategory( + data: TermFrequencyResult, cat: DataPoint + ): DataPoint { + cat.match_count = data.match_count; + cat.total_doc_count = data.total_doc_count; + cat.token_count = data.token_count; + cat.matches_by_doc_count = data.match_count / data.total_doc_count; + cat.matches_by_token_count = data.token_count + ? data.match_count / data.token_count + : undefined; + return cat; + } + + /** Whether total token counts are included in the data (which would enable + * normalisation) + */ + private checkTotalTokenCount(data: BarchartSeries[]): boolean { + return data.find((series) => + series.data.find((cat) => cat.token_count !== undefined) + ) !== undefined; + } + + /** Make request for the full term frequency data, rather than a sample */ + abstract fullDataRequest(): Promise; + + /** Request doc counts for a series */ + protected abstract requestSeriesDocCounts( + queryModel: QueryModel + ): Promise; + + /** Request term frequency counts for a series */ + protected abstract requestSeriesTermFrequency(series: BarchartSeries, queryModel: QueryModel): Promise; + + /** convert the output of an aggregation search to the relevant result type */ + protected abstract aggregateResultToDataPoint(cat: AggregateResult): DataPoint; + + +} diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index f75a2ec47..790d65389 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/member-ordering */ -import { Directive, EventEmitter, Host, HostBinding, Input, OnChanges, OnDestroy, OnInit, Output, SimpleChanges } from '@angular/core'; +import { Directive, EventEmitter, HostBinding, Input, OnChanges, OnDestroy, OnInit, Output, SimpleChanges } from '@angular/core'; import * as _ from 'lodash'; @@ -10,27 +10,19 @@ import { } from '@services/index'; import { Chart, ChartOptions } from 'chart.js'; import { - Corpus, FreqTableHeaders, QueryModel, CorpusField, - TaskResult, - BarchartSeries, - TimelineDataPoint, - HistogramDataPoint, - TermFrequencyResult, ChartParameters, } from '@models'; import Zoom from 'chartjs-plugin-zoom'; -import { BehaviorSubject, Subject } from 'rxjs'; +import { skip, Subject, takeUntil } from 'rxjs'; import { selectColor } from '@utils/select-color'; import { VisualizationService } from '@services/visualization.service'; -import { showLoading } from '@utils/utils'; -import { takeUntil } from 'rxjs/operators'; -import { DateHistogramResult, TermsResult } from '@models/aggregation'; import { ComparedQueries } from '@models/compared-queries'; import { RouterStoreService } from '../../store/router-store.service'; -import { hasPrefixTerm } from './query-utils'; +import { hasPrefixTerm, mainContentFields } from './query-utils'; +import { BarchartData, FrequencyMeasure } from './barchart-data'; const hintSeenSessionStorageKey = 'hasSeenTimelineZoomingHint'; const hintHidingMinDelay = 500; // milliseconds @@ -47,34 +39,26 @@ const documentLimitPrefixQueries = 1000; /** The barchartComponent is used to define shared functionality between the * histogram and timeline components. It does not function as a stand-alone component. */ export abstract class BarchartDirective< - AggregateResult extends TermsResult | DateHistogramResult, - DataPoint extends TimelineDataPoint | HistogramDataPoint + Data extends BarchartData > implements OnChanges, OnInit, OnDestroy { @HostBinding('style.display') display = 'block'; // needed for loading spinner positioning public showHint: boolean; - // rawData: a list of series - rawData: BarchartSeries[]; - // chart object chart: Chart; - @Input() corpus: Corpus; - @Input() queryModel: QueryModel; - @Input() visualizedField: CorpusField; + @Input({ required: true }) queryModel!: QueryModel; + @Input({ required: true }) visualizedField!: CorpusField; @Input() asTable: boolean; @Input() palette: string[]; - @Input() frequencyMeasure: 'documents' | 'tokens' = 'documents'; + @Input() frequencyMeasure: FrequencyMeasure = 'documents'; normalizer: 'raw' | 'percent' | 'documents' | 'terms' = 'raw'; chartType: 'bar' | 'line' | 'scatter' = 'bar'; comparedQueries: ComparedQueries; - documentLimitExceeded = false; // whether the results include documents than the limit - totalTokenCountAvailable: boolean; // whether the data includes token count totals - // table data tableHeaders: FreqTableHeaders; tableData: any[]; @@ -97,17 +81,15 @@ export abstract class BarchartDirective< }, }; - isLoading$ = new BehaviorSubject(false); - // eslint-disable-next-line @angular-eslint/no-output-native @Output() error = new EventEmitter(); - destroy$ = new Subject(); - stopPolling$ = new Subject(); + /** indicates that the data model should be re-instantiated */ + refresh$ = new Subject(); - tasksToCancel: string[]; + destroy$ = new Subject(); - dataHasLoaded: boolean; + data: Data; basicChartOptions: ChartOptions = { // chart options not suitable for Chart.defaults.global scales: { @@ -173,11 +155,10 @@ export abstract class BarchartDirective< chartDefault.plugins.tooltip.displayColors = false; chartDefault.plugins.tooltip.intersect = false; this.comparedQueries = new ComparedQueries(this.routerStoreService); - this.comparedQueries.allQueries$.subscribe(this.updateQueries.bind(this)); } get isLoading() { - return this.isLoading$.value; + return this.data.loading$.value; } get documentLimit(): number { @@ -189,25 +170,36 @@ export abstract class BarchartDirective< } ngOnChanges(changes: SimpleChanges) { - if (changes.queryModel) { - this.queryModel.update - .pipe(takeUntil(this.destroy$)) - .subscribe(this.refreshChart.bind(this)); - } - // new doc counts should be requested if query has changed if (this.changesRequireRefresh(changes)) { + this.refresh$.next(); this.refreshChart(); } else if (changes.palette) { - this.prepareChart(); + this.setChart(this.data.rawData$.value); } } ngOnDestroy(): void { - this.stopPolling$.next(); + this.clearCanvas(); this.destroy$.next(undefined); + this.refresh$.complete(); + this.destroy$.complete(); this.comparedQueries.complete(); } + onDataLoaded(data: typeof this.data.rawData$.value) { + // initialise or update chart + this.setChart(data); + + // update freqtable + this.setTableHeaders(data); + this.setTableData(data); + + // load zoomed-in data if needed + if (this.isZoomedIn) { + this.onZoomIn(this.chart, true); + } + } + /** check whether input changes should force reloading the data */ changesRequireRefresh(changes: SimpleChanges): boolean { const relevantChanges = [ @@ -227,8 +219,8 @@ export abstract class BarchartDirective< onOptionChange(chartParameters: ChartParameters) { this.normalizer = chartParameters.normalizer; this.chartType = chartParameters.chartType; - if (this.rawData && this.chart) { - this.prepareChart(); + if (this.data.rawData$.value && this.chart) { + this.onDataLoaded(this.data.rawData$.value); } } @@ -236,16 +228,24 @@ export abstract class BarchartDirective< * clear data and update chart */ refreshChart(): void { - this.initQueries(); + // clear canvas this.clearCanvas(); - this.prepareChart(); - } - initQueries(): void { - this.rawData = [ - this.newSeries(this.queryText), - ...this.comparedQueries.state$.value.compare.map(this.newSeries) - ]; + // complete existing data model + this.data?.complete(); + + // create new data model and subscribe + this.data = this.initData(); + this.data.rawData$.pipe( + skip(1), // skip the first (empty) event + takeUntil(this.refresh$), + takeUntil(this.destroy$), + ).subscribe(data => this.onDataLoaded(data)); + this.data.error$.pipe( + takeUntil(this.refresh$), + takeUntil(this.destroy$), + ).subscribe(message => this.onDataError(message)); + } /** if a chart is active, clear canvas and reset chart object */ @@ -257,88 +257,11 @@ export abstract class BarchartDirective< } } - /** update the queries in the graph to the input array. Preserve results if possible, and kick off loading the rest. */ - updateQueries(queries: string[]) { - if (this.rawData) { - this.rawData = queries.map((queryText) => { - const existingSeries = this.rawData.find( - (series) => series.queryText === queryText - ); - return existingSeries || this.newSeries(queryText); - }); - this.prepareChart(); - } - } - - /** make a blank series object */ - newSeries(queryText: string): BarchartSeries { - return { - queryText, - data: [], - total_doc_count: 0, - searchRatio: 1.0, - }; - } - - /** Show a loading spinner and load data for the graph. - * This function should be called after (potential) changes to parameters. - */ - prepareChart() { - showLoading(this.isLoading$, this.loadData()); - } - - /** load data for the graph (if needed), update the graph and freqtable. */ - loadData(): Promise { - // load data if needed - return this.requestDocumentData() - .then( - this.frequencyMeasure === 'tokens' - ? this.requestTermFrequencyData.bind(this) - : _.identity - ) - .then((rawData) => { - this.rawData = rawData; - - if (!this.rawData.length) { - this.error.emit({ message: 'No results' }); - } - - // initialise or update chart - this.setChart(); - - // update freqtable - this.setTableHeaders(); - this.setTableData(); - - // load zoomed-in data if needed - if (this.isZoomedIn) { - this.onZoomIn(this.chart, true); - } - }); - } - - /** Retrieve all document frequencies and store in `rawData`. - * Document frequencies are only loaded if they are not already in the data. */ - requestDocumentData(): Promise { - const dataPromises = this.rawData.map((series) => { - if (!series.data.length) { - // retrieve data if it was not already loaded - return this.getSeriesDocumentData(series); - } else { - return series; - } - }); - - return Promise.all(dataPromises).then( - this.checkDocumentLimitExceeded.bind(this) - ); - } - selectSearchFields(queryModel: QueryModel) { if (this.frequencyMeasure === 'documents') { return queryModel; } else { - const mainContentFields = this.corpus.fields.filter( + const mainContentFields = this.queryModel.corpus.fields.filter( (field) => field.searchable && field.displayType === 'text_content' ); @@ -348,215 +271,13 @@ export abstract class BarchartDirective< } } - /** - * Convert the result from an document search into a data series object. - * - Converts to the relevant DataPoint type using `aggregateResultToDataPoint` - * - Adds the total document count and search ratio for retrieving term frequencies. - * - Adds the relative document count. - * - * @param result result from aggregation search - * @param series series object that this data belongs to - * @param setSearchRatio whether the search ratio should be reset. Defaults to `true`, - * may be set to `false` when loading a portion of the series during zoom. - * @returns a copy of the series with the document counts included. - */ - docCountResultIntoSeries( - result: AggregateResult[], - series: BarchartSeries, - setSearchRatio = true - ): BarchartSeries { - let data = result.map(this.aggregateResultToDataPoint); - const totalDocCount = this.totalDocCount(result); - const searchRatio = setSearchRatio - ? this.documentLimit / totalDocCount - : series.searchRatio; - data = this.includeRelativeDocCount(data, totalDocCount); - return { - data, - total_doc_count: totalDocCount, - searchRatio, - queryText: series.queryText, - }; - } - - /** convert the output of an aggregation search to the relevant result type */ - abstract aggregateResultToDataPoint(cat: AggregateResult): DataPoint; - - /** fill in the `relative_doc_count` property for an array of datapoints. - */ - includeRelativeDocCount(data: DataPoint[], total: number): DataPoint[] { - return data.map((item) => { - const result = _.clone(item); - result.relative_doc_count = result.doc_count / total; - return result; - }); + onDataError(message: string) { + console.error(message); + this.error.emit(`could not load results: ${message}`); } - /** - * Check whether any series found more documents than the document limit. - * This means that not all documents will be read when counting term frequency. - */ - checkDocumentLimitExceeded( - rawData: typeof this.rawData - ): typeof this.rawData { - this.documentLimitExceeded = - rawData.find((series) => series.searchRatio < 1) !== undefined; - return rawData; - } - - /** Retrieve all term frequencies and store in `rawData`. - * Term frequencies are only loaded if they were not already there. - */ - requestTermFrequencyData(rawData: typeof this.rawData): Promise[]> { - // cancel and stop polling running tasks - this.stopPolling$.next(); - const dataPromises = rawData.map(series => { - if (series.queryText && series.data.length && series.data[0].match_count === undefined) { - // retrieve data if it was not already loaded - return this.getTermFrequencies(series, this.queryModel); - } else { - return series; - } - }); - - return Promise.all(dataPromises).then( - this.checkTotalTokenCount.bind(this) - ); - } - - checkTotalTokenCount(rawData: typeof this.rawData): typeof this.rawData { - const totalTokenCountAvailable = - rawData.find((series) => - series.data.find((cat) => cat.token_count) - ) !== undefined; - if ( - this.frequencyMeasure === 'tokens' && - totalTokenCountAvailable && - !this.totalTokenCountAvailable - ) { - this.normalizer = 'terms'; - } - this.totalTokenCountAvailable = totalTokenCountAvailable; - return rawData; - } - - getTermFrequencies( - series: BarchartSeries, - queryModel: QueryModel - ): Promise> { - this.dataHasLoaded = false; - const queryModelCopy = this.queryModelForSeries(series, queryModel); - return new Promise((resolve, reject) => { - this.requestSeriesTermFrequency(series, queryModelCopy).then(response => { - this.tasksToCancel = response.task_ids; - const poller$ = this.apiService.pollTasks(this.tasksToCancel, this.stopPolling$); - poller$.subscribe({ - error: (error) => { - this.onFailure(error); - reject(error); - }, - next: (result) => { - resolve(this.processSeriesTermFrequency(result['results'], series)); - }, - complete: () => { - // abort tasks if the Observable is completed via takeUntil - if (!this.dataHasLoaded) { - this.apiService.abortTasks({ task_ids: this.tasksToCancel }); - } - } - }); - }); - }); - } - - onFailure(error) { - console.error(error); - this.error.emit(`could not load results: ${error.message}`); - } - - abstract requestSeriesTermFrequency(series: BarchartSeries, queryModel: QueryModel): Promise; - - abstract makeTermFrequencyBins(series: BarchartSeries); - - processSeriesTermFrequency(results: TermFrequencyResult[], series: BarchartSeries): BarchartSeries { - this.dataHasLoaded = true; - series.data = _.zip(series.data, results).map(pair => { - const [bin, res] = pair; - return this.addTermFrequencyToCategory(res, bin); - }); - return series; - }; - - - /** total document count for a data array */ - totalDocCount(data: AggregateResult[]) { - return _.sumBy(data, (item) => item.doc_count); - } - - /** - * calculate the maximum number of documents to read through in a bin - * when determining term frequency. - */ - documentLimitForCategory( - cat: DataPoint, - series: BarchartSeries - ): number { - return _.min([ - this.documentLimit, - _.ceil(cat.doc_count * series.searchRatio), - ]); - } - - /** - * add term frequency data to a DataPoint object - * - * @param result output from request for term frequencies - * @param cat DataPoint object where the data should be added - */ - addTermFrequencyToCategory( - data: TermFrequencyResult, - cat: DataPoint - ): DataPoint { - cat.match_count = data.match_count; - cat.total_doc_count = data.total_doc_count; - cat.token_count = data.token_count; - cat.matches_by_doc_count = data.match_count / data.total_doc_count; - cat.matches_by_token_count = data.token_count - ? data.match_count / data.token_count - : undefined; - return cat; - } - - /** Request and fill in doc counts for a series */ - getSeriesDocumentData( - series: BarchartSeries, - queryModel: QueryModel = this.queryModel, - setSearchRatio = true - ): Promise> { - const queryModelCopy = this.queryModelForSeries(series, queryModel); - - return this.requestSeriesDocCounts(queryModelCopy).then((result) => - this.docCountResultIntoSeries(result, series, setSearchRatio) - ); - } - - /** adapt query model to fit series: use correct search fields and query text */ - queryModelForSeries( - series: BarchartSeries, - queryModel: QueryModel - ) { - return this.selectSearchFields( - this.setQueryText(queryModel, series.queryText) - ); - } - - /** Request doc counts for a series */ - abstract requestSeriesDocCounts( - queryModel: QueryModel - ): Promise; - requestFullData() { - this.fullDataRequest() + this.data.fullDataRequest() .then(() => this.notificationService.showMessage( 'Full data requested! You will receive an email when your download is ready.', @@ -576,18 +297,17 @@ export abstract class BarchartDirective< }); } - abstract fullDataRequest(): Promise; - /** update or initialise chart (should be ran after updates to `rawData`) */ - setChart(): void { + setChart(data: typeof this.data.rawData$.value): void { if (this.chart) { - this.updateChartData(); + this.updateChartData(data); } else { - this.initChart(); + this.initChart(data); } } + /** select the columns/headers for the frequency table */ - abstract setTableHeaders(): void; + abstract setTableHeaders(data: typeof this.data.rawData$.value): void; /** code to be executed when zooming in, or when parameters are updated while zoomed in */ onZoomIn(chart, triggeredByDataUpdate = false) {} /** options for the chart. @@ -597,9 +317,9 @@ export abstract class BarchartDirective< abstract chartOptions(datasets); /** initalise a new chart */ - initChart() { - const labels = this.getLabels(); - const datasets = this.getDatasets(); + initChart(data: typeof this.data.rawData$.value) { + const labels = this.getLabels(data); + const datasets = this.getDatasets(data); const options = this.chartOptions(datasets); this.chart = new Chart(barchartID, { @@ -621,9 +341,9 @@ export abstract class BarchartDirective< } /** After updating `rawData`, this executes the update in the chart. */ - updateChartData() { - const labels = this.getLabels(); - const datasets = this.getDatasets(); + updateChartData(data: typeof this.data.rawData$.value) { + const labels = this.getLabels(data); + const datasets = this.getDatasets(data); this.chart.options = this.chartOptions(datasets); this.chart.data.labels = labels; this.chart.data.datasets = datasets; @@ -634,12 +354,12 @@ export abstract class BarchartDirective< /** Return x-axis labels for the chartJS dataset. * Can be left undefined depending on data format. */ - getLabels(): string[] { + getLabels(data: typeof this.data.rawData$.value): string[] { return undefined; } /** return chartJS dataset objects based on rawData */ - abstract getDatasets(): any[]; + abstract getDatasets(data: typeof this.data.rawData$.value): any[]; /** * Show the zooming hint once per session, hide automatically with a delay @@ -693,17 +413,10 @@ export abstract class BarchartDirective< } } - /** return a copy of a query model with the query text set to the given value */ - setQueryText(query: QueryModel, queryText: string): QueryModel { - const queryModelCopy = query.clone(); - queryModelCopy.setQueryText(queryText); - return queryModelCopy; - } - /** assemble the array of table data */ - setTableData() { - if (this.rawData && this.rawData.length) { - this.tableData = _.flatMap(this.rawData, (series) => + setTableData(data: typeof this.data.rawData$.value) { + if (data && data.length) { + this.tableData = _.flatMap(data, (series) => series.data.map((item) => { const result = _.cloneDeep(item) as any; result.queryText = series.queryText; @@ -723,9 +436,9 @@ export abstract class BarchartDirective< * This differs per series, but user message only gives one value, currently the minimum. */ get percentageDocumentsSearched() { - if (this.rawData) { + if (this.data.rawData$.value) { return _.round( - 100 * _.min(this.rawData.map((series) => series.searchRatio)), + 100 * _.min(this.data.rawData$.value.map((series) => series.searchRatio)), 1 ); } @@ -740,22 +453,17 @@ export abstract class BarchartDirective< } get searchFields(): string { - if (this.corpus && this.queryModel) { - const searchFields = this.selectSearchFields( - this.queryModel - ).searchFields; - + if (this.queryModel) { + const searchFields = mainContentFields(this.queryModel); const displayNames = searchFields.map((field) => field.displayName); - return displayNames.join(', '); } - return 'all fields'; } chartTitle() { - const queryTexts = this.rawData.map((series) => series.queryText); - if (this.queryText == null && this.rawData.length === 1) { + const queryTexts = this.data.rawData$.value.map((series) => series.queryText); + if (!this.queryModel.queryText && this.data.rawData$.value.length === 1) { return `Frequency of documents by ${this.visualizedField.displayName} (n of ${this.frequencyMeasure}, ${this.normalizer})`; } else { const normalizationText = ['raw', 'percent'].includes( @@ -769,9 +477,5 @@ export abstract class BarchartDirective< } } - get queryText(): string { - if (this.queryModel) { - return this.queryModel.queryText; - } - } + abstract initData(): Data; } diff --git a/frontend/src/app/visualization/barchart/histogram.component.html b/frontend/src/app/visualization/barchart/histogram.component.html index 90f17cf1b..a5a20be5e 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.html +++ b/frontend/src/app/visualization/barchart/histogram.component.html @@ -6,10 +6,12 @@
+ (chartParameters)="onOptionChange($event)">
@@ -17,13 +19,13 @@ -
+
-
+
The results for your search include too many documents to count all matches. diff --git a/frontend/src/app/visualization/barchart/histogram.component.spec.ts b/frontend/src/app/visualization/barchart/histogram.component.spec.ts index a7fffb3e6..63bab550a 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.spec.ts +++ b/frontend/src/app/visualization/barchart/histogram.component.spec.ts @@ -5,6 +5,7 @@ import { commonTestBed } from '../../common-test-bed'; import { HistogramComponent } from './histogram.component'; import { corpusFactory, keywordFieldFactory } from '../../../mock-data/corpus'; +import { SimpleChange } from '@angular/core'; describe('HistogramCompoment', () => { let component: HistogramComponent; @@ -17,6 +18,13 @@ describe('HistogramCompoment', () => { beforeEach(() => { fixture = TestBed.createComponent(HistogramComponent); component = fixture.componentInstance; + + const corpus = corpusFactory(); + component.queryModel = new QueryModel(corpus); + component.visualizedField = corpus.fields.find(field => field.mappingType == 'keyword'); + component.ngOnChanges({ + queryModel: new SimpleChange(undefined, component.queryModel, true) + }); fixture.detectChanges(); }); @@ -26,18 +34,18 @@ describe('HistogramCompoment', () => { it('should filter text fields', () => { - component.corpus = corpusFactory(); - component.corpus.fields[0] = keywordFieldFactory(true); + const corpus = corpusFactory(); + corpus.fields[0] = keywordFieldFactory(true); component.frequencyMeasure = 'documents'; - const query1 = new QueryModel(component.corpus); + const query1 = new QueryModel(corpus); query1.setQueryText('test'); - const query2 = new QueryModel(component.corpus); + const query2 = new QueryModel(corpus); query2.setParams({ queryText: 'test', - searchFields: component.corpus.fields.slice(0, 2), + searchFields: corpus.fields.slice(0, 2), }); const queries = [ query1, query2 ]; @@ -51,7 +59,7 @@ describe('HistogramCompoment', () => { queries.forEach(query => { const newQuery = component.selectSearchFields(query); - expect(newQuery.searchFields).toEqual([component.corpus.fields[1]]); + expect(newQuery.searchFields).toEqual([corpus.fields[1]]); }); }); diff --git a/frontend/src/app/visualization/barchart/histogram.component.ts b/frontend/src/app/visualization/barchart/histogram.component.ts index a1fdac1a5..724e7d9f6 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.ts +++ b/frontend/src/app/visualization/barchart/histogram.component.ts @@ -1,16 +1,9 @@ import { Component, OnChanges, OnInit } from '@angular/core'; import * as _ from 'lodash'; -import { - HistogramDataPoint, - HistogramSeries, - MultipleChoiceFilterOptions, - QueryModel, - RangeFilterOptions, -} from '@models/index'; import { selectColor } from '@utils/select-color'; import { BarchartDirective } from './barchart.directive'; -import { TermsAggregator, TermsResult } from '@models/aggregation'; +import { HistogramData } from './barchart-data-histogram'; function formatXAxisLabel(value): string { const label = this.getLabelForValue(value); // from chartJS api @@ -28,7 +21,7 @@ function formatXAxisLabel(value): string { standalone: false }) export class HistogramComponent - extends BarchartDirective + extends BarchartDirective implements OnInit, OnChanges { /** On what property should the data be sorted? */ get defaultSort(): string { @@ -38,80 +31,24 @@ export class HistogramComponent return this.currentValueKey; } - /** specify aggregator object based on visualised field; - * used in document requests. - */ - getAggregator(): TermsAggregator { - let size = 100; - - const filterOptions = this.visualizedField.filterOptions; - if (filterOptions?.name === 'MultipleChoiceFilter') { - size = (filterOptions as MultipleChoiceFilterOptions).option_count; - } else if (filterOptions?.name === 'RangeFilter') { - const filterRange = - (filterOptions as RangeFilterOptions).upper - - (filterOptions as RangeFilterOptions).lower; - size = _.max([size, filterRange]) - } - return new TermsAggregator(this.visualizedField, size); - } - - requestSeriesDocCounts(queryModel: QueryModel) { - const aggregator = this.getAggregator(); - - return this.searchService.aggregateSearch(this.corpus, queryModel, aggregator); - } - - aggregateResultToDataPoint(cat: TermsResult) { - return cat; - } - - requestSeriesTermFrequency( - series: HistogramSeries, - queryModel: QueryModel - ) { - const bins = this.makeTermFrequencyBins(series); - return this.visualizationService.aggregateTermFrequencySearch( - this.corpus, - queryModel, - this.visualizedField.name, - bins + initData(): HistogramData { + return new HistogramData( + this.queryModel, + this.comparedQueries, + this.frequencyMeasure, + this.visualizedField, + this.searchService, + this.apiService, + this.visualizationService, + this.destroy$, ); } - makeTermFrequencyBins(series: HistogramSeries) { - return series.data.map((bin) => ({ - fieldValue: bin.key, - size: this.documentLimitForCategory(bin, series), - })); - } - - fullDataRequest() { - const paramsPerSeries = this.rawData.map((series) => { - const queryModel = this.queryModelForSeries( - series, - this.queryModel - ); - const bins = this.makeTermFrequencyBins(series); - return this.visualizationService.makeAggregateTermFrequencyParameters( - this.corpus, - queryModel, - this.visualizedField.name, - bins - ); - }); - return this.apiService.requestFullData({ - visualization: 'aggregate_term_frequency', - parameters: paramsPerSeries, - corpus_name: this.corpus.name, - }); - } - - getLabels(): string[] { + getLabels(data: typeof this.data.rawData$.value): string[] { // make an array of all unique labels and sort - if (this.rawData) { - const all_labels = _.flatMap(this.rawData, (series) => + if (data) { + const all_labels = _.flatMap(data, (series) => series.data.map((item) => item.key) ); const labels = all_labels.filter( @@ -123,7 +60,7 @@ export class HistogramComponent } else { const valueKey = this.currentValueKey; sorted_labels = _.sortBy(labels, (label) => - _.sumBy(this.rawData, (series) => { + _.sumBy(data, (series) => { const item = series.data.find((i) => i.key === label); return -1 * (item ? item[valueKey] : 0); }) @@ -133,11 +70,11 @@ export class HistogramComponent } } - getDatasets() { - const labels = this.getLabels(); + getDatasets(data: typeof this.data.rawData$.value) { + const labels = this.getLabels(data); const valueKey = this.currentValueKey; - return this.rawData.map((series, seriesIndex) => ({ + return data.map((series, seriesIndex) => ({ type: this.chartType, label: series.queryText ? series.queryText : '(no query)', data: labels.map((key) => { @@ -172,7 +109,7 @@ export class HistogramComponent return options; } - setTableHeaders() { + setTableHeaders(data: typeof this.data.rawData$.value) { /* Provides the table headers to the freqTable component. Determines optional headers. */ @@ -183,7 +120,7 @@ export class HistogramComponent this.normalizer === 'raw' ? 'Frequency' : 'Relative frequency'; const valueKey = this.currentValueKey; - if (this.rawData.length > 1) { + if (data.length > 1) { // if there are several queries, fulltable is disabled this.tableHeaders = [ { key: 'key', label, isSecondaryFactor: true }, diff --git a/frontend/src/app/visualization/barchart/query-utils.ts b/frontend/src/app/visualization/barchart/query-utils.ts index cee72a8d9..6a740bc7d 100644 --- a/frontend/src/app/visualization/barchart/query-utils.ts +++ b/frontend/src/app/visualization/barchart/query-utils.ts @@ -1,3 +1,5 @@ +import { CorpusField, QueryModel } from '@models'; + /** Whether the query contains a prefix term * * The term frequency graph requires more resources for prefix terms so these should use @@ -7,3 +9,9 @@ */ export const hasPrefixTerm = (queryText: string): boolean => /\*/.test(queryText); + + +export const mainContentFields = (query: QueryModel): CorpusField[] => + query.corpus.fields.filter( + (field) => field.searchable && field.displayType === 'text_content' + ); diff --git a/frontend/src/app/visualization/barchart/timeline.component.html b/frontend/src/app/visualization/barchart/timeline.component.html index 43fa40d45..27794e5ac 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.html +++ b/frontend/src/app/visualization/barchart/timeline.component.html @@ -10,10 +10,12 @@
+ (chartParameters)="onOptionChange($event)">
@@ -21,13 +23,13 @@
-
+
-
+
The results for your search include too many documents to count all matches. diff --git a/frontend/src/app/visualization/barchart/timeline.component.spec.ts b/frontend/src/app/visualization/barchart/timeline.component.spec.ts index 71a4a6e23..172a28bb1 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.spec.ts +++ b/frontend/src/app/visualization/barchart/timeline.component.spec.ts @@ -5,6 +5,7 @@ import { commonTestBed } from '../../common-test-bed'; import { TimelineComponent } from './timeline.component'; import { QueryModel } from '@models'; import { corpusFactory } from 'mock-data/corpus'; +import { SimpleChange } from '@angular/core'; describe('TimelineComponent', () => { let component: TimelineComponent; @@ -17,8 +18,12 @@ describe('TimelineComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(TimelineComponent); component = fixture.componentInstance; - component.queryModel = new QueryModel(corpusFactory()); - component.visualizedField = component.queryModel.corpus.fields[2]; + const corpus = corpusFactory(); + component.queryModel = new QueryModel(corpus); + component.visualizedField = corpus.fields.find(field => field.mappingType == 'date'); + component.ngOnChanges({ + queryModel: new SimpleChange(undefined, component.queryModel, true) + }); fixture.detectChanges(); }); @@ -27,17 +32,16 @@ describe('TimelineComponent', () => { }); it('should calculate correct time categories', () => { - expect(component.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-01-05'))).toEqual('day'); - expect(component.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-02-01'))).toEqual('week'); - expect(component.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-06-01'))).toEqual('month'); - expect(component.calculateTimeCategory(new Date('1950-01-01'), new Date('1951-01-01'))).toEqual('year'); + expect(component.data.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-01-05'))).toEqual('day'); + expect(component.data.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-02-01'))).toEqual('week'); + expect(component.data.calculateTimeCategory(new Date('1950-01-01'), new Date('1950-06-01'))).toEqual('month'); + expect(component.data.calculateTimeCategory(new Date('1950-01-01'), new Date('1953-01-01'))).toEqual('year'); }) it('should calibrate the x axis with a margin', () => { - component.setTimeDomain(); const min = new Date('1802-01-01'); const max = new Date('1804-01-01'); - expect(component.callibratexAxis(min, -1)).toEqual(new Date('1801-01-01')); - expect(component.callibratexAxis(max)).toEqual(new Date('1805-01-01')); + expect(component.addDateMargin(min, -1)).toEqual(new Date('1801-01-01')); + expect(component.addDateMargin(max)).toEqual(new Date('1805-01-01')); }) }); diff --git a/frontend/src/app/visualization/barchart/timeline.component.ts b/frontend/src/app/visualization/barchart/timeline.component.ts index 2069b1218..7d752a449 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.ts +++ b/frontend/src/app/visualization/barchart/timeline.component.ts @@ -1,24 +1,20 @@ import { Component, OnChanges, OnInit } from '@angular/core'; import * as _ from 'lodash'; -import { addDays, addMonths, addWeeks, addYears, differenceInDays, format } from 'date-fns'; +import { addDays, addMonths, addWeeks, addYears, format } from 'date-fns'; import 'chartjs-adapter-moment'; import { QueryModel, TimelineSeries, - TimelineDataPoint, TimeCategory, - DateFilterData, } from '@models/index'; import { BarchartDirective } from './barchart.directive'; import { selectColor } from '@utils/select-color'; import { showLoading } from '@utils/utils'; -import { - DateHistogramAggregator, - DateHistogramResult, -} from '@models/aggregation'; +import { TimelineData } from './barchart-data-timeline'; +import { BehaviorSubject } from 'rxjs'; @Component({ @@ -28,132 +24,37 @@ import { standalone: false }) export class TimelineComponent - extends BarchartDirective + extends BarchartDirective implements OnChanges, OnInit { - /** domain on the axis */ - public xDomain: [Date, Date]; - /** time unit on the x-axis */ - private currentTimeCategory: TimeCategory; - /** threshold for scaling down a unit on the time scale */ - private scaleDownThreshold = 10; - - refreshChart(): void { - this.initQueries(); - this.clearCanvas(); - this.setTimeDomain(); - this.prepareChart(); - } + /** time unit on the x-axis; may be the category of zoomed-in data */ + private viewTimeCategory: TimeCategory; - /** get min/max date for the entire graph and set domain and time category */ - setTimeDomain() { - const filter = this.queryModel.filterForField(this.visualizedField) - || this.visualizedField.makeSearchFilter(); - const currentDomain = filter.currentData as DateFilterData; - const min = new Date(currentDomain.min); - const max = new Date(currentDomain.max); - this.xDomain = [min, max]; - this.currentTimeCategory = this.calculateTimeCategory(min, max); - } + private zoomedDataLoading$ = new BehaviorSubject(false); - aggregateResultToDataPoint(cat: DateHistogramResult): TimelineDataPoint { - /* date fields are returned with keys containing identifiers by elasticsearch - replace with string representation, contained in 'key_as_string' field - */ - return { - date: new Date(cat.key_as_string), - doc_count: cat.doc_count, - }; + get isLoading(): boolean { + return this.data.loading$.value || this.zoomedDataLoading$.value; } - /** Retrieve doc counts for a series. - * - * @param series series object - * @param setSearchRatio whether the `searchRatio` property of the series should be updated. - * True when retrieving results for the entire series, false when retrieving a window. - */ - requestSeriesDocCounts(queryModel: QueryModel) { - const aggregation = new DateHistogramAggregator( + initData(): TimelineData { + return new TimelineData( + this.queryModel, + this.comparedQueries, + this.frequencyMeasure, this.visualizedField, - this.currentTimeCategory + this.searchService, + this.apiService, + this.visualizationService, + this.destroy$, ); - return this.searchService.aggregateSearch(this.corpus, queryModel, aggregation); - } - - requestSeriesTermFrequency(series: TimelineSeries, queryModel: QueryModel) { - const bins = this.makeTermFrequencyBins(series); - return this.visualizationService.dateTermFrequencySearch( - this.corpus, - queryModel, - this.visualizedField.name, - bins, - this.currentTimeCategory - ); - } - - makeTermFrequencyBins(series: TimelineSeries) { - return series.data.map((bin, index) => { - const [minDate, maxDate] = this.categoryTimeDomain( - bin, - index, - series - ); - return { - start_date: minDate, - end_date: maxDate, - size: this.documentLimitForCategory(bin, series), - }; - }); - } - - /** time domain for a bin */ - categoryTimeDomain(cat, catIndex, series): [Date, Date] { - const startDate = cat.date; - const endDate = - catIndex < series.data.length - 1 - ? series.data[catIndex + 1].date - : undefined; - return [startDate, endDate]; - } - - fullDataRequest() { - const paramsPerSeries = this.rawData.map((series) => { - const queryModel = this.queryModelForSeries( - series, - this.queryModel - ); - const bins = this.makeTermFrequencyBins(series); - const unit = this.calculateTimeCategory(...this.xDomain); // use initial unit, not zoomed-in-status - return this.visualizationService.makeDateTermFrequencyParameters( - this.corpus, - queryModel, - this.visualizedField.name, - bins, - unit - ); - }); - return this.apiService.requestFullData({ - visualization: 'date_term_frequency', - parameters: paramsPerSeries, - corpus_name: this.corpus.name, - }); } - setChart() { - if (this.chart) { - // reset time unit to the one set in the chart - const unit = (this.chart.options.scales.x as any).time - .unit as TimeCategory; - if (unit) { - this.currentTimeCategory = unit; - } - this.updateChartData(); - } else { - this.initChart(); - } + setChart(data: typeof this.data.rawData$.value) { + this.viewTimeCategory = this.data.timeCategory; + super.setChart(data); } - getDatasets() { - return this.rawData.map((series, seriesIndex) => { + getDatasets(data: typeof this.data.rawData$.value) { + return data.map((series, seriesIndex) => { const data = this.chartDataFromSeries(series); return { type: this.chartType, @@ -178,8 +79,8 @@ export class TimelineComponent })); } - callibratexAxis(date: Date, margin: number = 1) { - switch(this.currentTimeCategory) { + addDateMargin(date: Date, margin: number = 1): Date { + switch (this.data.timeCategory) { case 'day': return addDays(date, margin); case 'week': @@ -188,7 +89,6 @@ export class TimelineComponent return addMonths(date, margin); case 'year': return addYears(date, margin); - } } @@ -196,8 +96,8 @@ export class TimelineComponent const xLabel = this.visualizedField.displayName ? this.visualizedField.displayName : this.visualizedField.name; - const xMin = this.callibratexAxis(this.xDomain[0], -1); - const xMax = this.callibratexAxis(this.xDomain[1]); + const xMin = this.addDateMargin(this.data.xDomain[0], -1); + const xMax = this.addDateMargin(this.data.xDomain[1]); const options = this.basicChartOptions; options.plugins.title.text = this.chartTitle(); @@ -206,7 +106,7 @@ export class TimelineComponent x.type = 'time'; (x as any).time = { minUnit: 'day', - unit: this.currentTimeCategory, + unit: this.data.timeCategory, }; x.min = xMin.toISOString(); x.max = xMax.toISOString(); @@ -246,19 +146,19 @@ export class TimelineComponent * underlying data. */ onZoomIn(chart, triggeredByDataUpdate = false) { - const initialTimeCategory = this.calculateTimeCategory(...this.xDomain); - const previousTimeCategory = this.currentTimeCategory; + const initialTimeCategory = this.data.timeCategory; + const previousTimeCategory = this.viewTimeCategory; const min = new Date(chart.scales.x.min); const max = new Date(chart.scales.x.max); - this.currentTimeCategory = this.calculateTimeCategory(min, max); + this.viewTimeCategory = this.data.calculateTimeCategory(min, max); if ( - this.currentTimeCategory !== previousTimeCategory || + this.viewTimeCategory !== previousTimeCategory || (triggeredByDataUpdate && - this.currentTimeCategory !== initialTimeCategory) + this.viewTimeCategory !== initialTimeCategory) ) { showLoading( - this.isLoading$, + this.zoomedDataLoading$, this.loadZoomedInData( chart, min, @@ -288,85 +188,33 @@ export class TimelineComponent // when zooming, hide data for smooth transition chart.update(triggeredByDataUpdate ? 'none' : 'hide'); - const dataPromises: Promise[] = chart.data.datasets.map( - (dataset, seriesIndex) => { - const series = this.rawData[seriesIndex]; - const queryModelCopy = this.addQueryDateFilter( - this.queryModel, - min, - max - ); - return this.getSeriesDocumentData( - series, - queryModelCopy, - false - ).then((result) => { - if (this.frequencyMeasure === 'tokens') { - return this.getTermFrequencies(result, queryModelCopy); - } else { - return result; - } - }); - } - ); - - const zoomedInResults = await Promise.all(dataPromises); - + const zoomedInResults = await this.data.zoomedInData(min, max); zoomedInResults.forEach((data, seriesIndex) => { chart.data.datasets[seriesIndex].data = this.chartDataFromSeries(data); }); - chart.options.scales.x.time.unit = this.currentTimeCategory; + chart.options.scales.x.time.unit = this.viewTimeCategory; chart.update('show'); // fade into view } - /** - * Add a date filter to a query model restricting it to the provided min and max values. - */ - addQueryDateFilter(query: QueryModel, min: Date, max: Date): QueryModel { - const queryModelCopy = query.clone(); - // download zoomed in results - const filter = this.visualizedField.makeSearchFilter(); - filter.set({ min, max }); - queryModelCopy.addFilter(filter); - return queryModelCopy; - } - /** trigger zoom out, update chart data accordingly */ zoomOut(): void { this.chart.resetZoom(); - this.currentTimeCategory = this.calculateTimeCategory(...this.xDomain); + this.viewTimeCategory = this.data.timeCategory; (this.chart.options.scales.x as any).time.unit = - this.currentTimeCategory; + this.viewTimeCategory; this.chart.update(); - this.setChart(); - } - - /** - * Get the time category (year/month/week/day) that should be used in the graph, - * based on minimum and maximum dates on the x axis. - */ - public calculateTimeCategory(min: Date, max: Date): TimeCategory { - const diff = differenceInDays(max, min); - if (diff <= this.scaleDownThreshold) { - return 'day'; - } else if (diff <= this.scaleDownThreshold * 7) { - return 'week'; - } else if (diff <= this.scaleDownThreshold * 30) { - return 'month'; - } else { - return 'year'; - } + this.setChart(this.data.rawData$.value); } - setTableHeaders() { + setTableHeaders(data: typeof this.data.rawData$.value) { const rightColumnName = this.normalizer === 'raw' ? 'Frequency' : 'Relative frequency'; const valueKey = this.currentValueKey; - if (this.rawData.length > 1) { + if (data.length > 1) { this.tableHeaders = [ { key: 'date', @@ -431,7 +279,7 @@ export class TimelineComponent // eslint-disable-next-line @typescript-eslint/member-ordering get formatDate(): (date) => string { let dateFormat: string; - switch (this.currentTimeCategory) { + switch (this.viewTimeCategory) { case 'year': dateFormat = 'yyyy'; break; diff --git a/frontend/src/app/visualization/visualization.component.html b/frontend/src/app/visualization/visualization.component.html index 9623b1084..708528253 100644 --- a/frontend/src/app/visualization/visualization.component.html +++ b/frontend/src/app/visualization/visualization.component.html @@ -91,27 +91,27 @@
- - + + /> - - + + />