Merge pull request #1683 from github/koesie10/fix-duplicate-downloads

Fix duplicate variant analysis results downloads

Author: koesie10

extensions/ql-vscode/src/remote-queries/variant-analysis-manager.ts

@@ -26,6 +26,8 @@ import { createTimestampFile, showAndLogErrorMessage } from '../helpers';
 import * as fs from 'fs-extra';
 
 export class VariantAnalysisManager extends DisposableObject implements VariantAnalysisViewManager<VariantAnalysisView> {
+  private static readonly REPO_STATES_FILENAME = 'repo_states.json';
+
   private readonly _onVariantAnalysisAdded = this.push(new EventEmitter<VariantAnalysis>());
   public readonly onVariantAnalysisAdded = this._onVariantAnalysisAdded.event;
   private readonly _onVariantAnalysisStatusUpdated = this.push(new EventEmitter<VariantAnalysis>());
@@ -40,6 +42,8 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
   private static readonly maxConcurrentDownloads = 3;
   private readonly queue = new PQueue({ concurrency: VariantAnalysisManager.maxConcurrentDownloads });
 
+  private readonly repoStates = new Map<number, Record<number, VariantAnalysisScannedRepositoryState>>();
+
   constructor(
     private readonly ctx: ExtensionContext,
     private readonly storagePath: string,
@@ -60,6 +64,15 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
       this._onVariantAnalysisRemoved.fire(variantAnalysis);
     } else {
       await this.setVariantAnalysis(variantAnalysis);
+
+      try {
+        const repoStates = await fs.readJson(this.getRepoStatesStoragePath(variantAnalysis.id));
+        this.repoStates.set(variantAnalysis.id, repoStates);
+      } catch (e) {
+        // Ignore this error, we simply might not have downloaded anything yet
+        this.repoStates.set(variantAnalysis.id, {});
+      }
+
       if (!await isVariantAnalysisComplete(variantAnalysis, this.makeResultDownloadChecker(variantAnalysis))) {
         void commands.executeCommand('codeQL.monitorVariantAnalysis', variantAnalysis);
       }
@@ -120,6 +133,10 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
     return this.variantAnalyses.get(variantAnalysisId);
   }
 
+  public async getRepoStates(variantAnalysisId: number): Promise<VariantAnalysisScannedRepositoryState[]> {
+    return Object.values(this.repoStates.get(variantAnalysisId) ?? {});
+  }
+
   public get variantAnalysesSize(): number {
     return this.variantAnalyses.size;
   }
@@ -152,6 +169,8 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
 
     await this.prepareStorageDirectory(variantAnalysis.id);
 
+    this.repoStates.set(variantAnalysis.id, {});
+
     this._onVariantAnalysisAdded.fire(variantAnalysis);
   }
 
@@ -166,6 +185,14 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
 
   private async onRepoStateUpdated(variantAnalysisId: number, repoState: VariantAnalysisScannedRepositoryState): Promise<void> {
     await this.getView(variantAnalysisId)?.updateRepoState(repoState);
+
+    let repoStates = this.repoStates.get(variantAnalysisId);
+    if (!repoStates) {
+      repoStates = {};
+      this.repoStates.set(variantAnalysisId, repoStates);
+    }
+
+    repoStates[repoState.repositoryId] = repoState;
   }
 
   public async monitorVariantAnalysis(
@@ -180,6 +207,10 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
     variantAnalysis: VariantAnalysis,
     cancellationToken: CancellationToken
   ): Promise<void> {
+    if (this.repoStates.get(variantAnalysis.id)?.[scannedRepo.repository.id]?.downloadStatus === VariantAnalysisScannedRepositoryDownloadStatus.Succeeded) {
+      return;
+    }
+
     const repoState = {
       repositoryId: scannedRepo.repository.id,
       downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Pending,
@@ -216,11 +247,19 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
       repoState.downloadStatus = VariantAnalysisScannedRepositoryDownloadStatus.InProgress;
       await this.onRepoStateUpdated(variantAnalysis.id, repoState);
 
-      await this.variantAnalysisResultsManager.download(credentials, variantAnalysis.id, repoTask, this.getVariantAnalysisStorageLocation(variantAnalysis.id));
+      try {
+        await this.variantAnalysisResultsManager.download(credentials, variantAnalysis.id, repoTask, this.getVariantAnalysisStorageLocation(variantAnalysis.id));
+      } catch (e) {
+        repoState.downloadStatus = VariantAnalysisScannedRepositoryDownloadStatus.Failed;
+        await this.onRepoStateUpdated(variantAnalysis.id, repoState);
+        throw new Error(`Could not download the results for variant analysis with id: ${variantAnalysis.id}. Error: ${getErrorMessage(e)}`);
+      }
     }
 
     repoState.downloadStatus = VariantAnalysisScannedRepositoryDownloadStatus.Succeeded;
     await this.onRepoStateUpdated(variantAnalysis.id, repoState);
+
+    await fs.outputJson(this.getRepoStatesStoragePath(variantAnalysis.id), this.repoStates.get(variantAnalysis.id));
   }
 
   public async enqueueDownload(
@@ -242,6 +281,13 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
     );
   }
 
+  private getRepoStatesStoragePath(variantAnalysisId: number): string {
+    return path.join(
+      this.getVariantAnalysisStorageLocation(variantAnalysisId),
+      VariantAnalysisManager.REPO_STATES_FILENAME
+    );
+  }
+
   /**
    * Prepares a directory for storing results for a variant analysis.
    * This directory contains a timestamp file, which will be

extensions/ql-vscode/src/remote-queries/variant-analysis-view-manager.ts

@@ -1,4 +1,4 @@
-import { VariantAnalysis } from './shared/variant-analysis';
+import { VariantAnalysis, VariantAnalysisScannedRepositoryState } from './shared/variant-analysis';
 
 export interface VariantAnalysisViewInterface {
   variantAnalysisId: number;
@@ -10,4 +10,5 @@ export interface VariantAnalysisViewManager<T extends VariantAnalysisViewInterfa
   unregisterView(view: T): void;
 
   getVariantAnalysis(variantAnalysisId: number): Promise<VariantAnalysis | undefined>;
+  getRepoStates(variantAnalysisId: number): Promise<VariantAnalysisScannedRepositoryState[]>;
 }

extensions/ql-vscode/src/remote-queries/variant-analysis-view.ts

@@ -127,6 +127,16 @@ export class VariantAnalysisView extends AbstractWebview<ToVariantAnalysisMessag
       t: 'setVariantAnalysis',
       variantAnalysis,
     });
+
+    const repoStates = await this.manager.getRepoStates(this.variantAnalysisId);
+    if (repoStates.length === 0) {
+      return;
+    }
+
+    await this.postMessage({
+      t: 'setRepoStates',
+      repoStates,
+    });
   }
 
   private async openQueryFile(): Promise<void> {

extensions/ql-vscode/src/view/variant-analysis/RepoRow.tsx

@@ -119,7 +119,7 @@ export const RepoRow = ({
     }
   }, [resultsLoaded, resultsLoading]);
 
-  const disabled = !status || !isCompletedAnalysisRepoStatus(status);
+  const disabled = !status || !isCompletedAnalysisRepoStatus(status) || (status === VariantAnalysisRepoStatus.Succeeded && downloadStatus !== VariantAnalysisScannedRepositoryDownloadStatus.Succeeded);
   const expandableContentLoaded = status && (status !== VariantAnalysisRepoStatus.Succeeded || resultsLoaded);
 
   return (

extensions/ql-vscode/src/view/variant-analysis/__tests__/RepoRow.spec.tsx

@@ -1,6 +1,9 @@
 import * as React from 'react';
 import { render as reactRender, screen } from '@testing-library/react';
-import { VariantAnalysisRepoStatus } from '../../../remote-queries/shared/variant-analysis';
+import {
+  VariantAnalysisRepoStatus,
+  VariantAnalysisScannedRepositoryDownloadStatus
+} from '../../../remote-queries/shared/variant-analysis';
 import userEvent from '@testing-library/user-event';
 import { RepoRow, RepoRowProps } from '../RepoRow';
 import { createMockRepositoryWithMetadata } from '../../../vscode-tests/factories/remote-queries/shared/repository';
@@ -50,7 +53,7 @@ describe(RepoRow.name, () => {
     })).toBeDisabled();
   });
 
-  it('renders the succeeded state', () => {
+  it('renders the succeeded state without download status', () => {
     render({
       status: VariantAnalysisRepoStatus.Succeeded,
       resultCount: 178,
@@ -60,6 +63,42 @@ describe(RepoRow.name, () => {
       name: 'Success',
     })).toBeInTheDocument();
     expect(screen.getByText('178')).toBeInTheDocument();
+    expect(screen.getByRole<HTMLButtonElement>('button', {
+      expanded: false
+    })).toBeDisabled();
+  });
+
+  it('renders the succeeded state with pending download status', () => {
+    render({
+      status: VariantAnalysisRepoStatus.Succeeded,
+      resultCount: 178,
+      downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Pending,
+    });
+
+    expect(screen.getByRole<HTMLButtonElement>('button', {
+      expanded: false
+    })).toBeDisabled();
+  });
+
+  it('renders the succeeded state with in progress download status', () => {
+    render({
+      status: VariantAnalysisRepoStatus.Succeeded,
+      resultCount: 178,
+      downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.InProgress,
+    });
+
+    expect(screen.getByRole<HTMLButtonElement>('button', {
+      expanded: false
+    })).toBeDisabled();
+  });
+
+  it('renders the succeeded state with succeeded download status', () => {
+    render({
+      status: VariantAnalysisRepoStatus.Succeeded,
+      resultCount: 178,
+      downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Succeeded,
+    });
+
     expect(screen.getByRole<HTMLButtonElement>('button', {
       expanded: false
     })).toBeEnabled();
@@ -217,6 +256,7 @@ describe(RepoRow.name, () => {
   it('can expand the repo item when succeeded and loaded', async () => {
     render({
       status: VariantAnalysisRepoStatus.Succeeded,
+      downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Succeeded,
       interpretedResults: [],
     });
 
@@ -232,6 +272,7 @@ describe(RepoRow.name, () => {
   it('can expand the repo item when succeeded and not loaded', async () => {
     const { rerender } = render({
       status: VariantAnalysisRepoStatus.Succeeded,
+      downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Succeeded,
     });
 
     await userEvent.click(screen.getByRole('button', {

extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisAnalyzedRepos.spec.tsx

@@ -3,6 +3,7 @@ import { render as reactRender, screen } from '@testing-library/react';
 import userEvent from '@testing-library/user-event';
 import {
   VariantAnalysisRepoStatus,
+  VariantAnalysisScannedRepositoryDownloadStatus,
   VariantAnalysisStatus
 } from '../../../remote-queries/shared/variant-analysis';
 import { VariantAnalysisAnalyzedRepos, VariantAnalysisAnalyzedReposProps } from '../VariantAnalysisAnalyzedRepos';
@@ -110,6 +111,12 @@ describe(VariantAnalysisAnalyzedRepos.name, () => {
 
   it('renders the interpreted result for a succeeded repo', async () => {
     render({
+      repositoryStates: [
+        {
+          repositoryId: 2,
+          downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Succeeded,
+        }
+      ],
       repositoryResults: [
         {
           variantAnalysisId: 1,