Skip to content

Commit 4caa1e2

Browse files
koesie10koesie10
committed
Merge remote-tracking branch 'origin/main' into koesie10/export-results
2 parents 42c642d + 7e8fa5c commit 4caa1e2

File tree

12 files changed

+656
-602
lines changed

12 files changed

+656
-602
lines changed

‎.github/workflows/main.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ jobs:
144144
strategy:
145145
matrix:
146146
os: [ubuntu-latest, windows-latest]
147-
version: ['v2.7.6', 'v2.8.5', 'v2.9.4', 'v2.10.5', 'v2.11.2', 'nightly']
147+
version: ['v2.7.6', 'v2.8.5', 'v2.9.4', 'v2.10.5', 'v2.11.3', 'nightly']
148148
env:
149149
CLI_VERSION: ${{ matrix.version }}
150150
NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}

‎extensions/ql-vscode/src/extension.ts‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import {
3333
CliConfigListener,
3434
DistributionConfigListener,
3535
isCanary,
36+
isVariantAnalysisLiveResultsEnabled,
3637
joinOrderWarningThreshold,
3738
MAX_QUERIES,
3839
QueryHistoryConfigListener,
@@ -493,13 +494,13 @@ async function activateWithInstalledDistribution(
493494
const variantAnalysisStorageDir = path.join(ctx.globalStorageUri.fsPath, 'variant-analyses');
494495
await fs.ensureDir(variantAnalysisStorageDir);
495496
const variantAnalysisResultsManager = new VariantAnalysisResultsManager(cliServer, logger);
496-
const variantAnalysisManager = new VariantAnalysisManager(ctx, variantAnalysisStorageDir, variantAnalysisResultsManager);
497+
const variantAnalysisManager = new VariantAnalysisManager(ctx, cliServer, variantAnalysisStorageDir, variantAnalysisResultsManager);
497498
ctx.subscriptions.push(variantAnalysisManager);
498499
ctx.subscriptions.push(variantAnalysisResultsManager);
499500
ctx.subscriptions.push(workspace.registerTextDocumentContentProvider('codeql-variant-analysis', createVariantAnalysisContentProvider(variantAnalysisManager)));
500501

501502
void logger.log('Initializing remote queries manager.');
502-
const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger, variantAnalysisManager);
503+
const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger);
503504
ctx.subscriptions.push(rqm);
504505

505506
void logger.log('Initializing query history.');
@@ -910,11 +911,20 @@ async function activateWithInstalledDistribution(
910911
step: 0,
911912
message: 'Getting credentials'
912913
});
913-
await rqm.runRemoteQuery(
914-
uri || window.activeTextEditor?.document.uri,
915-
progress,
916-
token
917-
);
914+
915+
if (isVariantAnalysisLiveResultsEnabled()) {
916+
await variantAnalysisManager.runVariantAnalysis(
917+
uri || window.activeTextEditor?.document.uri,
918+
progress,
919+
token
920+
);
921+
} else {
922+
await rqm.runRemoteQuery(
923+
uri || window.activeTextEditor?.document.uri,
924+
progress,
925+
token
926+
);
927+
}
918928
} else {
919929
throw new Error('Variant analysis requires the CodeQL Canary version to run.');
920930
}

‎extensions/ql-vscode/src/remote-queries/remote-queries-api.ts‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import * as os from 'os';
2+
import { Credentials } from '../authentication';
3+
import { RepositorySelection } from './repository-selection';
4+
import { Repository } from './shared/repository';
5+
import { RemoteQueriesResponse } from './gh-api/remote-queries';
6+
import * as ghApiClient from './gh-api/gh-api-client';
7+
import { showAndLogErrorMessage, showAndLogInformationMessage } from '../helpers';
8+
import { getErrorMessage } from '../pure/helpers-pure';
9+
import { pluralize } from '../pure/word';
10+
11+
export async function runRemoteQueriesApiRequest(
12+
credentials: Credentials,
13+
ref: string,
14+
language: string,
15+
repoSelection: RepositorySelection,
16+
controllerRepo: Repository,
17+
queryPackBase64: string,
18+
): Promise<void | RemoteQueriesResponse> {
19+
try {
20+
const response = await ghApiClient.submitRemoteQueries(credentials, {
21+
ref,
22+
language,
23+
repositories: repoSelection.repositories,
24+
repositoryLists: repoSelection.repositoryLists,
25+
repositoryOwners: repoSelection.owners,
26+
queryPack: queryPackBase64,
27+
controllerRepoId: controllerRepo.id,
28+
});
29+
const { popupMessage, logMessage } = parseResponse(controllerRepo, response);
30+
void showAndLogInformationMessage(popupMessage, { fullMessage: logMessage });
31+
return response;
32+
} catch (error: any) {
33+
if (error.status === 404) {
34+
void showAndLogErrorMessage(`Controller repository was not found. Please make sure it's a valid repo name.${eol}`);
35+
} else {
36+
void showAndLogErrorMessage(getErrorMessage(error));
37+
}
38+
}
39+
}
40+
41+
const eol = os.EOL;
42+
const eol2 = os.EOL + os.EOL;
43+
44+
// exported for testing only
45+
export function parseResponse(controllerRepo: Repository, response: RemoteQueriesResponse) {
46+
const repositoriesQueried = response.repositories_queried;
47+
const repositoryCount = repositoriesQueried.length;
48+
49+
const popupMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. [Click here to see the progress](https://github.com/${controllerRepo.fullName}/actions/runs/${response.workflow_run_id}).`
50+
+ (response.errors ? `${eol2}Some repositories could not be scheduled. See extension log for details.` : '');
51+
52+
let logMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. See https://github.com/${controllerRepo.fullName}/actions/runs/${response.workflow_run_id}.`;
53+
logMessage += `${eol2}Repositories queried:${eol}${repositoriesQueried.join(', ')}`;
54+
if (response.errors) {
55+
const { invalid_repositories, repositories_without_database, private_repositories, cutoff_repositories, cutoff_repositories_count } = response.errors;
56+
logMessage += `${eol2}Some repositories could not be scheduled.`;
57+
if (invalid_repositories?.length) {
58+
logMessage += `${eol2}${pluralize(invalid_repositories.length, 'repository', 'repositories')} invalid and could not be found:${eol}${invalid_repositories.join(', ')}`;
59+
}
60+
if (repositories_without_database?.length) {
61+
logMessage += `${eol2}${pluralize(repositories_without_database.length, 'repository', 'repositories')} did not have a CodeQL database available:${eol}${repositories_without_database.join(', ')}`;
62+
logMessage += `${eol}For each public repository that has not yet been added to the database service, we will try to create a database next time the store is updated.`;
63+
}
64+
if (private_repositories?.length) {
65+
logMessage += `${eol2}${pluralize(private_repositories.length, 'repository', 'repositories')} not public:${eol}${private_repositories.join(', ')}`;
66+
logMessage += `${eol}When using a public controller repository, only public repositories can be queried.`;
67+
}
68+
if (cutoff_repositories_count) {
69+
logMessage += `${eol2}${pluralize(cutoff_repositories_count, 'repository', 'repositories')} over the limit for a single request`;
70+
if (cutoff_repositories) {
71+
logMessage += `:${eol}${cutoff_repositories.join(', ')}`;
72+
if (cutoff_repositories_count !== cutoff_repositories.length) {
73+
const moreRepositories = cutoff_repositories_count - cutoff_repositories.length;
74+
logMessage += `${eol}...${eol}And another ${pluralize(moreRepositories, 'repository', 'repositories')}.`;
75+
}
76+
} else {
77+
logMessage += '.';
78+
}
79+
logMessage += `${eol}Repositories were selected based on how recently they had been updated.`;
80+
}
81+
}
82+
83+
return {
84+
popupMessage,
85+
logMessage
86+
};
87+
}

‎extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts‎

Lines changed: 41 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { CancellationToken, commands, EventEmitter, ExtensionContext, Uri, env, window } from 'vscode';
1+
import { CancellationToken, commands, EventEmitter, ExtensionContext, Uri, env } from 'vscode';
22
import { nanoid } from 'nanoid';
33
import * as path from 'path';
44
import * as fs from 'fs-extra';
@@ -9,9 +9,11 @@ import { CodeQLCliServer } from '../cli';
99
import { ProgressCallback } from '../commandRunner';
1010
import { createTimestampFile, showAndLogErrorMessage, showAndLogInformationMessage, showInformationMessageWithAction } from '../helpers';
1111
import { Logger } from '../logging';
12-
import { runRemoteQuery } from './run-remote-query';
12+
import {
13+
prepareRemoteQueryRun,
14+
} from './run-remote-query';
1315
import { RemoteQueriesView } from './remote-queries-view';
14-
import { RemoteQuery } from './remote-query';
16+
import { buildRemoteQueryEntity, RemoteQuery } from './remote-query';
1517
import { RemoteQueriesMonitor } from './remote-queries-monitor';
1618
import { getRemoteQueryIndex, getRepositoriesMetadata, RepositoriesMetadata } from './gh-api/gh-actions-api-client';
1719
import { RemoteQueryResultIndex } from './remote-query-result-index';
@@ -22,7 +24,7 @@ import { assertNever } from '../pure/helpers-pure';
2224
import { QueryStatus } from '../query-status';
2325
import { DisposableObject } from '../pure/disposable-object';
2426
import { AnalysisResults } from './shared/analysis-result';
25-
import { VariantAnalysisManager } from './variant-analysis-manager';
27+
import { runRemoteQueriesApiRequest } from './remote-queries-api';
2628

2729
const autoDownloadMaxSize = 300 * 1024;
2830
const autoDownloadMaxCount = 100;
@@ -57,21 +59,18 @@ export class RemoteQueriesManager extends DisposableObject {
5759

5860
private readonly remoteQueriesMonitor: RemoteQueriesMonitor;
5961
private readonly analysesResultsManager: AnalysesResultsManager;
60-
private readonly variantAnalysisManager: VariantAnalysisManager;
6162
private readonly view: RemoteQueriesView;
6263

6364
constructor(
6465
private readonly ctx: ExtensionContext,
6566
private readonly cliServer: CodeQLCliServer,
6667
private readonly storagePath: string,
6768
logger: Logger,
68-
variantAnalysisManager: VariantAnalysisManager,
6969
) {
7070
super();
7171
this.analysesResultsManager = new AnalysesResultsManager(ctx, cliServer, storagePath, logger);
7272
this.view = new RemoteQueriesView(ctx, logger, this.analysesResultsManager);
7373
this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger);
74-
this.variantAnalysisManager = variantAnalysisManager;
7574

7675
this.remoteQueryAddedEventEmitter = this.push(new EventEmitter<NewQueryEvent>());
7776
this.remoteQueryRemovedEventEmitter = this.push(new EventEmitter<RemovedQueryEvent>());
@@ -122,18 +121,42 @@ export class RemoteQueriesManager extends DisposableObject {
122121
): Promise<void> {
123122
const credentials = await Credentials.initialize(this.ctx);
124123

125-
const querySubmission = await runRemoteQuery(this.cliServer, credentials, uri || window.activeTextEditor?.document.uri, progress, token, this.variantAnalysisManager);
126-
127-
if (querySubmission?.query) {
128-
const query = querySubmission.query;
129-
const queryId = this.createQueryId();
130-
131-
await this.prepareStorageDirectory(queryId);
132-
await this.storeJsonFile(queryId, 'query.json', query);
133-
134-
this.remoteQueryAddedEventEmitter.fire({ queryId, query });
135-
void commands.executeCommand('codeQL.monitorRemoteQuery', queryId, query);
124+
const {
125+
actionBranch,
126+
base64Pack,
127+
repoSelection,
128+
queryFile,
129+
queryMetadata,
130+
controllerRepo,
131+
queryStartTime,
132+
language,
133+
} = await prepareRemoteQueryRun(this.cliServer, credentials, uri, progress, token);
134+
135+
const apiResponse = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, controllerRepo, base64Pack);
136+
137+
if (!apiResponse) {
138+
return;
136139
}
140+
141+
const workflowRunId = apiResponse.workflow_run_id;
142+
const repositoryCount = apiResponse.repositories_queried.length;
143+
const query = await buildRemoteQueryEntity(
144+
queryFile,
145+
queryMetadata,
146+
controllerRepo,
147+
queryStartTime,
148+
workflowRunId,
149+
language,
150+
repositoryCount
151+
);
152+
153+
const queryId = this.createQueryId();
154+
155+
await this.prepareStorageDirectory(queryId);
156+
await this.storeJsonFile(queryId, 'query.json', query);
157+
158+
this.remoteQueryAddedEventEmitter.fire({ queryId, query });
159+
void commands.executeCommand('codeQL.monitorRemoteQuery', queryId, query);
137160
}
138161

139162
public async monitorRemoteQuery(

‎extensions/ql-vscode/src/remote-queries/remote-query.ts‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,44 @@
1-
import { Repository } from './repository';
1+
import * as fs from 'fs-extra';
2+
import { Repository as RemoteRepository } from './repository';
3+
import { QueryMetadata } from '../pure/interface-types';
4+
import { getQueryName } from './run-remote-query';
5+
import { Repository } from './shared/repository';
26

37
export interface RemoteQuery {
48
queryName: string;
59
queryFilePath: string;
610
queryText: string;
711
language: string;
8-
controllerRepository: Repository;
12+
controllerRepository: RemoteRepository;
913
executionStartTime: number; // Use number here since it needs to be serialized and desserialized.
1014
actionsWorkflowRunId: number;
1115
repositoryCount: number;
1216
}
17+
18+
export async function buildRemoteQueryEntity(
19+
queryFilePath: string,
20+
queryMetadata: QueryMetadata | undefined,
21+
controllerRepo: Repository,
22+
queryStartTime: number,
23+
workflowRunId: number,
24+
language: string,
25+
repositoryCount: number
26+
): Promise<RemoteQuery> {
27+
const queryName = getQueryName(queryMetadata, queryFilePath);
28+
const queryText = await fs.readFile(queryFilePath, 'utf8');
29+
const [owner, name] = controllerRepo.fullName.split('/');
30+
31+
return {
32+
queryName,
33+
queryFilePath,
34+
queryText,
35+
language,
36+
controllerRepository: {
37+
owner,
38+
name,
39+
},
40+
executionStartTime: queryStartTime,
41+
actionsWorkflowRunId: workflowRunId,
42+
repositoryCount,
43+
};
44+
}

0 commit comments

Comments
 (0)