Skip to content

Commit add9bed

Browse files
guillaume-dequennesonartech
guillaume-dequenne
authored and
sonartech
committed
SONARPY-3208 Add telemetry for parallel analysis (#489)
GitOrigin-RevId: fc284e985a4ca066f9bf1c11cefe5639485212a1
1 parent b8a9f4a commit add9bed

6 files changed

Lines changed: 82 additions & 2 deletions

File tree

‎python-commons/src/main/java/org/sonar/plugins/python/IPynbSensor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
*/
1717
package org.sonar.plugins.python;
1818

19+
import java.time.Duration;
20+
import java.time.Instant;
1921
import java.util.ArrayList;
2022
import java.util.Collections;
2123
import java.util.List;
@@ -107,6 +109,7 @@ public void describe(SensorDescriptor descriptor) {
107109

108110
@Override
109111
public void execute(SensorContext context) {
112+
Instant sensorStartTime = Instant.now();
110113
List<PythonInputFile> pythonFiles = getInputFiles(context);
111114
var pythonVersions = context.config().getStringArray(PYTHON_VERSION_KEY);
112115
if (pythonVersions.length != 0) {
@@ -118,6 +121,8 @@ public void execute(SensorContext context) {
118121
scanner.execute(pythonFiles, context);
119122
} else {
120123
processNotebooksFiles(pythonFiles, context);
124+
Duration sensorTime = Duration.between(sensorStartTime, Instant.now());
125+
sensorTelemetryStorage.updateMetric(TelemetryMetricKey.NOTEBOOKS_ANALYSIS_DURATION_KEY, sensorTime.getSeconds());
121126
}
122127

123128
sensorTelemetryStorage.updateMetric(TelemetryMetricKey.NOSONAR_RULE_ID_KEYS, noSonarLineInfoCollector.getSuppressedRuleIds());

‎python-commons/src/main/java/org/sonar/plugins/python/PythonSensor.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
package org.sonar.plugins.python;
1818

1919
import java.io.File;
20+
import java.time.Duration;
21+
import java.time.Instant;
2022
import java.util.ArrayList;
2123
import java.util.Collections;
2224
import java.util.List;
@@ -55,6 +57,8 @@
5557
import org.sonar.python.types.TypeShed;
5658
import org.sonarsource.performance.measure.PerformanceMeasure;
5759

60+
import static org.sonar.plugins.python.PythonScanner.THREADS_PROPERTY_NAME;
61+
import static org.sonar.plugins.python.Scanner.PARALLEL_PROPERTY_NAME;
5862
import static org.sonar.plugins.python.api.PythonVersionUtils.PYTHON_VERSION_KEY;
5963

6064
@DependedUpon(value = "org.sonar.plugins.python.PythonSensor_before_com.sonarsource.dbd.SonarLintPythonBugDetectionSensor")
@@ -126,6 +130,7 @@ public void describe(SensorDescriptor descriptor) {
126130

127131
@Override
128132
public void execute(SensorContext context) {
133+
Instant sensorStartTime = Instant.now();
129134
PerformanceMeasure.Duration durationReport = createPerformanceMeasureReport(context);
130135
List<PythonInputFile> pythonFiles = getInputFiles(context);
131136
String[] pythonVersionParameter = context.config().getStringArray(PYTHON_VERSION_KEY);
@@ -144,13 +149,23 @@ public void execute(SensorContext context) {
144149
PythonScanner scanner = new PythonScanner(context, checks, fileLinesContextFactory, noSonarFilter, PythonParser::create,
145150
pythonIndexer, architectureCallback, noSonarLineInfoCollector);
146151
scanner.execute(pythonFiles, context);
152+
Duration sensorTime = Duration.between(sensorStartTime, Instant.now());
147153

148154
updateDatabricksTelemetry(scanner);
149155
sensorTelemetryStorage.updateMetric(TelemetryMetricKey.NOSONAR_RULE_ID_KEYS, noSonarLineInfoCollector.getSuppressedRuleIds());
156+
updatePerformanceTelemetry(context, pythonFiles, scanner, sensorTime);
150157
sensorTelemetryStorage.send(context);
151158
durationReport.stop();
152159
}
153160

161+
private void updatePerformanceTelemetry(SensorContext context, List<PythonInputFile> pythonFiles, PythonScanner scanner, Duration sensorTime) {
162+
context.config().getInt(THREADS_PROPERTY_NAME).ifPresent(v -> sensorTelemetryStorage.updateMetric(TelemetryMetricKey.ANALYSIS_THREADS_PARAM_KEY, v));
163+
context.config().getBoolean(PARALLEL_PROPERTY_NAME).ifPresent(v -> sensorTelemetryStorage.updateMetric(TelemetryMetricKey.PARALLEL_ANALYSIS_KEY, v));
164+
sensorTelemetryStorage.updateMetric(TelemetryMetricKey.PYTHON_NUMBER_OF_FILES_KEY, pythonFiles.size());
165+
sensorTelemetryStorage.updateMetric(TelemetryMetricKey.ANALYSIS_THREADS_KEY, scanner.getNumberOfThreads(context));
166+
sensorTelemetryStorage.updateMetric(TelemetryMetricKey.ANALYSIS_DURATION_KEY, sensorTime.toSeconds());
167+
}
168+
154169
private static List<PythonInputFile> getInputFiles(SensorContext context) {
155170
FilePredicates p = context.fileSystem().predicates();
156171
Iterable<InputFile> it = context.fileSystem().inputFiles(p.and(p.hasLanguage(Python.KEY)));

‎python-commons/src/main/java/org/sonar/plugins/python/SensorTelemetryStorage.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public void updateMetric(TelemetryMetricKey key, String value) {
5151
data.put(key, value);
5252
}
5353

54-
public void updateMetric(TelemetryMetricKey key, int value) {
54+
public void updateMetric(TelemetryMetricKey key, Number value) {
5555
data.put(key, String.valueOf(value));
5656
}
5757

‎python-commons/src/main/java/org/sonar/plugins/python/TelemetryMetricKey.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,13 @@ public enum TelemetryMetricKey {
2828
PYTHON_DEPENDENCIES("python.dependencies"),
2929
PYTHON_DEPENDENCIES_FORMAT_VERSION("python.dependencies.format_version"),
3030
NOSONAR_RULE_ID_KEYS("python.nosonar.rule_ids"),
31-
NOSONAR_COMMENTS_KEYS("python.nosonar.comments");
31+
NOSONAR_COMMENTS_KEYS("python.nosonar.comments"),
32+
ANALYSIS_THREADS_PARAM_KEY("python.analysis.threads.parameter"),
33+
ANALYSIS_THREADS_KEY("python.analysis.threads.actual"),
34+
PARALLEL_ANALYSIS_KEY("python.analysis.parallel"),
35+
ANALYSIS_DURATION_KEY("python.analysis.duration"),
36+
NOTEBOOKS_ANALYSIS_DURATION_KEY("python.notebooks.analysis.duration"),
37+
PYTHON_NUMBER_OF_FILES_KEY("python.files.count");
3238

3339
private final String key;
3440

‎python-commons/src/test/java/org/sonar/plugins/python/IPynbSensorTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@
5959
import static org.assertj.core.api.Assertions.assertThat;
6060
import static org.junit.Assert.assertThrows;
6161
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
62+
import static org.mockito.ArgumentMatchers.anyString;
63+
import static org.mockito.ArgumentMatchers.eq;
6264
import static org.mockito.Mockito.mock;
6365
import static org.mockito.Mockito.spy;
6466
import static org.mockito.Mockito.verify;
@@ -235,6 +237,7 @@ void test_notebook_sensor_is_executed_on_json_file() {
235237
verify(contextSpy, Mockito.times(1)).addTelemetryProperty(TelemetryMetricKey.NOTEBOOK_RECOGNITION_ERROR_KEY.key(), "0");
236238
verify(contextSpy, Mockito.times(1)).addTelemetryProperty(TelemetryMetricKey.NOTEBOOK_TOTAL_KEY.key(), "1");
237239
verify(contextSpy, Mockito.times(1)).addTelemetryProperty(TelemetryMetricKey.NOTEBOOK_EXCEPTION_KEY.key(), "0");
240+
verify(contextSpy, Mockito.times(1)).addTelemetryProperty(eq(TelemetryMetricKey.NOTEBOOKS_ANALYSIS_DURATION_KEY.key()), anyString());
238241
}
239242

240243
@Test

‎python-commons/src/test/java/org/sonar/plugins/python/PythonSensorTest.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@
116116
import static org.assertj.core.api.Assertions.assertThat;
117117
import static org.assertj.core.api.Assertions.assertThatThrownBy;
118118
import static org.mockito.ArgumentMatchers.any;
119+
import static org.mockito.ArgumentMatchers.anyString;
120+
import static org.mockito.ArgumentMatchers.eq;
119121
import static org.mockito.Mockito.mock;
120122
import static org.mockito.Mockito.spy;
121123
import static org.mockito.Mockito.times;
@@ -1557,6 +1559,55 @@ void detects_databricks_negative() {
15571559
verify(spyContext, times(1)).addTelemetryProperty(TelemetryMetricKey.PYTHON_DATABRICKS_FOUND.key(), "0");
15581560
}
15591561

1562+
@Test
1563+
void send_telemetry_threads_and_files() {
1564+
activeRules = new ActiveRulesBuilder()
1565+
.addRule(new NewActiveRule.Builder()
1566+
.setRuleKey(RuleKey.of(PythonRuleRepository.REPOSITORY_KEY, "S930"))
1567+
.build())
1568+
.build();
1569+
1570+
inputFile(FILE_1);
1571+
inputFile(FILE_2);
1572+
1573+
context.setSettings(new MapSettings()
1574+
.setProperty("sonar.python.analysis.threads", "2")
1575+
.setProperty("sonar.python.analysis.parallel", true)
1576+
);
1577+
var contextSpy = spy(context);
1578+
PythonSensor sensor = sensor();
1579+
sensor.execute(contextSpy);
1580+
verify(contextSpy, times(1)).addTelemetryProperty(TelemetryMetricKey.ANALYSIS_THREADS_PARAM_KEY.key(), "2");
1581+
verify(contextSpy, times(1)).addTelemetryProperty(TelemetryMetricKey.PARALLEL_ANALYSIS_KEY.key(), "1");
1582+
verify(contextSpy, times(1)).addTelemetryProperty(TelemetryMetricKey.PYTHON_NUMBER_OF_FILES_KEY.key(), "2");
1583+
verify(contextSpy, Mockito.times(1)).addTelemetryProperty(eq(TelemetryMetricKey.ANALYSIS_DURATION_KEY.key()), anyString());
1584+
}
1585+
1586+
@Test
1587+
void send_telemetry_threads_disabled() {
1588+
activeRules = new ActiveRulesBuilder()
1589+
.addRule(new NewActiveRule.Builder()
1590+
.setRuleKey(RuleKey.of(PythonRuleRepository.REPOSITORY_KEY, "S930"))
1591+
.build())
1592+
.build();
1593+
1594+
inputFile(FILE_1);
1595+
inputFile(FILE_2);
1596+
1597+
context.setSettings(new MapSettings()
1598+
.setProperty("sonar.python.analysis.threads", "2")
1599+
.setProperty("sonar.python.analysis.parallel", false)
1600+
);
1601+
var contextSpy = spy(context);
1602+
PythonSensor sensor = sensor();
1603+
sensor.execute(contextSpy);
1604+
verify(contextSpy, times(1)).addTelemetryProperty(TelemetryMetricKey.ANALYSIS_THREADS_PARAM_KEY.key(), "2");
1605+
verify(contextSpy, times(1)).addTelemetryProperty(TelemetryMetricKey.ANALYSIS_THREADS_KEY.key(), "1");
1606+
verify(contextSpy, times(1)).addTelemetryProperty(TelemetryMetricKey.PARALLEL_ANALYSIS_KEY.key(), "0");
1607+
verify(contextSpy, times(1)).addTelemetryProperty(TelemetryMetricKey.PYTHON_NUMBER_OF_FILES_KEY.key(), "2");
1608+
verify(contextSpy, Mockito.times(1)).addTelemetryProperty(eq(TelemetryMetricKey.ANALYSIS_DURATION_KEY.key()), anyString());
1609+
}
1610+
15601611
private com.sonar.sslr.api.Token passToken(URI uri) {
15611612
return com.sonar.sslr.api.Token.builder()
15621613
.setType(PythonKeyword.PASS)

0 commit comments

Comments
 (0)