SONARPY-2451 Collect data for the Jupyter notebooks

SonarSource · Dec 13, 2024 · 5595ef4 · 5595ef4
1 parent 71d44d6
commit 5595ef4
Show file tree

Hide file tree

Showing 4 changed files with 127 additions and 10 deletions.
diff --git a/sonar-python-plugin/src/main/java/org/sonar/plugins/python/IPynbSensor.java b/sonar-python-plugin/src/main/java/org/sonar/plugins/python/IPynbSensor.java
@@ -47,17 +47,24 @@ public final class IPynbSensor implements Sensor {
   private final NoSonarFilter noSonarFilter;
   private final PythonIndexer indexer;
   private static final String FAIL_FAST_PROPERTY_NAME = "sonar.internal.analysis.failFast";
+  private final SensorTelemetryStorage sensorTelemetryStorage;
 
-  public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter) {
-    this(fileLinesContextFactory, checkFactory, noSonarFilter, null);
+  public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter, @Nullable PythonIndexer indexer) {
+    this(fileLinesContextFactory, checkFactory, noSonarFilter, indexer, new SensorTelemetryStorage());
   }
 
-  public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter, @Nullable PythonIndexer indexer) {
+  public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter mock, SensorTelemetryStorage sensorTelemetryStorage) {
+    this(fileLinesContextFactory, checkFactory, mock, null, sensorTelemetryStorage);
+  }
+
+  public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter, @Nullable PythonIndexer indexer,
+    SensorTelemetryStorage sensorTelemetryStorage) {
     this.checks = new PythonChecks(checkFactory)
       .addChecks(CheckList.IPYTHON_REPOSITORY_KEY, CheckList.getChecks());
     this.fileLinesContextFactory = fileLinesContextFactory;
     this.noSonarFilter = noSonarFilter;
     this.indexer = indexer;
+    this.sensorTelemetryStorage = sensorTelemetryStorage;
   }
 
   @Override
@@ -80,29 +87,40 @@ public void execute(SensorContext context) {
     } else {
       processNotebooksFiles(pythonFiles, context);
     }
+    sensorTelemetryStorage.send(context);
   }
 
   private void processNotebooksFiles(List<PythonInputFile> pythonFiles, SensorContext context) {
-    pythonFiles = parseNotebooks(pythonFiles, context);
+    pythonFiles = this.parseNotebooks(pythonFiles, context);
     // Disable caching for IPynb files for now see: SONARPY-2020
     CacheContext cacheContext = CacheContextImpl.dummyCache();
     PythonIndexer pythonIndexer = new SonarQubePythonIndexer(pythonFiles, cacheContext, context);
     PythonScanner scanner = new PythonScanner(context, checks, fileLinesContextFactory, noSonarFilter, PythonParser.createIPythonParser(), pythonIndexer);
     scanner.execute(pythonFiles, context);
+    sensorTelemetryStorage.updateMetric(SensorTelemetryStorage.MetricKey.NOTEBOOK_RECOGNITION_ERROR_KEY, String.valueOf(scanner.getRecognitionErrorCount()));
   }
 
-  private static List<PythonInputFile> parseNotebooks(List<PythonInputFile> pythonFiles, SensorContext context) {
+  private List<PythonInputFile> parseNotebooks(List<PythonInputFile> pythonFiles, SensorContext context) {
     List<PythonInputFile> generatedIPythonFiles = new ArrayList<>();
+
+    sensorTelemetryStorage.updateMetric(SensorTelemetryStorage.MetricKey.NOTEBOOK_TOTAL_KEY, String.valueOf(pythonFiles.size()));
+    var numberOfExceptions = 0;
+
     for (PythonInputFile inputFile : pythonFiles) {
       try {
+        sensorTelemetryStorage.updateMetric(SensorTelemetryStorage.MetricKey.NOTEBOOK_PRESENT_KEY, "1");
         var result = IpynbNotebookParser.parseNotebook(inputFile);
         result.ifPresent(generatedIPythonFiles::add);
       } catch (Exception e) {
+        numberOfExceptions++;
         if (context.config().getBoolean(FAIL_FAST_PROPERTY_NAME).orElse(false) && !isErrorOnTestFile(inputFile)) {
           throw new IllegalStateException("Exception when parsing " + inputFile, e);
         }
       }
     }
+
+    sensorTelemetryStorage.updateMetric(SensorTelemetryStorage.MetricKey.NOTEBOOK_EXCEPTION_KEY, String.valueOf(numberOfExceptions));
+
     return generatedIPythonFiles;
   }
 
@@ -123,4 +141,9 @@ private static List<PythonInputFile> getInputFiles(SensorContext context) {
   private static boolean isErrorOnTestFile(PythonInputFile inputFile) {
     return inputFile.wrappedFile().type() == InputFile.Type.TEST;
   }
+
+  public SensorTelemetryStorage getSensorTelemetryStorage() {
+    return sensorTelemetryStorage;
+  }
+
 }
diff --git a/sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java b/sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java
@@ -77,6 +77,7 @@ public class PythonScanner extends Scanner {
   private final PythonCpdAnalyzer cpdAnalyzer;
   private final PythonIndexer indexer;
   private final Map<PythonInputFile, Set<PythonCheck>> checksExecutedWithoutParsingByFiles = new HashMap<>();
+  private int recognitionErrorCount = 0;
 
   public PythonScanner(
     SensorContext context, PythonChecks checks,
@@ -123,6 +124,7 @@ protected void scanFile(PythonInputFile inputFile) throws IOException {
 
       LOG.error("Unable to parse file: " + inputFile);
       LOG.error(newMessage);
+      recognitionErrorCount++;
       context.newAnalysisError()
         .onFile(inputFile.wrappedFile())
         .at(inputFile.wrappedFile().newPointer(line, 0))
@@ -401,4 +403,8 @@ private static void addQuickFixes(InputFile inputFile, RuleKey ruleKey, Iterable
   private static TextRange rangeFromTextSpan(InputFile file, PythonTextEdit pythonTextEdit) {
     return file.newRange(pythonTextEdit.startLine(), pythonTextEdit.startLineOffset(), pythonTextEdit.endLine(), pythonTextEdit.endLineOffset());
   }
+
+  public int getRecognitionErrorCount() {
+    return recognitionErrorCount;
+  }
 }
diff --git a/sonar-python-plugin/src/main/java/org/sonar/plugins/python/SensorTelemetryStorage.java b/sonar-python-plugin/src/main/java/org/sonar/plugins/python/SensorTelemetryStorage.java
@@ -0,0 +1,66 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the Sonar Source-Available License Version 1, as published by SonarSource SA.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the Sonar Source-Available License for more details.
+ *
+ * You should have received a copy of the Sonar Source-Available License
+ * along with this program; if not, see https://sonarsource.com/license/ssal/
+ */
+package org.sonar.plugins.python;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.batch.sensor.SensorContext;
+
+public class SensorTelemetryStorage {
+  private static final Logger LOG = LoggerFactory.getLogger(SensorTelemetryStorage.class);
+
+  private final Map<String, String> data;
+
+  public SensorTelemetryStorage() {
+    data = new HashMap<>();
+  }
+
+  public Map<String, String> data() {
+    return Collections.unmodifiableMap(data);
+  }
+
+  public void send(SensorContext sensorContext) {
+    data.forEach((k, v) -> {
+      LOG.info("Metrics property: {}={}", k, v);
+      sensorContext.addTelemetryProperty(k, v);
+    });
+  }
+
+  public void updateMetric(MetricKey key, String value) {
+    data.put(key.key(), value);
+  }
+
+  public enum MetricKey {
+    NOTEBOOK_PRESENT_KEY("python.notebook.present"),
+    NOTEBOOK_TOTAL_KEY("python.notebook.total"),
+    NOTEBOOK_RECOGNITION_ERROR_KEY("python.notebook.recognition_error"),
+    NOTEBOOK_EXCEPTION_KEY("python.notebook.exceptions");
+
+    private final String key;
+
+    MetricKey(String key) {
+      this.key = key;
+    }
+
+    public String key() {
+      return key;
+    }
+  }
+}
diff --git a/sonar-python-plugin/src/test/java/org/sonar/plugins/python/IPynbSensorTest.java b/sonar-python-plugin/src/test/java/org/sonar/plugins/python/IPynbSensorTest.java
@@ -21,6 +21,7 @@
 import java.nio.file.Files;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -57,6 +58,8 @@
 import static org.junit.Assert.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 class IPynbSensorTest {
@@ -167,12 +170,16 @@ void test_python_version_parameter() {
     assertThat(ProjectPythonVersion.currentVersions()).containsExactly(PythonVersionUtils.Version.V_313);
   }
 
-  private IPynbSensor notebookSensor() {
+  private IPynbSensor notebookSensor(SensorTelemetryStorage sensorTelemetryStorage) {
     FileLinesContextFactory fileLinesContextFactory = mock(FileLinesContextFactory.class);
     FileLinesContext fileLinesContext = mock(FileLinesContext.class);
     when(fileLinesContextFactory.createFor(Mockito.any(InputFile.class))).thenReturn(fileLinesContext);
     CheckFactory checkFactory = new CheckFactory(activeRules);
-    return new IPynbSensor(fileLinesContextFactory, checkFactory, mock(NoSonarFilter.class));
+    return new IPynbSensor(fileLinesContextFactory, checkFactory, mock(NoSonarFilter.class), sensorTelemetryStorage);
+  }
+
+  private IPynbSensor notebookSensor() {
+    return notebookSensor(new SensorTelemetryStorage());
   }
 
   @Test
@@ -210,10 +217,19 @@ void test_notebook_sensor_cannot_parse_file() {
   }
 
   @Test
-  void test_notebook_sensor_is_excuted_on_json_file() {
+  void test_notebook_sensor_is_executed_on_json_file() {
     inputFile(NOTEBOOK_FILE);
     activeRules = new ActiveRulesBuilder().build();
-    assertDoesNotThrow(() -> notebookSensor().execute(context));
+    var sensorTelemetryStorage = spy(new SensorTelemetryStorage());
+    var sensor = spy(notebookSensor(sensorTelemetryStorage));
+    assertDoesNotThrow(() -> sensor.execute(context));
+    assertThat(sensor.getSensorTelemetryStorage().data())
+      .containsExactlyInAnyOrderEntriesOf(Map.of(
+        SensorTelemetryStorage.MetricKey.NOTEBOOK_PRESENT_KEY.key(), "1",
+        SensorTelemetryStorage.MetricKey.NOTEBOOK_RECOGNITION_ERROR_KEY.key(), "0",
+        SensorTelemetryStorage.MetricKey.NOTEBOOK_TOTAL_KEY.key(), "1",
+        SensorTelemetryStorage.MetricKey.NOTEBOOK_EXCEPTION_KEY.key(), "0"));
+    verify(sensorTelemetryStorage, Mockito.times(1)).send(context);
   }
 
   @Test
@@ -227,12 +243,18 @@ void test_notebook_sensor_does_not_execute_cpd_measures() {
   }
 
   @Test
-  void test_notebook_sensor_parse_error_on_valid_line(){
+  void test_notebook_sensor_parse_error_on_valid_line() {
     inputFile("notebook_parse_error.ipynb");
     activeRules = new ActiveRulesBuilder().build();
     var sensor = notebookSensor();
     sensor.execute(context);
     var logs = String.join("", logTester.logs());
     assertThat(logs).contains("Unable to parse file: notebook_parse_error.ipynbParse error at line 1");
+    assertThat(sensor.getSensorTelemetryStorage().data())
+      .containsExactlyInAnyOrderEntriesOf(Map.of(
+        SensorTelemetryStorage.MetricKey.NOTEBOOK_PRESENT_KEY.key(), "1",
+        SensorTelemetryStorage.MetricKey.NOTEBOOK_TOTAL_KEY.key(), "1",
+        SensorTelemetryStorage.MetricKey.NOTEBOOK_EXCEPTION_KEY.key(), "0",
+        SensorTelemetryStorage.MetricKey.NOTEBOOK_RECOGNITION_ERROR_KEY.key(), "1"));
   }
 }