diff --git a/docker/importer/importer.py b/docker/importer/importer.py index eb93521049b..3bd1afeff73 100755 --- a/docker/importer/importer.py +++ b/docker/importer/importer.py @@ -218,11 +218,13 @@ def _infer_id_from_invalid_data(self, name: str, content: bytes) -> str: def _record_quality_finding( self, + source: osv.SourceRepository.name, bug_id: str, maybe_new_finding: osv.ImportFindings = osv.ImportFindings.INVALID_JSON): """Record the quality finding about a record in Datastore. Args: + source: the name of the source of the vulnerability record bug_id: the ID of the vulnerability maybe_new_finding: the finding to record @@ -240,6 +242,7 @@ def _record_quality_finding( else: osv.ImportFinding( bug_id=bug_id, + source=source, findings=[maybe_new_finding], first_seen=findingtimenow, last_attempt=findingtimenow).put() @@ -575,7 +578,7 @@ def _process_updates_git(self, source_repo: osv.SourceRepository): content = f.read() bug_id = self._infer_id_from_invalid_data( os.path.basename(path), content) - self._record_quality_finding(bug_id) + self._record_quality_finding(source_repo.name, bug_id) # Don't include error stack trace as that might leak sensitive info import_failure_logs.append('Failed to parse vulnerability "' + path + '"') @@ -660,7 +663,7 @@ def _process_updates_bucket(self, source_repo: osv.SourceRepository): # This feels gross to redownload it again. bug_id = self._infer_id_from_invalid_data(blob.name, blob.download_as_bytes()) - self._record_quality_finding(bug_id) + self._record_quality_finding(source_repo.name, bug_id) import_failure_logs.append( 'Failed to parse vulnerability (when considering for import) "' + blob.name + '"') @@ -880,7 +883,7 @@ def _process_updates_rest(self, source_repo: osv.SourceRepository): bug_id = self._infer_id_from_invalid_data( source_repo.link + vuln.id + source_repo.extension, single_vuln.content) - self._record_quality_finding(bug_id) + self._record_quality_finding(source_repo.name, bug_id) logging.info('Requesting analysis of REST record: %s', vuln.id + source_repo.extension) self._request_analysis_external( diff --git a/docker/importer/importer_test.py b/docker/importer/importer_test.py index e19be076562..62472562099 100644 --- a/docker/importer/importer_test.py +++ b/docker/importer/importer_test.py @@ -233,6 +233,7 @@ def test_invalid(self, mock_publish: mock.MagicMock): self.assertIn( osv.ImportFinding( bug_id='OSV-2017-145', + source='oss-fuzz', findings=[osv.ImportFindings.INVALID_JSON], first_seen=importer.utcnow(), last_attempt=importer.utcnow()).to_dict(), @@ -557,6 +558,7 @@ def test_bucket(self, unused_mock_time: mock.MagicMock, self.assertIn( osv.ImportFinding( bug_id='GO-2021-0085', + source='test', findings=[osv.ImportFindings.INVALID_JSON], first_seen=importer.utcnow(), last_attempt=importer.utcnow()).to_dict(), @@ -1108,6 +1110,7 @@ class ImportFindingsTest(unittest.TestCase): def setUp(self): tests.reset_emulator() + self.tmp_dir = tempfile.mkdtemp() tests.mock_datetime(self) warnings.filterwarnings('ignore', category=SystemTimeWarning) @@ -1122,10 +1125,16 @@ def test_add_finding(self): ], first_seen=importer.utcnow(), last_attempt=importer.utcnow(), - ) - expected.put() + ).to_dict() + + imp = importer.Importer('fake_public_key', 'fake_private_key', self.tmp_dir, + importer.DEFAULT_PUBLIC_LOGGING_BUCKET, 'bucket', + False, False) + # pylint: disable-next=protected-access + imp._record_quality_finding('cve-osv', 'CVE-2024-1234', + osv.ImportFindings.INVALID_VERSION) - actual = osv.ImportFinding.get_by_id(expected.bug_id) + actual = osv.ImportFinding.get_by_id(expected['bug_id']).to_dict() self.assertEqual(expected, actual)