Skip to content

Commit

Permalink
Remove 'manual number' as supported source for 'test cases'.
Browse files Browse the repository at this point in the history 
The 'test cases' metric had 'manual number' as supported source for the metric, but that could never work since the metric needs sources that contain test cases with identifiers so they can be matched.

Fixes #9793.
  • Loading branch information
@fniessink
fniessink committed Sep 19, 2024
1 parent fe098f4 commit 8570430
Show file tree
Hide file tree
Showing 6 changed files with 85 additions and 25 deletions.
13 changes: 13 additions & 0 deletions components/api_server/src/initialization/migrations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ def perform_migrations(database: Database) -> None:
changes := [
change_accessibility_violation_metrics_to_violations(report),
fix_branch_parameters_without_value(report),
remove_test_cases_manual_number(report),
change_ci_subject_types_to_development_environment(report),
change_sonarqube_parameters(report),
]
Expand Down Expand Up @@ -59,6 +60,18 @@ def fix_branch_parameters_without_value(report) -> str:
return change


def remove_test_cases_manual_number(report) -> str:
"""Remove manual number sources from test cases metrics."""
# Added after Quality-time v5.15.0, see https://github.com/ICTU/quality-time/issues/9793
change = ""
for metric in metrics(report, metric_types=("test_cases",)):
for source_uuid, source in metric.get("sources", {}).copy().items():
if source["type"] == "manual_number":
del metric["sources"][source_uuid]
change = "remove manual number sources from test cases metrics"
return change


METRICS_WITH_SOURCES_WITH_BRANCH_PARAMETER = {
"commented_out_code": ["sonarqube"],
"complex_units": ["sonarqube"],
Expand Down
47 changes: 45 additions & 2 deletions components/api_server/tests/initialization/test_migrations.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Unit tests for database migrations."""

from collections.abc import Mapping

from initialization.migrations import perform_migrations

from tests.base import DataModelTestCase, disable_logging
Expand All @@ -15,7 +17,7 @@ def existing_report(
metric_type: str,
metric_name: str = "",
metric_unit: str = "",
sources: dict[SourceId, dict[str, str | dict[str, str]]] | None = None,
sources: Mapping[SourceId, Mapping[str, str | Mapping[str, str]]] | None = None,
):
"""Return a report fixture. To be extended in subclasses."""
report: dict = {
Expand All @@ -27,7 +29,7 @@ def existing_report(
report["subjects"][SUBJECT_ID]["metrics"][METRIC_ID]["name"] = metric_name
if metric_unit:
report["subjects"][SUBJECT_ID]["metrics"][METRIC_ID]["unit"] = metric_unit
if sources:
if sources is not None:
report["subjects"][SUBJECT_ID]["metrics"][METRIC_ID]["sources"] = sources
return report

Expand Down Expand Up @@ -308,3 +310,44 @@ def test_report_with_security_types_parameter_without_values(self):
self.database.reports.find.return_value = reports
perform_migrations(self.database)
self.database.reports.replace_one.assert_not_called()


class TestCasesManualNumberTest(MigrationTestCase):
"""Unit tests for the test cases manual number source migration."""

def sources(self, source_type: str = "manual_number", source_type2: str = ""):
"""Create the sources fixture."""
sources = {SOURCE_ID: {"type": source_type}}
if source_type2:
sources[SOURCE_ID2] = {"type": source_type2}
return sources

def test_report_with_test_cases_and_jira_source(self):
"""Test that the other sources are not removed."""
self.database.reports.find.return_value = [
self.existing_report(metric_type="test_cases", sources=self.sources("jira")),
]
perform_migrations(self.database)
self.database.reports.replace_one.assert_not_called()

def test_report_with_test_cases_and_manual_number_source(self):
"""Test that the manual number source is removed."""
self.database.reports.find.return_value = [
self.existing_report(metric_type="test_cases", sources=self.sources()),
]
perform_migrations(self.database)
self.database.reports.replace_one.assert_called_once_with(
{"_id": "id"},
self.inserted_report(metric_type="test_cases", sources={}),
)

def test_report_with_test_cases_and_jira_and_manual_number_source(self):
"""Test that the manual number source is removed."""
self.database.reports.find.return_value = [
self.existing_report(metric_type="test_cases", sources=self.sources("jira", "manual_number")),
]
perform_migrations(self.database)
self.database.reports.replace_one.assert_called_once_with(
{"_id": "id"},
self.inserted_report(metric_type="test_cases", sources=self.sources("jira")),
)
2 changes: 1 addition & 1 deletion components/shared_code/src/shared_data_model/metrics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -576,7 +576,7 @@
unit=Unit.TEST_CASES,
direction=Direction.MORE_IS_BETTER,
near_target="0",
sources=["jenkins_test_report", "jira", "junit", "manual_number", "robot_framework", "testng"],
sources=["jenkins_test_report", "jira", "junit", "robot_framework", "testng"],
tags=[Tag.TEST_QUALITY],
),
"tests": Metric(
Expand Down
1 change: 0 additions & 1 deletion components/shared_code/src/shared_data_model/sources/manual_number.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
"sentiment",
"slow_transactions",
"suppressed_violations",
"test_cases",
"tests",
"todo_and_fixme_comments",
"uncovered_branches",
Expand Down
4 changes: 4 additions & 0 deletions docs/src/changelog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ If your currently installed *Quality-time* version is not v5.15.0, please first

- Start replacing Semantic UI React with Material UI as frontend component library. This migration will probably span multiple releases. Until the migration is finished, the UI may contain inconsistent elements, such as fonts, colors, and icons. Closes [#9550](https://github.com/ICTU/quality-time/issues/9550).

### Removed

- The 'test cases' metric had 'manual number' as supported source for the metric, but that could never work since the metric needs sources that contain test cases with identifiers so they can be matched. Manual number sources for test cases metrics are removed from reports automatically on application startup. Fixes [#9793](https://github.com/ICTU/quality-time/issues/9793).

## v5.15.0 - 2024-07-30

### Deployment notes
Expand Down
43 changes: 22 additions & 21 deletions docs/src/versioning.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,28 @@ A new **patch** release of *Quality-time* is made when the new version has only

The table below contains the *Quality-time* releases since the last minor of the previous major release. For each release it shows the release date, the MongoDB version included, the MongoDB feature compatibility (FC) version set in the database, whether migration code was added or removed, and whether up- or downgrading is possible, and if so, to which versions.

| Version | Date | Mongo | FC | Migrations | Downgrade | Upgrade |
|------------|--------------|--------|--------|------------|----------------|-----------------|
| v5.15.0 | 2024-07-30 | v7 | v7 | | v5.14.0 | n/a |
| v5.14.0 | 2024-07-05 | v7 | **v7** | added | not possible | v5.15.0 |
| v5.13.0 | 2024-05-23 | v7 | v6 | added | not possible | v5.14.0-v5.15.0 |
| v5.12.0 | 2024-05-17 | v7 | v6 | added | not possible | v5.13.0-v5.15.0 |
| v5.11.0 | 2024-04-22 | v7 | v6 | | v5.6.0-v5.10.0 | v5.12.0-v5.15.0 |
| v5.10.0 | 2024-04-15 | v7 | v6 | | v5.6.0-v5.9.0 | v5.11.0-v5.15.0 |
| v5.9.0 | 2024-03-22 | v7 | v6 | | v5.6.0-v5.8.0 | v5.10.0-v5.15.0 |
| v5.8.0 | 2024-02-16 | v7 | v6 | | v5.6.0-v5.7.0 | v5.9.0-v5.15.0 |
| v5.7.0 | 2024-01-31 | v7 | v6 | | v5.6.0 | v5.8.0-v5.15.0 |
| v5.6.0 | 2024-01-12 | v7 | v6 | added | not possible | v5.7.0-v5.15.0 |
| v5.5.0 | 2023-12-15 | v7 | v6 | | v5.1.0-v5.4.0 | v5.6.0-v5.15.0 |
| v5.4.0 | 2023-12-11 | v7 | v6 | | v5.1.0-v5.3.1 | v5.5.0-v5.15.0 |
| v5.3.1 | 2023-11-08 | v7 | v6 | | v5.1.0-v5.3.0 | v5.4.0-v5.15.0 |
| v5.3.0 | 2023-11-07 | **v7** | v6 | | v5.1.0-v5.2.0 | v5.3.1-v5.15.0 |
| v5.2.0 | 2023-09-29 | v6 | v6 | | v5.1.0 | v5.3.0-v5.15.0 |
| v5.1.0 | 2023-09-05 | v6 | **v6** | | not possible | v5.2.0-v5.15.0 |
| v5.0.1 | 2023-06-26 | v6 | v5 | | v4.10.0-v5.0.0 | v5.1.0-v5.2.0 |
| **v5.0.0** | 2023-06-23 | **v6** | v5 | | v4.10.0 | v5.0.1-v5.2.0 |
| v4.10.0 | 2023-04-26 | v5 | v5 | | n/a | v5.0.0-v5.2.0 |
| Version | Date | Mongo | FC | Migrations | Downgrade | Upgrade |
|------------|--------------|--------|--------|------------|-----------------|-----------------|
| v5.16.0 | [unreleased] | v7 | v7 | added | v5.14.0-v5.15.0 | n/a |
| v5.15.0 | 2024-07-30 | v7 | v7 | | v5.14.0 | v5.16.0 |
| v5.14.0 | 2024-07-05 | v7 | **v7** | added | not possible | v5.15.0-v5.16.0 |
| v5.13.0 | 2024-05-23 | v7 | v6 | added | not possible | v5.14.0-v5.16.0 |
| v5.12.0 | 2024-05-17 | v7 | v6 | added | not possible | v5.13.0-v5.16.0 |
| v5.11.0 | 2024-04-22 | v7 | v6 | | v5.6.0-v5.10.0 | v5.12.0-v5.16.0 |
| v5.10.0 | 2024-04-15 | v7 | v6 | | v5.6.0-v5.9.0 | v5.11.0-v5.16.0 |
| v5.9.0 | 2024-03-22 | v7 | v6 | | v5.6.0-v5.8.0 | v5.10.0-v5.16.0 |
| v5.8.0 | 2024-02-16 | v7 | v6 | | v5.6.0-v5.7.0 | v5.9.0-v5.16.0 |
| v5.7.0 | 2024-01-31 | v7 | v6 | | v5.6.0 | v5.8.0-v5.16.0 |
| v5.6.0 | 2024-01-12 | v7 | v6 | added | not possible | v5.7.0-v5.16.0 |
| v5.5.0 | 2023-12-15 | v7 | v6 | | v5.1.0-v5.4.0 | v5.6.0-v5.16.0 |
| v5.4.0 | 2023-12-11 | v7 | v6 | | v5.1.0-v5.3.1 | v5.5.0-v5.16.0 |
| v5.3.1 | 2023-11-08 | v7 | v6 | | v5.1.0-v5.3.0 | v5.4.0-v5.16.0 |
| v5.3.0 | 2023-11-07 | **v7** | v6 | | v5.1.0-v5.2.0 | v5.3.1-v5.16.0 |
| v5.2.0 | 2023-09-29 | v6 | v6 | | v5.1.0 | v5.3.0-v5.16.0 |
| v5.1.0 | 2023-09-05 | v6 | **v6** | | not possible | v5.2.0-v5.16.0 |
| v5.0.1 | 2023-06-26 | v6 | v5 | | v4.10.0-v5.0.0 | v5.1.0-v5.2.0 |
| **v5.0.0** | 2023-06-23 | **v6** | v5 | | v4.10.0 | v5.0.1-v5.2.0 |
| v4.10.0 | 2023-04-26 | v5 | v5 | | n/a | v5.0.0-v5.2.0 |

## Upgrade path

Expand Down

0 comments on commit 8570430

Please sign in to comment.