Implement command to validate builds

Get builds from dashboard and validate
them with maestro result.

Command introduced with the commit:
`kci-dev validate build --all-checkouts --giturl <URL> \
--commit <commit-hash> --branch <git-branch>`
Provide `--all-checkouts` to build validation stats for all
available checkouts.
Build validation for a specific checkout can be performed by
using all three options: --giturl, --branch, and --commit.

The command will provide below information:
1. Build count validation
2. Missing build information
3. Build status validation

Signed-off-by: Jeny Sadadia <[email protected]>

Author: Jeny Sadadia

kcidev/libs/maestro_common.py

@@ -62,18 +62,29 @@ def maestro_get_node(url, nodeid):
     return response.json()
 
 
-def maestro_get_nodes(url, limit, offset, filter):
+def maestro_get_nodes(url, limit, offset, filter, paginate):
     headers = {
         "Content-Type": "application/json; charset=utf-8",
     }
-    url = url + "latest/nodes/fast?limit=" + str(limit) + "&offset=" + str(offset)
-    maestro_print_api_call(url)
-    if filter:
-        for f in filter:
-            # TBD: We need to add translate filter to API
-            # if we need anything more complex than eq(=)
-            url = url + "&" + f
 
+    if paginate:
+        url = url + "latest/nodes/fast?limit=" + str(limit) + "&offset=" + str(offset)
+        if filter:
+            for f in filter:
+                # TBD: We need to add translate filter to API
+                # if we need anything more complex than eq(=)
+                url = url + "&" + f
+
+    else:
+        url = url + "latest/nodes/fast"
+        if filter:
+            url = url + "?"
+            for f in filter:
+                # TBD: We need to add translate filter to API
+                # if we need anything more complex than eq(=)
+                url = url + "&" + f
+
+    maestro_print_api_call(url)
     response = requests.get(url, headers=headers)
     try:
         response.raise_for_status()

kcidev/main.py

@@ -15,6 +15,7 @@
     patch,
     results,
     testretry,
+    validate,
     watch,
 )
 
@@ -65,6 +66,7 @@ def run():
     cli.add_command(maestro_results.maestro_results)
     cli.add_command(testretry.testretry)
     cli.add_command(results.results)
+    cli.add_command(validate.validate)
     cli.add_command(watch.watch)
     cli()
 

kcidev/subcommands/maestro_results.py

@@ -44,8 +44,21 @@
     multiple=True,
     help="Print only particular field(s) from node data",
 )
+@click.option(
+    "--count",
+    is_flag=True,
+    required=False,
+    help="Print only count of nodes",
+)
+@click.option(
+    "--paginate",
+    is_flag=True,
+    required=False,
+    default=True,
+    help="Set True if pagination is required in the output. Default is True",
+)
 @click.pass_context
-def maestro_results(ctx, nodeid, nodes, limit, offset, filter, field):
+def maestro_results(ctx, nodeid, nodes, limit, offset, filter, field, count, paginate):
     config = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
     url = config[instance]["api"]
@@ -55,7 +68,11 @@ def maestro_results(ctx, nodeid, nodes, limit, offset, filter, field):
     if nodeid:
         results = maestro_get_node(url, nodeid)
     if nodes:
-        results = maestro_get_nodes(url, limit, offset, filter)
+        results = maestro_get_nodes(url, limit, offset, filter, paginate)
+        if count:
+            kci_msg_cyan("Maestro:")
+            kci_msg(f"Build count:{len(results)}")
+            return results
     maestro_print_nodes(results, field)
 
 

kcidev/subcommands/results/__init__.py

@@ -59,7 +59,7 @@ def summary(origin, git_folder, giturl, branch, commit, latest, arch, use_json):
 @results_display_options
 def trees(origin, use_json):
     """List trees from a give origin."""
-    cmd_list_trees(origin, use_json)
+    return cmd_list_trees(origin, use_json)
 
 
 @results.command()
@@ -84,7 +84,7 @@ def builds(
         origin, giturl, branch, commit, latest, git_folder
     )
     data = dashboard_fetch_builds(origin, giturl, branch, commit, arch, use_json)
-    cmd_builds(data, commit, download_logs, status, count, use_json)
+    return cmd_builds(data, commit, download_logs, status, count, use_json)
 
 
 @results.command()

kcidev/subcommands/results/parser.py

@@ -135,6 +135,7 @@ def cmd_list_trees(origin, use_json):
         kci_msg(f"  giturl: {t['git_repository_url']}")
         kci_msg(f"  latest: {t['git_commit_hash']} ({t['git_commit_name']})")
         kci_msg(f"  latest: {t['start_time']}")
+    return trees
 
 
 def cmd_builds(data, commit, download_logs, status, count, use_json):
@@ -171,9 +172,11 @@ def cmd_builds(data, commit, download_logs, status, count, use_json):
     if count and use_json:
         kci_msg(f'{{"count":{filtered_builds}}}')
     elif count:
-        kci_msg(filtered_builds)
+        kci_msg_cyan("Dashboard:")
+        kci_msg(f"Build count:{filtered_builds}")
     elif use_json:
         kci_msg(json.dumps(builds))
+    return data["builds"]
 
 
 def print_build(build, log_path):

kcidev/subcommands/validate/__init__.py

@@ -0,0 +1,81 @@
+import click
+
+from kcidev.libs.common import kci_msg_yellow
+from kcidev.libs.git_repo import set_giturl_branch_commit
+from kcidev.subcommands.results import trees
+
+from .helper import get_build_stats
+
+
+@click.group(
+    help="Get results from the dashboard and validate them with maestro",
+)
+def validate():
+    """Commands related to results validation"""
+
+
+@validate.command()
+@click.option(
+    "--all-checkouts",
+    is_flag=True,
+    help="Get build validation stats for all available checkouts",
+)
+@click.option(
+    "--origin",
+    help="Select KCIDB origin",
+    default="maestro",
+)
+@click.option(
+    "--giturl",
+    help="Git URL of kernel tree",
+)
+@click.option(
+    "--branch",
+    help="Branch to get results for",
+)
+@click.option(
+    "--commit",
+    help="Commit or tag to get results for",
+)
+@click.option(
+    "--git-folder",
+    help="Path of git repository folder",
+)
+@click.option(
+    "--latest",
+    is_flag=True,
+    help="Select latest results available",
+)
+@click.option("--arch", help="Filter by arch")
+@click.pass_context
+def build(ctx, all_checkouts, origin, giturl, branch, commit, git_folder, latest, arch):
+    """
+    Provide --all-checkouts flag to pull all builds of available
+    checkouts from dashboard and compare them with results from maestro.
+
+    Build validation for a specific checkout can be performed by
+    using all three options: --giturl, --branch, and --commit
+    If above options are not provided, if will take latest/provided commit
+    checkout from the git folder specified.
+    """
+    if all_checkouts:
+        if giturl or branch or commit:
+            raise click.UsageError(
+                "Cannot use --all-checkouts with --giturl, --branch, or --commit"
+            )
+        trees_list = ctx.invoke(trees, origin=origin)
+        for tree in trees_list:
+            giturl = tree["git_repository_url"]
+            branch = tree["git_repository_branch"]
+            commit = tree["git_commit_hash"]
+            kci_msg_yellow(f"URL: {giturl}\nbranch: {branch}\ncommit: {commit}")
+            get_build_stats(ctx, giturl, branch, commit, arch)
+    else:
+        giturl, branch, commit = set_giturl_branch_commit(
+            origin, giturl, branch, commit, latest, git_folder
+        )
+        get_build_stats(ctx, giturl, branch, commit, arch)
+
+
+if __name__ == "__main__":
+    main_kcidev()

kcidev/subcommands/validate/helper.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+import click
+
+from kcidev.libs.common import kci_msg_green, kci_msg_red
+from kcidev.subcommands.maestro_results import maestro_results
+from kcidev.subcommands.results import builds
+
+
+def get_builds(ctx, giturl, branch, commit, arch=None):
+    """Get builds matching git URL, branch, and commit
+    Architecture can also be provided for filtering"""
+    maestro_builds = []
+    dashboard_builds = []
+    filters = [
+        "kind=kbuild",
+        "data.kernel_revision.url=" + giturl,
+        "data.kernel_revision.branch=" + branch,
+        "data.kernel_revision.commit=" + commit,
+    ]
+    if arch:
+        filters.append("data.arch=" + arch)
+    maestro_builds = ctx.invoke(
+        maestro_results,
+        count=True,
+        nodes=True,
+        filter=filters,
+        paginate=False,
+    )
+    try:
+        dashboard_builds = ctx.invoke(
+            builds, giturl=giturl, branch=branch, commit=commit, count=True
+        )
+    except click.Abort:
+        kci_msg_red("Aborted while fetching dashboard builds")
+        return maestro_builds, []
+    return maestro_builds, dashboard_builds
+
+
+def find_missing_builds(maestro_builds, dashboard_builds):
+    """
+    Compare build IDs found in maestro with dashboard builds
+    Return missing builds in dashboard.
+    """
+    missing_builds = []
+    dashboard_build_id = [b["id"].split(":")[1] for b in dashboard_builds]
+    for b in maestro_builds:
+        if b["id"] not in dashboard_build_id:
+            missing_builds.append(b)
+    if missing_builds:
+        print("Missing builds:", missing_builds)
+
+
+def validate_build_status(maestro_builds, dashboard_builds):
+    """
+    Validate if the build status of dashboard pulled build
+    matches with the maestro build status
+    """
+    status_map = {
+        "pass": "PASS",
+        "fail": "FAIL",
+        "incomplete": "ERROR",
+    }
+    dashboard_build_status_dict = {b["id"]: b["status"] for b in dashboard_builds}
+    for b in maestro_builds:
+        build_id = f"maestro:{b['id']}"
+        if build_id in dashboard_build_status_dict:
+            if dashboard_build_status_dict[build_id] != status_map.get(b["result"]):
+                kci_msg_red(f"Build status mismatched: {build_id}")
+
+
+def get_build_stats(ctx, giturl, branch, commit, arch=None):
+    """Get build stats"""
+    maestro_builds, dashboard_builds = get_builds(ctx, giturl, branch, commit, arch)
+    if len(dashboard_builds) == len(maestro_builds):
+        kci_msg_green("Build count matched")
+    else:
+        kci_msg_red("Build count didn't match")
+        find_missing_builds(maestro_builds, dashboard_builds)
+    validate_build_status(maestro_builds, dashboard_builds)