Implement command to validate builds

Get builds from dashboard and validate
them with maestro result.

Command introduced with the commit:
`kci-dev validate build --all-checkouts --giturl <URL> \
--commit <commit-hash> --branch <git-branch>`
Provide `--all-checkouts` to build validation stats for all
available checkouts.
Build validation for a specific checkout can be performed by
using all three options: --giturl, --branch, and --commit.

The command will provide below information:
1. Build count validation
2. Missing build information
3. Build status validation

Signed-off-by: Jeny Sadadia <[email protected]>

Author: Jeny Sadadia

kcidev/libs/common.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import json
 import logging
 import os
 import sys
@@ -142,3 +143,7 @@ def kci_msg_yellow(content, nl=True):
 
 def kci_msg_cyan(content, nl=True):
     click.secho(content, fg="cyan", nl=nl)
+
+
+def kci_msg_json(content, indent=1):
+    click.echo(json.dumps(content, indent=indent))

kcidev/libs/maestro_common.py

@@ -80,20 +80,29 @@ def maestro_get_node(url, nodeid):
     return node_data
 
 
-def maestro_get_nodes(url, limit, offset, filter):
+def maestro_get_nodes(url, limit, offset, filter, paginate):
     headers = {
         "Content-Type": "application/json; charset=utf-8",
     }
-    url = url + "latest/nodes/fast?limit=" + str(limit) + "&offset=" + str(offset)
 
-    logging.info(f"Fetching Maestro nodes - limit: {limit}, offset: {offset}")
-    if filter:
-        logging.debug(f"Applying filters: {filter}")
-        for f in filter:
-            # TBD: We need to add translate filter to API
-            # if we need anything more complex than eq(=)
-            url = url + "&" + f
+    if paginate:
+        url = url + "latest/nodes/fast?limit=" + str(limit) + "&offset=" + str(offset)
+        logging.info(f"Fetching Maestro nodes - limit: {limit}, offset: {offset}")
+        if filter:
+            for f in filter:
+                logging.debug(f"Applying filters: {filter}")
+                # TBD: We need to add translate filter to API
+                # if we need anything more complex than eq(=)
+                url = url + "&" + f
 
+    else:
+        url = url + "latest/nodes/fast"
+        if filter:
+            url = url + "?"
+            for f in filter:
+                # TBD: We need to add translate filter to API
+                # if we need anything more complex than eq(=)
+                url = url + "&" + f
     logging.debug(f"Full nodes URL: {url}")
     maestro_print_api_call(url)
 

kcidev/main.py

@@ -14,6 +14,7 @@
     maestro_results,
     results,
     testretry,
+    validate,
     watch,
 )
 
@@ -63,6 +64,7 @@ def run():
     cli.add_command(maestro_results.maestro_results)
     cli.add_command(testretry.testretry)
     cli.add_command(results.results)
+    cli.add_command(validate.validate)
     cli.add_command(watch.watch)
     cli()
 

kcidev/subcommands/maestro_results.py

@@ -65,6 +65,19 @@
     required=False,
     help="Filter results by tree name",
 )
+@click.option(
+    "--count",
+    is_flag=True,
+    required=False,
+    help="Print only count of nodes",
+)
+@click.option(
+    "--paginate",
+    is_flag=True,
+    required=False,
+    default=True,
+    help="Set True if pagination is required in the output. Default is True",
+)
 @add_filter_options
 @click.pass_context
 def maestro_results(
@@ -82,6 +95,8 @@ def maestro_results(
     compiler,
     config,
     git_branch,
+    count,
+    paginate,
 ):
     logging.info("Starting maestro-results command")
     logging.debug(
@@ -125,9 +140,14 @@ def maestro_results(
         logging.info(
             f"Fetching nodes with {len(filter)} filters, limit: {limit}, offset: {offset}"
         )
-        results = maestro_get_nodes(url, limit, offset, filter)
+        results = maestro_get_nodes(url, limit, offset, filter, paginate)
+        if count:
+            kci_msg_cyan("Maestro:")
+            kci_msg(f"Build count:{len(results)}")
+            return results
 
     logging.debug(f"Displaying results with fields: {field if field else 'all'}")
+
     maestro_print_nodes(results, field)
 
 

kcidev/subcommands/results/__init__.py

@@ -98,10 +98,16 @@ def summary(origin, git_folder, giturl, branch, commit, latest, arch, tree, use_
     type=int,
     default=7,
 )
+@click.option(
+    "--verbose",
+    is_flag=True,
+    default=True,
+    help="Print tree details",
+)
 @results_display_options
-def trees(origin, use_json, days):
+def trees(origin, use_json, days, verbose):
     """List trees from a give origin."""
-    cmd_list_trees(origin, use_json, days)
+    return cmd_list_trees(origin, use_json, days, verbose)
 
 
 @results.command()
@@ -139,7 +145,7 @@ def builds(
     data = dashboard_fetch_builds(
         origin, giturl, branch, commit, arch, tree, start_date, end_date, use_json
     )
-    cmd_builds(
+    return cmd_builds(
         data,
         commit,
         download_logs,

kcidev/subcommands/results/parser.py

@@ -149,21 +149,24 @@ def get_command_summary(command_data):
     return inconclusive_cmd, pass_cmd, fail_cmd
 
 
-def cmd_list_trees(origin, use_json, days):
+def cmd_list_trees(origin, use_json, days, verbose):
     logging.info(f"Listing trees for origin: {origin}")
     trees = dashboard_fetch_tree_list(origin, use_json, days)
     logging.debug(f"Found {len(trees)} trees")
     if use_json:
         kci_msg(json.dumps(list(map(lambda t: create_tree_json(t), trees))))
         return
-    for t in trees:
+
+    if verbose:
         logging.debug(
             f"Tree: {t['tree_name']}/{t['git_repository_branch']} - {t['git_commit_hash']}"
         )
-        kci_msg_green(f"- {t['tree_name']}/{t['git_repository_branch']}:")
-        kci_msg(f"  giturl: {t['git_repository_url']}")
-        kci_msg(f"  latest: {t['git_commit_hash']} ({t['git_commit_name']})")
-        kci_msg(f"  latest: {t['start_time']}")
+        for t in trees:
+            kci_msg_green(f"- {t['tree_name']}/{t['git_repository_branch']}:")
+            kci_msg(f"  giturl: {t['git_repository_url']}")
+            kci_msg(f"  latest: {t['git_commit_hash']} ({t['git_commit_name']})")
+            kci_msg(f"  latest: {t['start_time']}")
+    return trees
 
 
 def cmd_builds(
@@ -219,9 +222,11 @@ def cmd_builds(
     if count and use_json:
         kci_msg(f'{{"count":{filtered_builds}}}')
     elif count:
-        kci_msg(filtered_builds)
+        kci_msg_cyan("Dashboard:")
+        kci_msg(f"Build count:{filtered_builds}")
     elif use_json:
         kci_msg(json.dumps(builds))
+    return data["builds"]
 
 
 def print_build(build, log_path):

kcidev/subcommands/validate/__init__.py

@@ -0,0 +1,105 @@
+import click
+
+from kcidev.libs.common import kci_msg_yellow
+from kcidev.libs.git_repo import set_giturl_branch_commit
+from kcidev.subcommands.results import trees
+
+from .helper import get_build_stats
+
+
+@click.group(
+    help="Get results from the dashboard and validate them with maestro",
+)
+def validate():
+    """Commands related to results validation"""
+
+
+@validate.command()
+@click.option(
+    "--all-checkouts",
+    is_flag=True,
+    help="Get build validation stats for all available checkouts",
+)
+@click.option(
+    "--days",
+    help="Provide a period of time in days to get results for",
+    type=int,
+    default="7",
+)
+@click.option(
+    "--origin",
+    help="Select KCIDB origin",
+    default="maestro",
+)
+@click.option(
+    "--giturl",
+    help="Git URL of kernel tree",
+)
+@click.option(
+    "--branch",
+    help="Branch to get results for",
+)
+@click.option(
+    "--commit",
+    help="Commit or tag to get results for",
+)
+@click.option(
+    "--git-folder",
+    help="Path of git repository folder",
+)
+@click.option(
+    "--latest",
+    is_flag=True,
+    help="Select latest results available",
+)
+@click.option("--arch", help="Filter by arch")
+@click.option(
+    "--verbose",
+    is_flag=True,
+    default=False,
+    help="Get detailed output",
+)
+@click.pass_context
+def build(
+    ctx,
+    all_checkouts,
+    origin,
+    giturl,
+    branch,
+    commit,
+    git_folder,
+    latest,
+    arch,
+    days,
+    verbose,
+):
+    """
+    Provide --all-checkouts flag to pull all builds of available
+    checkouts from dashboard and compare them with results from maestro.
+
+    Build validation for a specific checkout can be performed by
+    using all three options: --giturl, --branch, and --commit
+    If above options are not provided, if will take latest/provided commit
+    checkout from the git folder specified.
+    """
+    if all_checkouts:
+        if giturl or branch or commit:
+            raise click.UsageError(
+                "Cannot use --all-checkouts with --giturl, --branch, or --commit"
+            )
+        trees_list = ctx.invoke(trees, origin=origin, days=days, verbose=False)
+        for tree in trees_list:
+            giturl = tree["git_repository_url"]
+            branch = tree["git_repository_branch"]
+            commit = tree["git_commit_hash"]
+            kci_msg_yellow(f"URL: {giturl}\nbranch: {branch}\ncommit: {commit}")
+            get_build_stats(ctx, giturl, branch, commit, verbose, arch)
+    else:
+        giturl, branch, commit = set_giturl_branch_commit(
+            origin, giturl, branch, commit, latest, git_folder
+        )
+        get_build_stats(ctx, giturl, branch, commit, verbose, arch)
+
+
+if __name__ == "__main__":
+    main_kcidev()

kcidev/subcommands/validate/helper.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+import click
+
+from kcidev.libs.common import kci_msg, kci_msg_green, kci_msg_json, kci_msg_red
+from kcidev.subcommands.maestro_results import maestro_results
+from kcidev.subcommands.results import builds
+
+
+def get_builds(ctx, giturl, branch, commit, arch=None):
+    """Get builds matching git URL, branch, and commit
+    Architecture can also be provided for filtering"""
+    maestro_builds = []
+    dashboard_builds = []
+    filters = [
+        "kind=kbuild",
+        "data.error_code__ne=node_timeout",  # maestro doesn't submit timed-out nodes
+        "data.kernel_revision.url=" + giturl,
+        "data.kernel_revision.branch=" + branch,
+        "data.kernel_revision.commit=" + commit,
+        "state__in=done,available",
+    ]
+    if arch:
+        filters.append("data.arch=" + arch)
+    maestro_builds = ctx.invoke(
+        maestro_results,
+        count=True,
+        nodes=True,
+        filter=filters,
+        paginate=False,
+    )
+    try:
+        dashboard_builds = ctx.invoke(
+            builds, giturl=giturl, branch=branch, commit=commit, count=True
+        )
+    except click.Abort:
+        kci_msg_red("Aborted while fetching dashboard builds")
+        return maestro_builds, None
+    return maestro_builds, dashboard_builds
+
+
+def find_missing_builds(maestro_builds, dashboard_builds, verbose):
+    """
+    Compare build IDs found in maestro with dashboard builds
+    Return missing builds in dashboard.
+    """
+    missing_builds = []
+    dashboard_build_id = [b["id"].split(":")[1] for b in dashboard_builds]
+    for b in maestro_builds:
+        if b["id"] not in dashboard_build_id:
+            missing_builds.append(b)
+    if missing_builds:
+        if verbose:
+            kci_msg("Missing builds:")
+            kci_msg_json(missing_builds)
+        else:
+            missing_build_ids = [b["id"] for b in missing_builds]
+            kci_msg(f"Missing build IDs:{missing_build_ids}")
+
+
+def validate_build_status(maestro_builds, dashboard_builds):
+    """
+    Validate if the build status of dashboard pulled build
+    matches with the maestro build status
+    """
+    status_map = {
+        "pass": "PASS",
+        "fail": "FAIL",
+        "incomplete": "ERROR",
+    }
+    dashboard_build_status_dict = {b["id"]: b["status"] for b in dashboard_builds}
+    for b in maestro_builds:
+        build_id = f"maestro:{b['id']}"
+        if build_id in dashboard_build_status_dict:
+            if dashboard_build_status_dict[build_id] != status_map.get(b["result"]):
+                kci_msg_red(f"Build status mismatched: {build_id}")
+
+
+def get_build_stats(ctx, giturl, branch, commit, verbose, arch=None):
+    """Get build stats"""
+    maestro_builds, dashboard_builds = get_builds(ctx, giturl, branch, commit, arch)
+    if dashboard_builds is not None:
+        if len(dashboard_builds) == len(maestro_builds):
+            kci_msg_green("Build count matched")
+        else:
+            kci_msg_red("Build count didn't match")
+            find_missing_builds(maestro_builds, dashboard_builds, verbose)
+        validate_build_status(maestro_builds, dashboard_builds)