diff --git a/kcidev/libs/common.py b/kcidev/libs/common.py index d7dfdd1..c9a746d 100644 --- a/kcidev/libs/common.py +++ b/kcidev/libs/common.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import json import logging import os import sys @@ -128,17 +129,21 @@ def kci_msg_nonl(content): click.echo(content, nl=False) -def kci_msg_green_nonl(content): - click.secho(content, fg="green", nl=False) +def kci_msg_green(content, nl=True): + click.secho(content, fg="green", nl=nl) -def kci_msg_red_nonl(content): - click.secho(content, fg="red", nl=False) +def kci_msg_red(content, nl=True): + click.secho(content, fg="red", nl=nl) -def kci_msg_yellow_nonl(content): - click.secho(content, fg="bright_yellow", nl=False) +def kci_msg_yellow(content, nl=True): + click.secho(content, fg="bright_yellow", nl=nl) -def kci_msg_cyan_nonl(content): - click.secho(content, fg="cyan", nl=False) +def kci_msg_cyan(content, nl=True): + click.secho(content, fg="cyan", nl=nl) + + +def kci_msg_json(content, indent=1): + click.echo(json.dumps(content, indent=indent)) diff --git a/kcidev/libs/dashboard.py b/kcidev/libs/dashboard.py index c6e0d69..d2e1d18 100644 --- a/kcidev/libs/dashboard.py +++ b/kcidev/libs/dashboard.py @@ -8,7 +8,7 @@ from kcidev.libs.common import * -DASHBOARD_API = "https://dashboard.kernelci.org/api/" +DASHBOARD_API = "https://dashboard.kernelci.org:9000/api/" def _dashboard_request(func): @@ -187,9 +187,10 @@ def dashboard_fetch_build(build_id, use_json): return dashboard_api_fetch(endpoint, {}, use_json) -def dashboard_fetch_tree_list(origin, use_json): +def dashboard_fetch_tree_list(origin, use_json, days=7): params = { "origin": origin, + "interval_in_days": days, } logging.info(f"Fetching tree list for origin: {origin}") return dashboard_api_fetch("tree-fast", params, use_json) diff --git a/kcidev/libs/git_repo.py b/kcidev/libs/git_repo.py index 768bab5..2a3dcd5 100644 --- a/kcidev/libs/git_repo.py +++ b/kcidev/libs/git_repo.py @@ -402,11 +402,11 @@ def set_giturl_branch_commit(origin, giturl, branch, commit, latest, git_folder) logging.info( f"Final parameters - URL: {giturl}, Branch: {branch}, Commit: {commit if commit else 'latest'}" ) - kci_msg("git folder: " + str(git_folder)) - kci_msg("tree: " + giturl) - kci_msg("branch: " + branch) + logging.info("git folder: " + str(git_folder)) + logging.info("tree: " + giturl) + logging.info("branch: " + branch) if commit: - kci_msg("commit: " + commit) + logging.info("commit: " + commit) # If commit looks like a tag or short hash (not 40 chars), try to resolve it if len(commit) != 40 or not all( c in "0123456789abcdef" for c in commit.lower() @@ -423,6 +423,20 @@ def set_giturl_branch_commit(origin, giturl, branch, commit, latest, git_folder) if latest: logging.info("Fetching latest commit from dashboard") commit = get_latest_commit(origin, giturl, branch) - kci_msg("commit: " + commit) + logging.info("commit: " + commit) return giturl, branch, commit + + +def get_tree_name(origin, giturl, branch, commit): + """Get tree name from git URL, branch, and commit""" + trees = dashboard_fetch_tree_list(origin, False) + + for t in trees: + if ( + t["git_repository_url"] == giturl + and t["git_repository_branch"] == branch + and t["git_commit_hash"] == commit + ): + return t["tree_name"] + return None diff --git a/kcidev/libs/maestro_common.py b/kcidev/libs/maestro_common.py index 8eed180..e5278fc 100644 --- a/kcidev/libs/maestro_common.py +++ b/kcidev/libs/maestro_common.py @@ -80,20 +80,29 @@ def maestro_get_node(url, nodeid): return node_data -def maestro_get_nodes(url, limit, offset, filter): +def maestro_get_nodes(url, limit, offset, filter, paginate): headers = { "Content-Type": "application/json; charset=utf-8", } - url = url + "latest/nodes/fast?limit=" + str(limit) + "&offset=" + str(offset) - logging.info(f"Fetching Maestro nodes - limit: {limit}, offset: {offset}") - if filter: - logging.debug(f"Applying filters: {filter}") - for f in filter: - # TBD: We need to add translate filter to API - # if we need anything more complex than eq(=) - url = url + "&" + f + if paginate: + url = url + "latest/nodes/fast?limit=" + str(limit) + "&offset=" + str(offset) + logging.info(f"Fetching Maestro nodes - limit: {limit}, offset: {offset}") + if filter: + for f in filter: + logging.debug(f"Applying filters: {filter}") + # TBD: We need to add translate filter to API + # if we need anything more complex than eq(=) + url = url + "&" + f + else: + url = url + "latest/nodes/fast" + if filter: + url = url + "?" + for f in filter: + # TBD: We need to add translate filter to API + # if we need anything more complex than eq(=) + url = url + "&" + f logging.debug(f"Full nodes URL: {url}") maestro_print_api_call(url) @@ -190,18 +199,18 @@ def maestro_node_result(node): or result == "done" or result == "pass" ): - kci_msg_green_nonl("PASS") + kci_msg_green("PASS", nl=False) elif result == None: - kci_msg_green_nonl("PASS") + kci_msg_green("PASS", nl=False) else: - kci_msg_red_nonl("FAIL") + kci_msg_red("FAIL", nl=False) else: if node["result"] == "pass": - kci_msg_green_nonl("PASS") + kci_msg_green("PASS", nl=False) elif node["result"] == "fail": - kci_msg_red_nonl("FAIL") + kci_msg_red("FAIL", nl=False) else: - kci_msg_yellow_nonl(node["result"]) + kci_msg_yellow(node["result"]) if node["kind"] == "checkout": kci_msg_nonl(" branch checkout") diff --git a/kcidev/main.py b/kcidev/main.py index 7eefada..db9e49d 100755 --- a/kcidev/main.py +++ b/kcidev/main.py @@ -14,6 +14,7 @@ maestro_results, results, testretry, + validate, watch, ) @@ -63,6 +64,7 @@ def run(): cli.add_command(maestro_results.maestro_results) cli.add_command(testretry.testretry) cli.add_command(results.results) + cli.add_command(validate.validate) cli.add_command(watch.watch) cli() diff --git a/kcidev/subcommands/maestro_results.py b/kcidev/subcommands/maestro_results.py index 396904b..7eaa8c8 100644 --- a/kcidev/subcommands/maestro_results.py +++ b/kcidev/subcommands/maestro_results.py @@ -65,6 +65,19 @@ required=False, help="Filter results by tree name", ) +@click.option( + "--count", + is_flag=True, + required=False, + help="Print only count of nodes", +) +@click.option( + "--paginate", + is_flag=True, + required=False, + default=True, + help="Set True if pagination is required in the output. Default is True", +) @add_filter_options @click.pass_context def maestro_results( @@ -82,6 +95,8 @@ def maestro_results( compiler, config, git_branch, + count, + paginate, ): logging.info("Starting maestro-results command") logging.debug( @@ -125,9 +140,12 @@ def maestro_results( logging.info( f"Fetching nodes with {len(filter)} filters, limit: {limit}, offset: {offset}" ) - results = maestro_get_nodes(url, limit, offset, filter) + results = maestro_get_nodes(url, limit, offset, filter, paginate) + if count: + return results logging.debug(f"Displaying results with fields: {field if field else 'all'}") + maestro_print_nodes(results, field) diff --git a/kcidev/subcommands/results/__init__.py b/kcidev/subcommands/results/__init__.py index 096f849..e3d7318 100644 --- a/kcidev/subcommands/results/__init__.py +++ b/kcidev/subcommands/results/__init__.py @@ -92,10 +92,22 @@ def summary(origin, git_folder, giturl, branch, commit, latest, arch, tree, use_ help="Select KCIDB origin", default="maestro", ) +@click.option( + "--days", + help="Provide a period of time in days to get results for", + type=int, + default=7, +) +@click.option( + "--verbose", + is_flag=True, + default=True, + help="Print tree details", +) @results_display_options -def trees(origin, use_json): +def trees(origin, use_json, days, verbose): """List trees from a give origin.""" - cmd_list_trees(origin, use_json) + return cmd_list_trees(origin, use_json, days, verbose) @results.command() @@ -133,7 +145,7 @@ def builds( data = dashboard_fetch_builds( origin, giturl, branch, commit, arch, tree, start_date, end_date, use_json ) - cmd_builds( + return cmd_builds( data, commit, download_logs, diff --git a/kcidev/subcommands/results/parser.py b/kcidev/subcommands/results/parser.py index 7ba21a2..676a258 100644 --- a/kcidev/subcommands/results/parser.py +++ b/kcidev/subcommands/results/parser.py @@ -27,12 +27,12 @@ def print_summary(type, n_pass, n_fail, n_inconclusive): kci_msg_nonl(f"{type}:\t") - kci_msg_green_nonl(f"{n_pass}") if n_pass else kci_msg_nonl(f"{n_pass}") + kci_msg_green(f"{n_pass}") if n_pass else kci_msg_nonl(f"{n_pass}", nl=False) kci_msg_nonl("/") - kci_msg_red_nonl(f"{n_fail}") if n_fail else kci_msg_nonl(f"{n_fail}") + kci_msg_red(f"{n_fail}") if n_fail else kci_msg_nonl(f"{n_fail}", nl=False) kci_msg_nonl("/") ( - kci_msg_yellow_nonl(f"{n_inconclusive}") + kci_msg_yellow(f"{n_inconclusive}", nl=False) if n_inconclusive else kci_msg_nonl(f"{n_inconclusive}") ) @@ -149,22 +149,24 @@ def get_command_summary(command_data): return inconclusive_cmd, pass_cmd, fail_cmd -def cmd_list_trees(origin, use_json): +def cmd_list_trees(origin, use_json, days, verbose): logging.info(f"Listing trees for origin: {origin}") - trees = dashboard_fetch_tree_list(origin, use_json) + trees = dashboard_fetch_tree_list(origin, use_json, days) logging.debug(f"Found {len(trees)} trees") - if use_json: kci_msg(json.dumps(list(map(lambda t: create_tree_json(t), trees)))) return + for t in trees: logging.debug( f"Tree: {t['tree_name']}/{t['git_repository_branch']} - {t['git_commit_hash']}" ) - kci_msg_green_nonl(f"- {t['tree_name']}/{t['git_repository_branch']}:\n") - kci_msg(f" giturl: {t['git_repository_url']}") - kci_msg(f" latest: {t['git_commit_hash']} ({t['git_commit_name']})") - kci_msg(f" latest: {t['start_time']}") + if verbose: + kci_msg_green(f"- {t['tree_name']}/{t['git_repository_branch']}:") + kci_msg(f" giturl: {t['git_repository_url']}") + kci_msg(f" latest: {t['git_commit_hash']} ({t['git_commit_name']})") + kci_msg(f" latest: {t['start_time']}") + return trees def cmd_builds( @@ -219,28 +221,27 @@ def cmd_builds( if count and use_json: kci_msg(f'{{"count":{filtered_builds}}}') - elif count: - kci_msg(filtered_builds) elif use_json: kci_msg(json.dumps(builds)) + return data["builds"] def print_build(build, log_path): kci_msg_nonl("- config:") - kci_msg_cyan_nonl(build["config_name"]) + kci_msg_cyan(build["config_name"], nl=False) kci_msg_nonl(" arch: ") - kci_msg_cyan_nonl(build["architecture"]) + kci_msg_cyan(build["architecture"], nl=False) kci_msg_nonl(" compiler: ") - kci_msg_cyan_nonl(build["compiler"]) + kci_msg_cyan(build["compiler"], nl=False) kci_msg("") kci_msg_nonl(" status:") if build["status"] == "PASS": - kci_msg_green_nonl("PASS") + kci_msg_green("PASS", nl=False) elif build["status"] == "FAIL": - kci_msg_red_nonl("FAIL") + kci_msg_red("FAIL", nl=False) else: - kci_msg_yellow_nonl(f"INCONCLUSIVE (status: {build['status']})") + kci_msg_yellow(f"INCONCLUSIVE (status: {build['status']})", nl=False) kci_msg("") kci_msg(f" config_url: {build['config_url']}") @@ -376,38 +377,38 @@ def cmd_tests( def print_test(test, log_path): kci_msg_nonl("- test path: ") - kci_msg_cyan_nonl(test["path"]) + kci_msg_cyan(test["path"], nl=False) kci_msg("") kci_msg_nonl(" hardware: ") - kci_msg_cyan_nonl(test["environment_misc"]["platform"]) + kci_msg_cyan(test["environment_misc"]["platform"], nl=False) kci_msg("") if test["environment_compatible"]: kci_msg_nonl(" compatibles: ") - kci_msg_cyan_nonl(" | ".join(test["environment_compatible"])) + kci_msg_cyan(" | ".join(test["environment_compatible"]), nl=False) kci_msg("") kci_msg_nonl(" config: ") if "config" in test: - kci_msg_cyan_nonl(test["config"]) + kci_msg_cyan(test["config"], nl=False) elif "config_name" in test: - kci_msg_cyan_nonl(test["config_name"]) + kci_msg_cyan(test["config_name"], nl=False) else: - kci_msg_cyan_nonl("No config available") + kci_msg_cyan("No config available", nl=False) kci_msg_nonl(" arch: ") - kci_msg_cyan_nonl(test["architecture"]) + kci_msg_cyan(test["architecture"], nl=False) kci_msg_nonl(" compiler: ") - kci_msg_cyan_nonl(test["compiler"]) + kci_msg_cyan(test["compiler"], nl=False) kci_msg("") kci_msg_nonl(" status:") if test["status"] == "PASS": - kci_msg_green_nonl("PASS") + kci_msg_green("PASS", nl=False) elif test["status"] == "FAIL": - kci_msg_red_nonl("FAIL") + kci_msg_red("FAIL", nl=False) else: - kci_msg_yellow_nonl(f"INCONCLUSIVE (status: {test['status']})") + kci_msg_yellow(f"INCONCLUSIVE (status: {test['status']})", nl=False) kci_msg("") kci_msg(f" log: {log_path}") @@ -467,10 +468,10 @@ def cmd_hardware_list(data, use_json): f"Hardware: {hardware['hardware_name']} - {hardware['platform']}" ) kci_msg_nonl("- name: ") - kci_msg_cyan_nonl(hardware["hardware_name"]) + kci_msg_cyan(hardware["hardware_name"], nl=False) kci_msg("") kci_msg_nonl(" compatibles: ") - kci_msg_cyan_nonl(hardware["platform"]) + kci_msg_cyan(hardware["platform"], nl=False) kci_msg("") kci_msg("") diff --git a/kcidev/subcommands/validate/__init__.py b/kcidev/subcommands/validate/__init__.py new file mode 100644 index 0000000..078e216 --- /dev/null +++ b/kcidev/subcommands/validate/__init__.py @@ -0,0 +1,112 @@ +import click + +from kcidev.libs.git_repo import get_tree_name, set_giturl_branch_commit +from kcidev.subcommands.results import trees + +from .helper import get_build_stats, print_build_stats + + +@click.group( + help="Get results from the dashboard and validate them with maestro", +) +def validate(): + """Commands related to results validation""" + + +@validate.command() +@click.option( + "--all-checkouts", + is_flag=True, + help="Get build validation stats for all available checkouts", +) +@click.option( + "--days", + help="Provide a period of time in days to get results for", + type=int, + default="7", +) +@click.option( + "--origin", + help="Select KCIDB origin", + default="maestro", +) +@click.option( + "--giturl", + help="Git URL of kernel tree", +) +@click.option( + "--branch", + help="Branch to get results for", +) +@click.option( + "--commit", + help="Commit or tag to get results for", +) +@click.option( + "--git-folder", + help="Path of git repository folder", +) +@click.option( + "--latest", + is_flag=True, + help="Select latest results available", +) +@click.option("--arch", help="Filter by arch") +@click.option( + "--verbose", + is_flag=True, + default=False, + help="Get detailed output", +) +@click.pass_context +def build( + ctx, + all_checkouts, + origin, + giturl, + branch, + commit, + git_folder, + latest, + arch, + days, + verbose, +): + """ + Provide --all-checkouts flag to pull all builds of available + checkouts from dashboard and compare them with results from maestro. + + Build validation for a specific checkout can be performed by + using all three options: --giturl, --branch, and --commit + If above options are not provided, if will take latest/provided commit + checkout from the git folder specified. + """ + final_stats = [] + print("Fetching build information...") + if all_checkouts: + if giturl or branch or commit: + raise click.UsageError( + "Cannot use --all-checkouts with --giturl, --branch, or --commit" + ) + trees_list = ctx.invoke(trees, origin=origin, days=days, verbose=False) + for tree in trees_list: + giturl = tree["git_repository_url"] + branch = tree["git_repository_branch"] + commit = tree["git_commit_hash"] + tree_name = tree["tree_name"] + stats = get_build_stats( + ctx, giturl, branch, commit, tree_name, verbose, arch + ) + final_stats.append(stats) + else: + giturl, branch, commit = set_giturl_branch_commit( + origin, giturl, branch, commit, latest, git_folder + ) + tree_name = get_tree_name(origin, giturl, branch, commit) + stats = get_build_stats(ctx, giturl, branch, commit, tree_name, verbose, arch) + final_stats.append(stats) + print_build_stats(final_stats) + + +if __name__ == "__main__": + main_kcidev() diff --git a/kcidev/subcommands/validate/helper.py b/kcidev/subcommands/validate/helper.py new file mode 100644 index 0000000..79f2493 --- /dev/null +++ b/kcidev/subcommands/validate/helper.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +import click +from tabulate import tabulate + +from kcidev.libs.common import kci_msg, kci_msg_json, kci_msg_red +from kcidev.subcommands.maestro_results import maestro_results +from kcidev.subcommands.results import builds + + +def get_builds(ctx, giturl, branch, commit, arch=None): + """Get builds matching git URL, branch, and commit + Architecture can also be provided for filtering""" + maestro_builds = [] + dashboard_builds = [] + filters = [ + "kind=kbuild", + "data.error_code__ne=node_timeout", # maestro doesn't submit timed-out nodes + "data.kernel_revision.url=" + giturl, + "data.kernel_revision.branch=" + branch, + "data.kernel_revision.commit=" + commit, + "state__in=done,available", + ] + if arch: + filters.append("data.arch=" + arch) + maestro_builds = ctx.invoke( + maestro_results, + count=True, + nodes=True, + filter=filters, + paginate=False, + ) + try: + dashboard_builds = ctx.invoke( + builds, giturl=giturl, branch=branch, commit=commit, count=True + ) + except click.Abort: + kci_msg_red("Aborted while fetching dashboard builds") + return maestro_builds, None + return maestro_builds, dashboard_builds + + +def find_missing_builds(maestro_builds, dashboard_builds, verbose): + """ + Compare build IDs found in maestro with dashboard builds + Return missing builds in dashboard. + """ + dashboard_build_id = [b["id"].split(":")[1] for b in dashboard_builds] + maestro_build_id = [b["id"] for b in maestro_builds] + + if len(maestro_build_id) > len(dashboard_build_id): + missing_builds = [ + b for b in maestro_builds if b["id"] not in dashboard_build_id + ] + missing_build_ids = [b["id"] for b in missing_builds] + else: + missing_builds = [ + b for b in dashboard_builds if b["id"].split(":")[1] not in maestro_build_id + ] + missing_build_ids = [b["id"].split(":")[1] for b in missing_builds] + + if missing_builds and verbose: + kci_msg("Missing builds:") + kci_msg_json(missing_builds) + + return missing_build_ids + + +def validate_build_status(maestro_builds, dashboard_builds): + """ + Validate if the build status of dashboard pulled build + matches with the maestro build status + """ + status_map = { + "pass": "PASS", + "fail": "FAIL", + "incomplete": "ERROR", + } + builds_with_status_mismatch = [] + dashboard_build_status_dict = { + b["id"].split(":")[1]: b["status"] for b in dashboard_builds + } + for b in maestro_builds: + build_id = b["id"] + if build_id in dashboard_build_status_dict: + if dashboard_build_status_dict[build_id] != status_map.get(b["result"]): + builds_with_status_mismatch.append(build_id) + return builds_with_status_mismatch + + +def get_build_stats(ctx, giturl, branch, commit, tree_name, verbose, arch=None): + """Get build stats""" + maestro_builds, dashboard_builds = get_builds(ctx, giturl, branch, commit, arch) + if dashboard_builds is not None: + missing_build_ids = [] + if len(dashboard_builds) == len(maestro_builds): + count_comparison_flag = "✅" + else: + count_comparison_flag = "❌" + missing_build_ids = find_missing_builds( + maestro_builds, dashboard_builds, verbose + ) + builds_with_status_mismatch = validate_build_status( + maestro_builds, dashboard_builds + ) + stats = [ + f"{tree_name}/{branch}", + commit, + len(maestro_builds), + len(dashboard_builds), + count_comparison_flag, + missing_build_ids, + builds_with_status_mismatch, + ] + return stats + + +def print_build_stats(stats): + """Print build statistics in tabular format""" + print("Creating a build stats report...") + headers = [ + "tree/branch", + "Commit", + "Maestro\nbuilds", + "Dashboard\nbuilds", + "Build count\ncomparison", + "Missing build IDs", + "Builds with\nstatus mismatch", + ] + print( + tabulate( + stats, + headers=headers, + maxcolwidths=[None, 40, 3, 3, 2, 30, 30], + tablefmt="simple_grid", + ) + ) diff --git a/pyproject.toml b/pyproject.toml index 50d9aa0..fe7fbc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ requests = "^2.32.3" gitpython = "^3.1.43" tomli = { version = "^2.2.1", python = "<3.11" } pyyaml = "^6.0.2" +tabulate = "0.9.0" [tool.poetry.scripts] kci-dev = 'kcidev.main:run'