From 8e0391b3718955716929feec38c326d06d458b8c Mon Sep 17 00:00:00 2001 From: "Filipe Oliveira (Redis)" Date: Tue, 30 Jul 2024 12:39:18 +0300 Subject: [PATCH] Fixed git_timestamp_ms usage on cli triggering (for historical/past data filling) (#255) * Fixed coordinator filtering based uppon test regexp * Fixed git_timestamp_ms usage on cli triggering (for historical/past data filling) --- pyproject.toml | 2 +- redis_benchmarks_specification/__cli__/cli.py | 19 ++++++++++++++++--- .../__common__/builder_schema.py | 3 ++- .../self_contained_coordinator.py | 2 +- .../__spec__/cli.py | 14 +++++++------- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d1a41da..048e25b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.205" +version = "0.1.207" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__cli__/cli.py b/redis_benchmarks_specification/__cli__/cli.py index 62b41a5..2fcb79f 100644 --- a/redis_benchmarks_specification/__cli__/cli.py +++ b/redis_benchmarks_specification/__cli__/cli.py @@ -73,6 +73,10 @@ def get_commits_by_branch(args, repo): commits = [] for commit in repo.iter_commits(): commit_datetime = commit.committed_datetime + git_timestamp_ms = int( + datetime.datetime.utcfromtimestamp(commit_datetime.timestamp()).timestamp() + * 1000 + ) if ( args.from_date <= datetime.datetime.utcfromtimestamp(commit_datetime.timestamp()) @@ -87,6 +91,7 @@ def get_commits_by_branch(args, repo): "git_branch": repo.active_branch.name, "commit_summary": commit.summary, "commit_datetime": str(commit_datetime), + "git_timestamp_ms": git_timestamp_ms, } ) return commits, total_commits @@ -107,6 +112,14 @@ def get_commits_by_tags(args, repo): tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime) for tag in tags: + + git_timestamp_ms = int( + datetime.datetime.utcfromtimestamp( + tag.commit.committed_datetime.timestamp() + ).timestamp() + * 1000 + ) + if ( args.from_date <= datetime.datetime.utcfromtimestamp( @@ -137,6 +150,7 @@ def get_commits_by_tags(args, repo): "git_version": git_version, "commit_summary": tag.commit.summary, "commit_datetime": commit_datetime, + "git_timestamp_ms": git_timestamp_ms, } ) except packaging.version.InvalidVersion: @@ -265,6 +279,7 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): for cdict in commits: commit_hash = cdict["git_hash"] commit_summary = cdict["commit_summary"] + commit_datetime = cdict["commit_datetime"] match_obj = re.search(hash_regexp_string, commit_hash) if match_obj is None: logging.info( @@ -274,9 +289,7 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): ) else: print( - "Commit with hash: {} added. summary: {}".format( - commit_hash, commit_summary - ) + f"Commit with hash: {commit_hash} from {commit_datetime} added. summary: {commit_summary}" ) filtered_hash_commits.append(cdict) diff --git a/redis_benchmarks_specification/__common__/builder_schema.py b/redis_benchmarks_specification/__common__/builder_schema.py index d02a008..0f2fcd0 100644 --- a/redis_benchmarks_specification/__common__/builder_schema.py +++ b/redis_benchmarks_specification/__common__/builder_schema.py @@ -107,7 +107,8 @@ def get_commit_dict_from_sha( commit.commit.author.date.timestamp() * 1000.0 ) else: - use_git_timestamp = False + if "git_timestamp_ms" not in commit_dict: + use_git_timestamp = False commit_dict["use_git_timestamp"] = str(use_git_timestamp) commit_dict["git_hash"] = git_hash if gh_branch is not None: diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 64ae9eb..19c8f9c 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -532,7 +532,7 @@ def process_self_contained_coordinator_stream( tests_regexp = ".*" if b"tests_regexp" in testDetails: - tests_regexp = testDetails[b"tests_regexp"] + tests_regexp = testDetails[b"tests_regexp"].decode() logging.info( f"detected a regexp definition on the streamdata {tests_regexp}" ) diff --git a/redis_benchmarks_specification/__spec__/cli.py b/redis_benchmarks_specification/__spec__/cli.py index 96c0ebe..b661d38 100644 --- a/redis_benchmarks_specification/__spec__/cli.py +++ b/redis_benchmarks_specification/__spec__/cli.py @@ -92,13 +92,12 @@ def cli_command_logic(args, project_name, project_version): total_commits = 0 if args.use_branch: for commit in repo.iter_commits(): - if ( - args.from_date - <= datetime.datetime.utcfromtimestamp( - commit.committed_datetime.timestamp() - ) - <= args.to_date - ): + + commit_datetime_utc = datetime.datetime.utcfromtimestamp( + commit.committed_datetime.timestamp() + ) + git_timestamp_ms = int(commit_datetime_utc.timestamp() * 1000) + if args.from_date <= commit_datetime_utc <= args.to_date: if ( args.last_n > 0 and total_commits < args.last_n ) or args.last_n == -1: @@ -109,6 +108,7 @@ def cli_command_logic(args, project_name, project_version): "git_hash": commit.hexsha, "git_branch": repo.active_branch.name, "commit_summary": commit.summary, + "git_timestamp_ms": git_timestamp_ms, } ) if args.use_tags: