From b97fae2dadeda2426edc08ffa2087767dd0cb1a8 Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Wed, 6 Nov 2024 13:00:06 +1100 Subject: [PATCH 1/7] Support cached images for cloud runner. --- experiment/builder_runner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/experiment/builder_runner.py b/experiment/builder_runner.py index 862bef50c..64048d2e2 100644 --- a/experiment/builder_runner.py +++ b/experiment/builder_runner.py @@ -907,6 +907,12 @@ def build_and_run_cloud( f'--upload_corpus={corpus_path}', f'--experiment_name={self.experiment_name}' ] + + if oss_fuzz_checkout.ENABLE_CACHING and oss_fuzz_checkout.is_image_cached( + self.benchmark.project, sanitizer): + logger.info('Using cached image') + command.append('--use_cached_image') + if cloud_build_tags: command += ['--tags'] + cloud_build_tags command += ['--'] + self._libfuzzer_args() From 1be0c64a2fb3f8959ec4abb20a6d28eac5a7c2fa Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Wed, 6 Nov 2024 13:03:32 +1100 Subject: [PATCH 2/7] fix --- experiment/builder_runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/experiment/builder_runner.py b/experiment/builder_runner.py index 64048d2e2..ce5f1cf9e 100644 --- a/experiment/builder_runner.py +++ b/experiment/builder_runner.py @@ -908,9 +908,10 @@ def build_and_run_cloud( f'--experiment_name={self.experiment_name}' ] - if oss_fuzz_checkout.ENABLE_CACHING and oss_fuzz_checkout.is_image_cached( - self.benchmark.project, sanitizer): - logger.info('Using cached image') + if oss_fuzz_checkout.ENABLE_CACHING and ( + oss_fuzz_checkout.is_image_cached(project_name, 'address') and + oss_fuzz_checkout.is_image_cached(project_name, 'coverage')): + logger.info(f'Using cached image for {project_name}.') command.append('--use_cached_image') if cloud_build_tags: From b09d8de6bd07b43fc007465374d64f5b36d124bf Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Wed, 6 Nov 2024 13:44:45 +1100 Subject: [PATCH 3/7] override var --- experiment/oss_fuzz_checkout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment/oss_fuzz_checkout.py b/experiment/oss_fuzz_checkout.py index 20b33a891..e18ee24c9 100644 --- a/experiment/oss_fuzz_checkout.py +++ b/experiment/oss_fuzz_checkout.py @@ -30,7 +30,7 @@ BUILD_DIR: str = 'build' GLOBAL_TEMP_DIR: str = '' -ENABLE_CACHING = bool(int(os.getenv('OFG_USE_CACHING', '0'))) +ENABLE_CACHING = bool(int(os.getenv('OFG_USE_CACHING', '1'))) # Assume OSS-Fuzz is at repo root dir by default. # This will change if temp_dir is used. OSS_FUZZ_DIR: str = os.path.join( From 028f7cf76c34c0f156560ee15b3fd7026ac0322d Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Wed, 6 Nov 2024 14:04:00 +1100 Subject: [PATCH 4/7] fix docker image existance check --- experiment/oss_fuzz_checkout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment/oss_fuzz_checkout.py b/experiment/oss_fuzz_checkout.py index e18ee24c9..ac3a937f9 100644 --- a/experiment/oss_fuzz_checkout.py +++ b/experiment/oss_fuzz_checkout.py @@ -285,7 +285,7 @@ def is_image_cached(project_name: str, sanitizer: str) -> bool: cached_image_name = _get_project_cache_image_name(project_name, sanitizer) try: sp.run( - ['docker', 'inspect', '--type=image', cached_image_name], + ['docker', 'manifest', 'inspect', cached_image_name], check=True, stdin=sp.DEVNULL, stdout=sp.DEVNULL, From a55143e868ee825c0fa0bb7f40ab176c41166fbd Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Wed, 6 Nov 2024 15:00:32 +1100 Subject: [PATCH 5/7] Set real_project --- experiment/builder_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/experiment/builder_runner.py b/experiment/builder_runner.py index ce5f1cf9e..ed296c8ab 100644 --- a/experiment/builder_runner.py +++ b/experiment/builder_runner.py @@ -905,7 +905,8 @@ def build_and_run_cloud( f'--upload_coverage={coverage_path}', f'--upload_reproducer={reproducer_path}', f'--upload_corpus={corpus_path}', - f'--experiment_name={self.experiment_name}' + f'--experiment_name={self.experiment_name}', + f'--real_project={project_name}', ] if oss_fuzz_checkout.ENABLE_CACHING and ( From 2aa1238cb6321c63a8f868789d4fef1a120ae5ab Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Wed, 6 Nov 2024 15:47:06 +1100 Subject: [PATCH 6/7] debug --- experiment/builder_runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/experiment/builder_runner.py b/experiment/builder_runner.py index ed296c8ab..8bc80e9e9 100644 --- a/experiment/builder_runner.py +++ b/experiment/builder_runner.py @@ -919,6 +919,8 @@ def build_and_run_cloud( command += ['--tags'] + cloud_build_tags command += ['--'] + self._libfuzzer_args() + logger.info(f'Command: {command}') + if not self._run_with_retry_control(os.path.realpath(target_path), command, cwd=oss_fuzz_checkout.OSS_FUZZ_DIR): From a8246987ec8c5eb0dddf021a4dcfe00ea4eca612 Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Thu, 7 Nov 2024 10:19:58 +1100 Subject: [PATCH 7/7] use OSS-Fuzz PR branch. --- experiment/oss_fuzz_checkout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment/oss_fuzz_checkout.py b/experiment/oss_fuzz_checkout.py index 19b62cd17..d154ef74a 100644 --- a/experiment/oss_fuzz_checkout.py +++ b/experiment/oss_fuzz_checkout.py @@ -68,7 +68,7 @@ def _set_temp_oss_fuzz_repo(): def _clone_oss_fuzz_repo(): """Clones OSS-Fuzz to |OSS_FUZZ_DIR|.""" clone_command = [ - 'git', 'clone', 'https://github.com/google/oss-fuzz', '--depth', '1', + 'git', 'clone', '-b', 'add-arg-cloud-build', 'https://github.com/google/oss-fuzz', '--depth', '1', OSS_FUZZ_DIR ] proc = sp.Popen(clone_command,