diff --git a/infra/build/functions/fuzzbench.py b/infra/build/functions/fuzzbench.py index 2f353cbf02c2..0936cb6debe8 100644 --- a/infra/build/functions/fuzzbench.py +++ b/infra/build/functions/fuzzbench.py @@ -34,6 +34,7 @@ FUZZBENCH_PATH = '/fuzzbench' GCB_WORKSPACE_DIR = '/workspace' OOD_OUTPUT_CORPUS_DIR = f'{GCB_WORKSPACE_DIR}/ood_output_corpus' +OOD_CRASHES_DIR = f'{GCB_WORKSPACE_DIR}/crashes' def get_engine_project_image_name(fuzzing_engine, project): @@ -129,16 +130,14 @@ def get_build_fuzzers_steps(fuzzing_engine, project, env): project), '--file', engine_dockerfile_path, os.path.join(FUZZBENCH_PATH, 'fuzzers') ] - engine_step = [ - { - 'name': 'gcr.io/cloud-builders/docker', - 'args': build_args, - 'volumes': [{ - 'name': 'fuzzbench_path', - 'path': FUZZBENCH_PATH, - }], - }, - ] + engine_step = { + 'name': 'gcr.io/cloud-builders/docker', + 'args': build_args, + 'volumes': [{ + 'name': 'fuzzbench_path', + 'path': FUZZBENCH_PATH, + }], + } steps.append(engine_step) compile_project_step = { @@ -240,7 +239,7 @@ def get_build_ood_image_steps(fuzzing_engine, project, env_dict): 'build', '--tag', ood_image, '--file', fuzzer_runtime_dockerfile_path, os.path.join(GCB_WORKSPACE_DIR + FUZZBENCH_PATH, 'fuzzers') ] - }, + } steps.append(build_runtime_step) oss_fuzz_on_demand_dockerfile_path = f'{GCB_WORKSPACE_DIR}/oss-fuzz/infra/build/functions/ood.Dockerfile' @@ -310,16 +309,15 @@ def get_extract_crashes_steps(fuzzing_engine, project, env_dict): } steps.append(download_libfuzzer_build_step) - crashes_dir = f'{GCB_WORKSPACE_DIR}/crashes/' extract_crashes_step = { 'name': get_engine_project_image_name(fuzzing_engine, project), 'args': [ 'bash', '-c', f'unzip {libfuzzer_build_dir}{build_filename} ' - f'-d {libfuzzer_build_dir} && mkdir -p {crashes_dir} && ' + f'-d {libfuzzer_build_dir} && mkdir -p {OOD_CRASHES_DIR} && ' f'{libfuzzer_build_dir}{env_dict["FUZZ_TARGET"]} {OOD_OUTPUT_CORPUS_DIR} ' - f'-runs=0 -artifact_prefix={crashes_dir}; ' - f'echo "\nCrashes found by OOD:" && ls {crashes_dir} ' + f'-runs=0 -artifact_prefix={OOD_CRASHES_DIR}/; ' + f'echo "\nCrashes found by OOD:" && ls {OOD_CRASHES_DIR} ' ], } steps.append(extract_crashes_step) @@ -327,6 +325,38 @@ def get_extract_crashes_steps(fuzzing_engine, project, env_dict): return steps +def get_upload_testcase_steps(project, env_dict): + """Returns the build steps to upload a testcase in the ClusterFuzz External + upload testcase endpoint.""" + steps = [] + + access_token_file_path = f'{GCB_WORKSPACE_DIR}/at.txt' + get_access_token_step = { + 'name': + 'google/cloud-sdk', + 'args': [ + 'bash', '-c', + f'gcloud auth print-access-token > {access_token_file_path}' + ] + } + steps.append(get_access_token_step) + + upload_testcase_script_path = f'{GCB_WORKSPACE_DIR}/oss-fuzz/infra/build/functions/ood_upload_testcase.py' + job_name = f'libfuzzer_asan_{project.name}' + target_name = f'{project.name}_{env_dict["FUZZ_TARGET"]}' + upload_testcase_step = { + 'name': + 'python:3.8', + 'args': [ + 'python3', upload_testcase_script_path, OOD_CRASHES_DIR, job_name, + target_name, access_token_file_path + ] + } + steps.append(upload_testcase_step) + + return steps + + def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments project_name, project_yaml, dockerfile_lines, config): """Returns build steps for project.""" @@ -349,6 +379,7 @@ def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments steps += get_push_and_run_ood_image_steps(config.fuzzing_engine, project, env_dict) steps += get_extract_crashes_steps(config.fuzzing_engine, project, env_dict) + steps += get_upload_testcase_steps(project, env_dict) return steps diff --git a/infra/build/functions/ood_upload_testcase.py b/infra/build/functions/ood_upload_testcase.py new file mode 100644 index 000000000000..eb34fdbd0d2d --- /dev/null +++ b/infra/build/functions/ood_upload_testcase.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +"""Upload OSS-Fuzz on Demand testcases.""" + +import json +import logging +import os +import sys +import subprocess + +try: + import requests +except ImportError: + print("requests library not found. Installing...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "requests"]) + import requests + +POST_URL = 'https://oss-fuzz.com/upload-testcase/upload-oauth' + + +def get_access_token(access_token_path): + """Returns the ACCESS_TOKEN for upload testcase requests""" + with open(access_token_path, 'r') as f: + line = f.readline() + return line.strip() + + +def get_headers(access_token_path): + """Returns the headers required to upload testcase requests""" + access_token = get_access_token(access_token_path) + return { + 'Authorization': 'Bearer ' + access_token, + } + + +def upload_testcase(upload_url, testcase_path, job, target, access_token_path): + """Make an upload testcase request.""" + files = { + 'file': open(testcase_path, 'rb'), + } + data = { + 'job': job, + 'target': target, + } + try: + resp = requests.post(upload_url, + files=files, + data=data, + headers=get_headers(access_token_path)) + resp.raise_for_status() + result = json.loads(resp.text) + print('Upload succeeded. Testcase ID is', result['id']) + except: + print('Failed to upload with status', resp.status_code) + print(resp.text) + + +def get_file_path(dir_path): + """Returns the path of a file inside 'dir_path'. Returns None if there are no + files inside the the given directory.""" + files = [] + for entry in os.scandir(dir_path): + if entry.is_file(): + return f'{dir_path}/{entry.name}' + return None + + +def main(): + """Upload an OSS-Fuzz on Demand testcase.""" + testcase_dir_path = sys.argv[1] + job = sys.argv[2] + target = sys.argv[3] + access_token_path = sys.argv[4] + testcase_path = get_file_path(testcase_dir_path) + + if not testcase_path: + print('OSS-Fuzz on Demand did not find any crashes.') + else: + upload_testcase(POST_URL, testcase_path, job, target, access_token_path) + + return 0 + + +if __name__ == '__main__': + main() diff --git a/infra/build/functions/ood_upload_testcase_test.py b/infra/build/functions/ood_upload_testcase_test.py new file mode 100644 index 000000000000..fa59e0d44eee --- /dev/null +++ b/infra/build/functions/ood_upload_testcase_test.py @@ -0,0 +1,69 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +"""Tests for ood_upload_testcase_test.py.""" + +import os +import shutil +import tempfile +import unittest +from unittest import mock + +import ood_upload_testcase + + +class GetFilePath(unittest.TestCase): + """Tests for get_file_path.""" + + def setUp(self): + """Set tem_dir attribute""" + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + """Remove temp_dir after tests""" + shutil.rmtree(self.temp_dir) + + def test_no_files(self): + """Test for empty directory""" + self.assertIsNone(ood_upload_testcase.get_file_path(self.temp_dir)) + + def test_single_file(self): + """Test for single file""" + file_name = 'test_file.txt' + file_path = os.path.join(self.temp_dir, file_name) + open(file_path, 'w').close() + self.assertEqual(ood_upload_testcase.get_file_path(self.temp_dir), + file_path) + + def test_multiple_files(self): + """Test for multiple files""" + file_names = ['file1.txt', 'file2.csv', 'data.json'] + file_paths = [] + for name in file_names: + file_path = os.path.join(self.temp_dir, name) + file_paths.append(file_path) + open(file_path, 'w').close() + self.assertIn(ood_upload_testcase.get_file_path(self.temp_dir), file_paths) + + def test_with_subdirectory(self): + """Test for directory with subdirectory""" + os.makedirs(os.path.join(self.temp_dir, 'subdir')) + self.assertIsNone(ood_upload_testcase.get_file_path(self.temp_dir)) + + file_name = 'test_file.txt' + file_path = os.path.join(self.temp_dir, file_name) + open(file_path, 'w').close() + self.assertEqual(ood_upload_testcase.get_file_path(self.temp_dir), + file_path)