From 3a968e4b8e0982e8e8bbe208ab02e0638978f148 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Mon, 22 Jul 2024 18:07:34 +0000 Subject: [PATCH 01/22] ready for review Signed-off-by: Yizhen Zhang --- README.md | 90 +++++++ build_scripts/build_collect_diff.sh | 162 ++++++++++++ build_scripts/git_shortlog.sh | 20 ++ build_scripts/parse_dep_list.py | 71 ++++++ build_scripts/parse_json.py | 75 ++++++ build_scripts/tokenize.py | 331 +++++++++++++++++++++++++ fixdep-patch.file | 50 ++++ generate_build_filelists.sh | 108 ++++++++ generate_web_reports.sh | 27 ++ run_tool.sh | 59 +++++ web_scripts/generate_data.sh | 51 ++++ web_source_template/header.html | 369 ++++++++++++++++++++++++++++ web_source_template/index.html | 40 +++ web_source_template/sourcecode.html | 369 ++++++++++++++++++++++++++++ web_source_template/versions.js | 1 + 15 files changed, 1823 insertions(+) create mode 100644 README.md create mode 100755 build_scripts/build_collect_diff.sh create mode 100755 build_scripts/git_shortlog.sh create mode 100644 build_scripts/parse_dep_list.py create mode 100755 build_scripts/parse_json.py create mode 100755 build_scripts/tokenize.py create mode 100644 fixdep-patch.file create mode 100755 generate_build_filelists.sh create mode 100755 generate_web_reports.sh create mode 100755 run_tool.sh create mode 100755 web_scripts/generate_data.sh create mode 100644 web_source_template/header.html create mode 100644 web_source_template/index.html create mode 100644 web_source_template/sourcecode.html create mode 100644 web_source_template/versions.js diff --git a/README.md b/README.md new file mode 100644 index 0000000..fb19ebc --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# Change-impact-analysis Tool + +The Change Impact Analysis Tool generates a comprehensive visual report detailing changes in both header files and source code between two Linux versions (tags in the Linux kernel repository: old_tag and new_tag). This tool helps developers view updates from the old version. + +The diff report includes a subset of files from the Linux repository that are included in building the kernel, contributing to a focused and detailed report on the compile-time source code in Linux. + +## Table of Content + +- [How to Use](#how-to-use) +- [Files Generated](#files-generated) +- [Structure of the Tool](#structure-of-the-tool) + - [I. Compilation File List Generation](#i-compilation-file-list-generation) + - [II. Git Diff Report Generation](#ii-git-diff-report-generation) + - [III. Commit Metadata Retrieval](#iii-commit-metadata-retrieval) + - [IV. Web Script Generation](#iv-web-script-generation) + +## How to use + +To utilize this tool in your Linux environment (compatible with Ubuntu and Debian), follow these steps: + +Clone the repository: + +```bash +git clone +``` + +Navigate to the cloned repository: + +```bash +cd +``` + +Execute the tool by specifying the old and new tags: + +```bash +./run_tool.sh +``` + +## Files Generated + +**/build_data:** + +- `sourcefile.txt` - List of all built source code files +- `headerfile.txt` - List of all built dependency files +- `git_diff_sourcefile.txt` - Git diff report for source code files +- `git_diff_headerfile.txt` - Git diff report for dependency files +- `tokenize_header.json` - Metadata for commit git diff for dependency files +- `tokenize_source.json` - Metadata for commit git diff for source files + +**/web_source_codes:** + +- `index.html` - Click on to view the result + +## Structure of the Tool + +The tool operates through a structured process to generate a comprehensive change impact analysis report. Here's a detailed breakdown of its operation: + +### I. Compilation File List Generation + +#### Header File + +During linux kernel compilation, `Makefile.build` calls `$K/scripts/basic/fixdep.c` to generate a .cmd file for each source that collects dependency information during compilation. + +This tool incorporates a modification that applies a patch (`patch.file`) to `scripts/basic/fixdep.c`, enabling it to output dependency information into a **list of header files** when building the kernel. + +#### Source code + +This tool leverages the `$K/scripts/clang-tools/gen_compile_commands.py` script to generate a `compile_commands.json` file. This file documents all source files involved in the compilation process. The `gen_compile_commands.py` script traverses each `.cmd` file to aggregate the list of source files. + +Then, the tool invokes `parse_json.py` to parse `compile_commands.json`, generating **a list of source files**. + +### II. Git Diff Report Generation + +Using the file lists, the tool generates 2 separate git diff reports (dependency diff report & source diff report) for updates from **old_tag** to **new_tag**. + +### III. Commit Metadata Retrieval + +Based on the git diff reports, the tool retrieves commit metadata for each newly added line in the reports. + +- **Tokenization**: If multiple commits modify a single line between two tags, the tool breaks down each commit line into smaller parts and associates commit information with relevant tokens. The results after tokenization are stored in JSON files. + +### IV. Web Script Generation + +Using the git diff reports and metadata stored in JSON files, the tool generates a web report displaying the changes. + +The web report contains three source html: + +- `index.html`: with on-click directions to: + - `sourcecode.html`: renders the content in source diff report, with embedded url and on-hover metadata box for each newly added lines/tokens in new_tag. + - `header.html`: renders teh content in dependency diff report, with embedded url and on-hover metadata box for each newly added lines/tokens in new_tag. diff --git a/build_scripts/build_collect_diff.sh b/build_scripts/build_collect_diff.sh new file mode 100755 index 0000000..ed91847 --- /dev/null +++ b/build_scripts/build_collect_diff.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# +# Script to build the kernel, collect compiled file lists using modified kernel scripts, +# and generate a git diff report based on the collected lists. + +set -e + +DEFAULT_TAG1="v6.9" +DEFAULT_TAG2="v6.10" +TAG1="${TAG1_ENV:-$DEFAULT_TAG1}" +TAG2="${TAG2_ENV:-$DEFAULT_TAG2}" + +# check and install gcc-11 if not already installed +install_package_safe() { + if ! command -v gcc-11 &> /dev/null; then + sudo apt update + sudo apt install gcc-11 + else + echo "GCC-11 is already installed." + fi + if ! command -v libssl-dev &> /dev/null; then + sudo apt-get update + sudo apt-get install -y libssl-dev + else + echo "libssl-dev is already installed." + fi +} + +# safely apply a patch to linux kernel +apply_patch() { + # shellcheck disable=SC2154 + local patch_path="$root_path/scripts/change-impact-tools/fixdep-patch.file" + + # Stash any changes if there is any + if ! git diff --quiet; then + git stash + fi + + # Abort `git am` only if there is a patch being applied + if git am --show-current-patch &> /dev/null; then + git am --abort + fi + echo "path check: $(pwd)" + git apply < "$patch_path" + echo "applied the git patch" + echo "path check: $(pwd)" +} + +# parse the JSON file +parse_source_json_file() { + local python_path="$root_path/scripts/change-impact-tools/build_scripts/parse_json.py" + # shellcheck disable=SC2154 + local cloned_repo_name="/$clone_dir/" + local input_path="$root_path/scripts/clang-tools/compile_commands.json" + local output_path="$root_path/scripts/change-impact-tools/build_data/sourcefile.txt" + + display_file_head "$root_path/scripts/clang-tools" "compile_commands.json" 3 + python3 "$python_path" "$cloned_repo_name" "$input_path" "$output_path" + display_file_head "$root_path/scripts/change-impact-tools/build_data" "sourcefile.txt" 3 +} + +# generate the build file list after building the kernel +generate_compiled_file_lists() { + # Generate compiled source files list + local json_output_path="$root_path/scripts/clang-tools/compile_commands.json" + echo "path check: $(pwd)" + python3 scripts/clang-tools/gen_compile_commands.py -o "$json_output_path" + + parse_source_json_file + echo "source compiled filelist generated to sourcefile.txt" + + # Generate compiled header files list + + local output_list="$root_path/scripts/change-impact-tools/build_data/headerfile.txt" + local output_json="$root_path/scripts/change-impact-tools/build_data/source_dep.json" + local dep_path="dependency_file.txt" + local python_tool_path="$root_path/scripts/change-impact-tools/build_scripts/parse_dep_list.py" + + python3 "$python_tool_path" "$dep_path" "$output_json" "$output_list" + echo "dependency compiled filelist generated to headerfile.txt$" + +} + +# clean up the working directory +cleanup_working_directory() { + git reset --hard + git clean -fdx +} + +# generate diff for build between TAG1 and TAG2 +generate_git_diff() { + + # collect and setup input & output file + file_type=${1:-source} + local root_build_data_path="$root_path/scripts/change-impact-tools/build_data" + local diff_input="$root_build_data_path/sourcefile.txt" + local diff_output="$root_build_data_path/filtered_diff_source.txt" + + if [ "$file_type" = "header" ]; then + echo "[generate_git_diff] Generating dependency git diff report ..." + diff_input="$root_build_data_path/headerfile.txt" + diff_output="$root_build_data_path/filtered_diff_header.txt" + else + echo "[generate_git_diff] Generating source git diff report ..." + fi + + while IFS= read -r file + do + if git show "$TAG2:$file" &> /dev/null; then + local diff_result + diff_result=$(git diff "$TAG1" "$TAG2" -- "$file") + if [[ -n "$diff_result" ]]; then + { + echo "Diff for $file" + echo "$diff_result" + echo "" + } >> "$diff_output" + + fi + fi + done < "$diff_input" + echo "[generate_git_diff] Git diff report for $file_type files save to compiled_data" + +} + + +if [ $# -eq 2 ]; then + TAG1="$1" + TAG2="$2" +fi + +# Fetch tags from the repository +git fetch --tags +echo "Generating source file list for $TAG1" +git checkout "$TAG1" +echo "starting to run make olddefconfig" +make olddefconfig +echo "finished make olddefconfig" + + +# Preparation before running make +apply_patch +install_package_safe + +# Build linux kernel +echo "the current os-linux version: " +cat /etc/os-release + +echo "start running make" +make HOSTCC=gcc-11 CC=gcc-11 +echo "finished compile kernel using gcc 11" + + +# Collect build metadata +generate_compiled_file_lists + +# Generate git diff report +generate_git_diff source +generate_git_diff header + +# Clean up the working directory +cleanup_working_directory diff --git a/build_scripts/git_shortlog.sh b/build_scripts/git_shortlog.sh new file mode 100755 index 0000000..b04c9dd --- /dev/null +++ b/build_scripts/git_shortlog.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Fetch name email information for linux kernel contributors + +DEFAULT_TAG="v6.10" +TAG="${1:-$DEFAULT_TAG}" +git checkout "$TAG" + +echo "Starting to generate the email name list ..." +# shellcheck disable=SC2154 +git shortlog -e -s -n HEAD > "$curr_dir"/build_data/name_list.txt + +# shellcheck disable=SC2154 +if [ -s "$curr_dir"/build_data/name_list.txt ]; then + echo "build_data/name_list.txt created successfully" +else + echo "build_data/name_list.txt is empty or not created" +fi + +echo "Finished generating name list" diff --git a/build_scripts/parse_dep_list.py b/build_scripts/parse_dep_list.py new file mode 100644 index 0000000..2c5a08a --- /dev/null +++ b/build_scripts/parse_dep_list.py @@ -0,0 +1,71 @@ +""" +The script parses the dependency list generated by patching `fixdep.c`. + +This script takes three arguments: +1. The path of dependency list +2. The output path for a json file +3. The output path for the list of header files. + +Usage: + parse_json.py + +""" +import re +import argparse +import json + +# Regular expression patterns +source_file_pattern = re.compile(r'^source file := (.+)$') + +# Function to parse the input data + + +def parse_dependencies(dep_list_file, output_json, output_dep_list): + """Parse dependency file generated by 'fixdep.c'.""" + dependencies = [] + dep_set = set() + current_source_file = None + + for line in dep_list_file: + line = line.strip() + if not line: + continue + + source_match = source_file_pattern.match(line) + if source_match: + current_source_file = source_match.group(1) + dependencies.append({ + 'source_file': current_source_file, + 'dependency_files': [] + }) + else: + dependencies[-1]['dependency_files'].append(line) + dep_set.add(line) + + # Write dependency list to output file + with open(output_dep_list, 'w', encoding='utf-8') as output_list_file: + for header_file in dep_set: + output_list_file.write(header_file + '\n') + + # Dump dependencies into JSON file + with open(output_json, 'w', encoding='utf-8') as json_file: + json.dump(dependencies, json_file, indent=4) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process dependency list generated while compiling kernel.") + parser.add_argument('input_file', type=str, + help="Path to input dependency file") + parser.add_argument('output_json', type=str, + help="Path to output JSON file") + parser.add_argument('output_header_list', type=str, + help="Path to output dependency list file") + + args = parser.parse_args() + + with open(args.input_file, 'r', encoding='utf-8') as input_file: + parse_dependencies(input_file, args.output_json, + args.output_header_list) + + print("Dependency parsing complete.") diff --git a/build_scripts/parse_json.py b/build_scripts/parse_json.py new file mode 100755 index 0000000..f597c00 --- /dev/null +++ b/build_scripts/parse_json.py @@ -0,0 +1,75 @@ +""" +The script collects a list of compile-time source files. + +Running `$K/scripts/clang-tools/gen_compile_commands.py` post kernel build +generates `compile_commands.json`. + +This file is created by parsing `.cmd` files to collect all metadata for the +source code. + +This script parses `compile_commands.json` to collect all source files into +a list `sourcefile.txt`. + +This script takes three arguments: +1. The root path of the Linux repository. +2. The path to the `compile_commands.json` file. +3. The output path for the list of source files. + +Usage: + parse_json.py + +""" + +import json +import os +import argparse + + +def load_json(file_path): + """Load a JSON file.""" + with open(file_path, encoding='utf-8') as file: + return json.load(file) + + +def find_common_parent(data): + """Find the common parent directory dynamically.""" + return os.path.commonpath([entry['file'] for entry in data]) + + +def extract_source_files(data, common_parent): + """Extract .c files with full paths and store them in a list.""" + return [entry['file'].replace(common_parent, '').lstrip('/') + for entry in data if entry['file'].endswith('.c')] + + +def write_to_file(file_list, output_file, common_parent, cloned_repo): + """Write the list of .c files to the output text file.""" + if os.path.exists(output_file): + print(f"{output_file} already exists and will be overwritten.") + with open(output_file, 'w', encoding='utf-8') as file: + for file_path in file_list: + # Combine with common_parent + full_path = os.path.join(common_parent, file_path) + repo_index = full_path.find(cloned_repo) # fix + if repo_index != -1: + relative_path = full_path[repo_index + len(cloned_repo):] + file.write(relative_path.strip() + '\n') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process and extract C files from compile_commands.json") + parser.add_argument('cloned_repo_name', type=str, help="clone repo name") + parser.add_argument('input_file', type=str, + help="Path to the compile_commands.json file") + parser.add_argument('output_file', type=str, + help="Path to the output file to save the list of source files") + + args = parser.parse_args() + + input_data = load_json(args.input_file) + common_roots = find_common_parent(input_data) + source_files = extract_source_files(input_data, common_roots) + write_to_file(source_files, args.output_file, + common_roots, args.cloned_repo_name) + print(f"Source files list saved in {args.output_file}", flush=True) diff --git a/build_scripts/tokenize.py b/build_scripts/tokenize.py new file mode 100755 index 0000000..60d4c4d --- /dev/null +++ b/build_scripts/tokenize.py @@ -0,0 +1,331 @@ +""" +This script parses a git diff report, fetches commit details, and tokenizes the results. + +This script takes three arguments: +1. The path to `git_diff_report.txt`. +2. The path to an intermediate file for storing diff information. +3. The path to the output file for storing the tokenized results. + +Usage: + tokenize.py +""" + +import subprocess +import re +import json +import argparse +from datetime import datetime + +CHUNK_HEADER_PATTERN = r'^@@ -\d+,\d+ \+(\d+),\d+ @@' +COMMIT_REGEX = r'^commit ([0-9a-f]{40})$' +AUTHOR_REGEX = r'^Author: (.+)$' +DATE_REGEX = r'^Date:\s+(.+)$' +OBJECT_ID_REGEX = r'^@@ -\d+,\d+ \+(?P\d+),(?P\d+) @@' +diff_info = {} + + +def run_git_command(arguments): + """Run git command in python.""" + result = subprocess.run(arguments, capture_output=True, + text=True, check=True) + + return result.stdout + + +def parse(git_dif_path): + """Parse git diff report.""" + added_lines_parse = {} + start_file_content = True + line_number = 0 + + with open(git_dif_path, 'r', encoding='utf-8') as git_diff_report: + + # Parse the git diff report + current_file = None + for line in git_diff_report: + if line.startswith('Diff for '): + start_file_content = False + current_file = line.split(' ')[2][:-1] # get rid of \n + added_lines_parse[current_file] = [] + + if start_file_content and not line.startswith('-'): + line_number += 1 + + if current_file and re.search(CHUNK_HEADER_PATTERN, line): + # Extract line number from @@ -983,14 +983,19 @@ + match = re.search(CHUNK_HEADER_PATTERN, line) + start_file_content = True + line_number = int(match.group(1)) - 1 + + if start_file_content and line.startswith('+'): + added_lines_parse[current_file].append(line_number) + + return added_lines_parse + + +def get_log_line(tag1, tag2, line_num, file_path, git_blame): + """Get all log line report for each line.""" + if git_blame: + command = ['git', 'blame', '-L', f'{line_num},{line_num}', file_path] + else: + command = ['git', 'log', f'{tag1}..{tag2}', + f'-L{line_num},{line_num}:{file_path}'] + log_commit_info = run_git_command(command) + return log_commit_info.splitlines() # Split the output into lines + + +def parse_blame_line(blame_line): + """Parse blame line to retrieve commit metadata.""" + # Regular expression to match the blame line + if blame_line[0] == '^': + blame_line = blame_line[1:] + + regex = r'^([0-9a-f]+) \(([^)]+) (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [+-]\d{4})' \ + r'\s+\d+\)\s?(.*)$' + + match = re.match(regex, blame_line) + if match: + commit_hash = match.group(1) + author_name = match.group(2) + date = match.group(3) + line_content = match.group(4) if match.group(4) else "" + return { + "Commit_Hash": commit_hash, + "Author": author_name, + "Date": date, + "Line": line_content + } + + return None + + +def parse_git_add(added_lines_parse): + """Parse git add info.""" + print("Inside parse git add", flush=True) + + for file, line_nums in added_lines_parse.items(): + diff_info[file] = {} + print(f"Parsing git line info in file: {file}", flush=True) + for line_num in line_nums: + log_commit_info = get_log_line(TAG1, TAG2, line_num, file, False) + + if len(log_commit_info) == 0: # need to use git blame retrieve all commit history + log_commit_info = get_log_line( + TAG1, TAG2, line_num, file, True) + diff_info.setdefault(file, {}).setdefault( + line_num, []).append(parse_blame_line(log_commit_info[0])) + + else: + process_commit_info(log_commit_info, file, line_num) + + +def process_commit_info(lines, file, line_num): + """Process commit info.""" + cnt = -1 + line_cnt = 0 + for line in lines: + if re.match(COMMIT_REGEX, line): + cnt += 1 + current_commit = re.match(COMMIT_REGEX, line).group(1) + diff_info.setdefault(file, {}).setdefault(line_num, []).append({ + 'Author': '', + 'Date': '', + 'Commit_Hash': current_commit, + 'Line': '' + }) + + elif re.match(AUTHOR_REGEX, line): + diff_info[file][line_num][cnt]['Author'] = re.match( + AUTHOR_REGEX, line).group(1) + + elif re.match(DATE_REGEX, line): + diff_info[file][line_num][cnt]['Date'] = re.match( + DATE_REGEX, line).group(1) + + elif re.search(OBJECT_ID_REGEX, line): + i = 1 + match = re.search(OBJECT_ID_REGEX, line) + max_cnt = int(match.group('n4')) + start_line = int(match.group('n3')) - 1 + add_cnt = 1 + while add_cnt <= max_cnt: + next_line = lines[i + line_cnt] + if next_line[0] != '-': + add_cnt += 1 + start_line += 1 + if next_line[0] == '+' and start_line == line_num: + diff_info[file][line_num][cnt]['Line'] = next_line + i += 1 + + line_cnt += 1 + + +def tokenize(json_path, output_json_path): + """Tokenize git added line with relevant commit message.""" + result_data = {} + with open(json_path, encoding='utf-8') as git_history_file: + data = json.load(git_history_file) + + for file_name in data: + if file_name not in result_data: + result_data[file_name] = {} + + for line_num in data[file_name]: + commit_list = data[file_name][line_num] + result_data[file_name][line_num] = {} + + try: + result_data[file_name][line_num]["root_line"] = commit_list[0]["Line"] + except (TypeError, KeyError) as exception: + # Parsing Error + print(f"Error accessing commit_list: {exception}", flush=True) + continue + + if len(commit_list) > 1: + + result_data[file_name][line_num]["tokenization"] = highlight_substring( + commit_list) + else: + result_data[file_name][line_num]["tokenization"] = [ + { + "token": commit_list[0]["Line"], + "meta": { + "commit_hash": commit_list[0]["Commit_Hash"], + "author_name": parse_author(commit_list[0]["Author"])[0], + "email": parse_author(commit_list[0]["Author"])[1], + "time": format_date(commit_list[0]["Date"]) + } + } + ] + + with open(output_json_path, 'w', encoding='utf-8') as json_output_file: + json.dump(result_data, json_output_file, ensure_ascii=False, indent=4) + + +def parse_author(author): + """Parse author string to extract name and email.""" + match = re.match(r'^(.*)\s*<([^>]+)>$', author) + if match: + return [match.group(1).strip(), match.group(2).strip()] + return [author, "None"] + + +def format_date(date_str): + """Format date to 'YYYY-MM-DD HH:MM:SS'.""" + # Define possible date formats + formats = [ + '%a %b %d %H:%M:%S %Y %z', # Format: 'Wed Jan 19 18:08:56 2022 -0800' + '%Y-%m-%d %H:%M:%S %z' # Format: '2022-01-19 18:08:56 -0800' + ] + + # Try to parse the date string with the defined formats + for date_format in formats: + try: + date_obj = datetime.strptime(date_str, date_format) + return date_obj.strftime('%Y-%m-%d %H:%M:%S') + except ValueError: + continue # Try the next format if current one fails + + # If none of the formats match, raise an error + raise ValueError(f"Date format not recognized: {date_str}") + + +def highlight_substring(commit_list): + """Produce tokenized JSON with highlighted substrings based on overlap.""" + # Retrieve lines info for string parsing + root_line, child_line = commit_list[0]["Line"], commit_list[1]["Line"] + root_length, child_length = len(root_line), len(child_line) + + # Retrieve other meta info for lines + meta_root = { + "commit_hash": commit_list[0]["Commit_Hash"], + "author_name": parse_author(commit_list[0]["Author"])[0], + "email": parse_author(commit_list[0]["Author"])[1], + "time": format_date(commit_list[0]["Date"]) + } + + meta_child = { + "commit_hash": commit_list[1]["Commit_Hash"], + "author_name": parse_author(commit_list[1]["Author"])[0], + "email": parse_author(commit_list[1]["Author"])[1], + "time": format_date(commit_list[1]["Date"]) + } + + # Return val + segments = [] + + # Find overlap at the start + start_idx = 0 + for i in range(min(root_length, child_length)): + if root_line[i] != child_line[i]: + break + start_idx = i + 1 + + # Find overlap at the end + end_idx = 0 + for i in range(1, min(root_length, child_length) + 1): + if root_line[-i] != child_line[-i]: + break + end_idx = i + + start_overlap = root_line[:start_idx] + end_overlap = root_line[-end_idx:] if end_idx > 0 else "" + mid_segment = root_line[start_idx:root_length - + end_idx] if start_idx < root_length - end_idx else "" + + # Case 1: start + mid + end + if start_overlap and end_overlap and mid_segment: + segments.append({"token": start_overlap, "meta": meta_child}) + segments.append({"token": mid_segment, "meta": meta_root}) + segments.append({"token": end_overlap, "meta": meta_child}) + + # Case 2: start + end (mid is empty) + elif start_overlap and not mid_segment and end_overlap: + segments.append({"token": start_overlap, "meta": meta_child}) + segments.append({"token": end_overlap, "meta": meta_child}) + + # Case 3: mid + end (start is empty) + elif not start_overlap and mid_segment and end_overlap: + segments.append({"token": mid_segment, "meta": meta_root}) + segments.append({"token": end_overlap, "meta": meta_child}) + # Case 4: no overlap + else: + segments.append({"token": root_line, "meta": meta_root}) + + return segments + + +# Main execution +if __name__ == '__main__': + # Parse arguments + parser = argparse.ArgumentParser(description='Process some files.') + parser.add_argument('git_diff_report', type=str, + help='Path to git_diff_report.txt') + parser.add_argument('intermediate_file', type=str, + help='Path to intermediate_file') + parser.add_argument('output_file', type=str, help='Path to output_file') + parser.add_argument('tag1', type=str, help='old version tag') + parser.add_argument('tag2', type=str, help='new verison tag') + + args = parser.parse_args() + + git_diff_report_path = args.git_diff_report + intermediate_file_path = args.intermediate_file + output_file_path = args.output_file + TAG1 = args.tag1 + TAG2 = args.tag2 + + # Identify all the (line number in git diff report) + added_lines = parse(git_diff_report_path) + + # Retrieve git report + parse_git_add(added_lines) + + with open(intermediate_file_path, 'w', encoding='utf-8') as the_file: + json.dump(diff_info, the_file, indent=4) + print(f"Diff info stored in {intermediate_file_path}") + + # Tokenize + print("starting tokenize function test", flush=True) + tokenize(intermediate_file_path, output_file_path) + print("Result exported to tokenize.json", flush=True) diff --git a/fixdep-patch.file b/fixdep-patch.file new file mode 100644 index 0000000..a047054 --- /dev/null +++ b/fixdep-patch.file @@ -0,0 +1,50 @@ +diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c +index 44e887cff49b..acf2b6b4c0b7 100644 +--- a/scripts/basic/fixdep.c ++++ b/scripts/basic/fixdep.c +@@ -98,7 +98,7 @@ + #include + #include + #include +- ++FILE *build_dep_file; + static void usage(void) + { + fprintf(stderr, "Usage: fixdep \n"); +@@ -321,10 +321,15 @@ static void parse_dep_file(char *m, const char *target) + xprintf("source_%s := %s\n\n", + target, m); + xprintf("deps_%s := \\\n", target); ++ fprintf(build_dep_file, "source file := %s\n", m); ++ fprintf(build_dep_file, "dependency files :\n"); ++ + } + is_first_dep = 0; + } else { + xprintf(" %s \\\n", m); ++ fprintf(build_dep_file, " %s\n", m); ++ + } + + buf = read_file(m); +@@ -349,6 +354,7 @@ static void parse_dep_file(char *m, const char *target) + + xprintf("\n%s: $(deps_%s)\n\n", target, target); + xprintf("$(deps_%s):\n", target); ++ fprintf(build_dep_file, "\n"); + } + + int main(int argc, char *argv[]) +@@ -364,7 +370,11 @@ int main(int argc, char *argv[]) + cmdline = argv[3]; + + xprintf("cmd_%s := %s\n\n", target, cmdline); +- ++ build_dep_file = fopen("dependency_file.txt", "a"); ++ if (!build_dep_file) { ++ perror("fixdep: ./dependency_file.txt"); ++ exit(1); ++ } + buf = read_file(depfile); + parse_dep_file(buf, target); + free(buf); diff --git a/generate_build_filelists.sh b/generate_build_filelists.sh new file mode 100755 index 0000000..c3a7b20 --- /dev/null +++ b/generate_build_filelists.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# This script compiles kernel and collects a list of complite-time files and related Git metadata +# for each newly added line from tag 1 to tag 2. + +show_help() { + echo "Usage: ./generate_build_filelists.sh [tag1] [tag2]" + echo "Example: ./generate_rugenerate_build_filelistsntime_web_reports.sh tag1 tag2" + echo + echo "Options:" + echo " tag1 Optional. Specify tag1 (default: $DEFAULT_TAG1)" + echo " tag2 Optional. Specify tag2 (default: $DEFAULT_TAG2)" + echo " -h Display this help message" + exit 0 +} + +clone_and_goto_cloned_repo() { + clone_dir=${1:-"linux-clone"} + + # Capture the absolute path of the root directory + root_path=$(cd ../.. && pwd) + parent_root_path=$(cd ../../.. && pwd) + clone_root_path="$parent_root_path/$clone_dir" + echo "root_path: $root_path" + echo "clone root path: $clone_root_path" + export clone_root_path + export root_path + export clone_dir + + mkdir -p "$clone_root_path" + + if [ ! -d "$clone_root_path/.git" ]; then + git clone "$root_path" "$clone_root_path" + fi + cd "$clone_root_path" || exit +} + +check_and_create_dir() { + local new_dir_name="$1" + + if [ ! -d "$new_dir_name" ]; then + mkdir -p "$new_dir_name" + echo "Folder '$new_dir_name' created." + else + echo "Folder '$new_dir_name' already exists." + fi +} + +display_file_head() { + local dir_name="$1" + local file_name="$2" + local line_num="$3" + + echo "First $line_num lines of $file_name:" + head -n "$line_num" "$dir_name/$file_name" + echo +} + +export -f display_file_head + +if [[ "$1" == "-h" ]]; then + show_help +fi + +DEFAULT_TAG1="v6.9" +DEFAULT_TAG2="v6.10" +TAG1="${1:-$DEFAULT_TAG1}" +TAG2="${2:-$DEFAULT_TAG2}" +DEFAULT_CLONE_PATH="linux-clone" +CLONE_PATH="${3:-$DEFAULT_CLONE_PATH}" + +mkdir -p "build_data" + +curr_dir=$(pwd) +echo "$curr_dir" +export curr_dir + +clone_and_goto_cloned_repo "$CLONE_PATH" + +bash "$curr_dir"/build_scripts/build_collect_diff.sh "$TAG1" "$TAG2" +echo "Finishing generating build source file lists" + +# Fetch email and name pairing information for linux contributor +bash "$curr_dir"/build_scripts/git_shortlog.sh "$TAG2" +display_file_head "$curr_dir/build_data" "name_list.txt" 3 +echo "finished generating email and name list" + + +# Reframe the git diff report for javascript front-end syntax +input_file="$curr_dir/build_data/filtered_diff_header.txt" +output_file="$curr_dir/build_data/filtered_diff_header_replace.txt" +sed 's/\\/\\\\/g' "$input_file" > "$output_file" +display_file_head "$curr_dir/build_data" "filtered_diff_header_replace.txt" 3 + +input_file="$curr_dir/build_data/filtered_diff_source.txt" +output_file="$curr_dir/build_data/filtered_diff_source_replace.txt" +sed 's/\\/\\\\/g' "$input_file" > "$output_file" +display_file_head "$curr_dir/build_data" "filtered_diff_source_replace.txt" 3 + + +# Retrieve and tokenize commit info per added line +echo "tokenizing each commit line ..." +git checkout "$TAG2" +python3 "$curr_dir"/build_scripts/tokenize.py "$curr_dir/build_data/filtered_diff_header.txt" "$curr_dir/build_data/parse_git_header.json" "$curr_dir/build_data/tokenize_header.json" "$TAG1" "$TAG2" +python3 "$curr_dir"/build_scripts/tokenize.py "$curr_dir/build_data/filtered_diff_source.txt" "$curr_dir/build_data/parse_git_source.json" "$curr_dir/build_data/tokenize_source.json" "$TAG1" "$TAG2" +display_file_head "$curr_dir/build_data" "tokenize_source.json" 3 +display_file_head "$curr_dir/build_data" "tokenize_header.json" 3 +echo "finished tokenization" diff --git a/generate_web_reports.sh b/generate_web_reports.sh new file mode 100755 index 0000000..4c0b57d --- /dev/null +++ b/generate_web_reports.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Generate the web report source codes (*.html and *.js) for the web report + +function show_help { + echo "Usage: ./generate_build_web_reports.sh [tag1] [tag2]" + echo "Example: ./generate_build_web_reports.sh tag1 tag2" + echo + echo "Options:" + echo " tag1 Optional. Specify tag1 (default: $DEFAULT_TAG1)" + echo " tag2 Optional. Specify tag2 (default: $DEFAULT_TAG2)" + echo " -h Display this help message" + exit 0 +} + +if [[ "$1" == "-h" ]]; then + show_help +fi + +DEFAULT_TAG1="v6.9" +DEFAULT_TAG2="v6.10" +TAG1="${1:-$DEFAULT_TAG1}" +TAG2="${2:-$DEFAULT_TAG2}" + +./web_scripts/generate_data.sh "$TAG1" "$TAG2" + +echo "finishing generating webpage, report can be viewed in web_source_code/index.html" diff --git a/run_tool.sh b/run_tool.sh new file mode 100755 index 0000000..dec21cc --- /dev/null +++ b/run_tool.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# A script to run the tool with all stages + +show_help() { + echo "Usage: ./run_tool.sh [tag1] [tag2]" + echo "Example: ./run_tool.sh v6.9 v6.10" + echo + echo "Options:" + echo " tag1 Optional. Specify old_linux_version (default: $DEFAULT_TAG1)" + echo " tag2 Optional. Specify new_linux_version (default: $DEFAULT_TAG2)" + echo " -h Display this help message" + exit 0 +} + +if [[ "$1" == "-h" ]]; then + show_help +fi + +DEFAULT_TAG1="v6.9" +DEFAULT_TAG2="v6.10" +TAG1="${1:-$DEFAULT_TAG1}" +TAG2="${2:-$DEFAULT_TAG2}" +DEFAULT_CLONE_PATH="linux-clone" +CLONE_PATH="${3:-$DEFAULT_CLONE_PATH}" + +# Define the repository URL and the target directory +REPO_URL="https://github.com/gregkh/linux" +TARGET_DIR="linux/scripts/change-impact-tools" + +# Clone the repository +git clone $REPO_URL + +# Create the target directory if it doesn't exist +mkdir -p $TARGET_DIR + +# Copy necessary scripts to the target directory +cp -r build_scripts $TARGET_DIR/ +cp -r web_scripts $TARGET_DIR/ +cp fixdep-patch.file $TARGET_DIR/ +cp generate_build_filelists.sh $TARGET_DIR/ +cp generate_web_reports.sh $TARGET_DIR/ +mkdir -p $TARGET_DIR/web_source_code +mv web_source_template/* $TARGET_DIR/web_source_code/ + + +# Save the current directory +CUR_DIR=$(pwd) + +# Navigate to the target directory and execute the scripts +cd "$TARGET_DIR" || exit +./generate_build_filelists.sh "$TAG1" "$TAG2" "$CLONE_PATH" +./generate_web_reports.sh "$TAG1" "$TAG2" + +# Return to the original directory +cd "$CUR_DIR" || exit + +# Copy the web source code to the current directory +cp -r $TARGET_DIR/web_source_code . diff --git a/web_scripts/generate_data.sh b/web_scripts/generate_data.sh new file mode 100755 index 0000000..af9ec1d --- /dev/null +++ b/web_scripts/generate_data.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Generate metadata *.js files for web report + +echo "Generating metadata js files ..." +DEFAULT_TAG1="v6.9" +DEFAULT_TAG2="v6.10" +# shellcheck disable=SC2034 +TAG1="${1:-$DEFAULT_TAG1}" +# shellcheck disable=SC2034 +TAG2="${2:-$DEFAULT_TAG2}" + +# Generate front-end commit info metadata +echo "Generating source_data.js ..." +if [ ! -f "build_data/tokenize_source.json" ]; then + echo "Error: build_data/tokenize_source.json not found!" + exit 1 +fi +TOKEN_DATA_SOURCE=$(cat build_data/tokenize_source.json) +echo "let token_data = $TOKEN_DATA_SOURCE;" > web_source_code/source_data.js + +echo "Generating header_data.js ..." +if [ ! -f "build_data/tokenize_header.json" ]; then + echo "Error: build_data/tokenize_header.json not found!" + exit 1 +fi +TOKEN_DATA_HEADER=$(cat build_data/tokenize_header.json) +echo "let token_data = $TOKEN_DATA_HEADER;" > web_source_code/header_data.js + +# Generate front-end git diff report metadata +echo "Generating git_diff_source.js ..." +DIFF_OUTPUT_SOURCE=$(cat build_data/filtered_diff_source_replace.txt) +SOURCE_TRIM=${DIFF_OUTPUT_SOURCE//\`/\\\`} +# shellcheck disable=SC2001 +SOURCE_TRIM_ESCAPED=$(echo "$SOURCE_TRIM" | sed 's/\$/\\$/g') +echo "let diffs = \` +$SOURCE_TRIM_ESCAPED +\`.trim();" > web_source_code/git_diff_source.js + +echo "Generating git_diff_header.js ..." +DIFF_OUTPUT_HEADER=$(cat build_data/filtered_diff_header_replace.txt) +HEADER_TRIM=${DIFF_OUTPUT_HEADER//\`/\\\`} +# shellcheck disable=SC2001 +HEADER_TRIM_ESCAPED=$(echo "$HEADER_TRIM" | sed 's/\$/\\$/g') +echo "let diffs = \` +$HEADER_TRIM_ESCAPED +\`.trim();" > web_source_code/git_diff_header.js + +echo "let tag1 = \"$TAG1\"; let tag2 = \"$TAG2\";" > web_source_template/versions.js + +echo "SUCCESS generated metadata js files" diff --git a/web_source_template/header.html b/web_source_template/header.html new file mode 100644 index 0000000..44d889a --- /dev/null +++ b/web_source_template/header.html @@ -0,0 +1,369 @@ + + + + + + + + + +

Header File Git Diff Visualization

+

Comparing tags v5.15 and v5.15.100 in the linux source code repository.

+ + + + + + + + +
+ + + + + + + + diff --git a/web_source_template/index.html b/web_source_template/index.html new file mode 100644 index 0000000..c18e136 --- /dev/null +++ b/web_source_template/index.html @@ -0,0 +1,40 @@ + + + + + Index Page + + + + +

Git Diff Visualization

+ + + + diff --git a/web_source_template/sourcecode.html b/web_source_template/sourcecode.html new file mode 100644 index 0000000..0620c0e --- /dev/null +++ b/web_source_template/sourcecode.html @@ -0,0 +1,369 @@ + + + + + + + + + +

Source File Git Diff Visualization

+

Comparing tags v5.15 and v5.15.100 in the linux source code repository.

+ + + + + + + + +
+ + + + + + + + diff --git a/web_source_template/versions.js b/web_source_template/versions.js new file mode 100644 index 0000000..88c4dc3 --- /dev/null +++ b/web_source_template/versions.js @@ -0,0 +1 @@ +let tag1 = "v5.15"; let tag2 = "v5.16"; From 8a44553df00647351a38a3c53b18855d8faae629 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Mon, 22 Jul 2024 18:34:36 +0000 Subject: [PATCH 02/22] fix Signed-off-by: Yizhen Zhang --- web_source_template/versions.js | 1 - 1 file changed, 1 deletion(-) delete mode 100644 web_source_template/versions.js diff --git a/web_source_template/versions.js b/web_source_template/versions.js deleted file mode 100644 index 88c4dc3..0000000 --- a/web_source_template/versions.js +++ /dev/null @@ -1 +0,0 @@ -let tag1 = "v5.15"; let tag2 = "v5.16"; From b112be048b2f76cff4f378a6bb38a8e80c20003d Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Wed, 24 Jul 2024 14:21:46 +0000 Subject: [PATCH 03/22] allow user set linux repo url and add subsystem Signed-off-by: Yizhen Zhang --- build_scripts/build_collect_diff.sh | 9 +- build_scripts/git_shortlog.sh | 2 +- generate_build_filelists.sh | 22 +- generate_web_reports.sh | 23 +- run_tool.sh | 64 ++-- web_scripts/generate_data.sh | 42 ++- web_source_template/header.html | 369 -------------------- web_source_template/index.html | 504 ++++++++++++++++++++++++++-- web_source_template/sourcecode.html | 369 -------------------- 9 files changed, 574 insertions(+), 830 deletions(-) delete mode 100644 web_source_template/header.html delete mode 100644 web_source_template/sourcecode.html diff --git a/build_scripts/build_collect_diff.sh b/build_scripts/build_collect_diff.sh index ed91847..9343283 100755 --- a/build_scripts/build_collect_diff.sh +++ b/build_scripts/build_collect_diff.sh @@ -5,8 +5,8 @@ set -e -DEFAULT_TAG1="v6.9" -DEFAULT_TAG2="v6.10" +DEFAULT_TAG1="v5.15" +DEFAULT_TAG2="v5.15.100" TAG1="${TAG1_ENV:-$DEFAULT_TAG1}" TAG2="${TAG2_ENV:-$DEFAULT_TAG2}" @@ -51,10 +51,9 @@ parse_source_json_file() { local python_path="$root_path/scripts/change-impact-tools/build_scripts/parse_json.py" # shellcheck disable=SC2154 local cloned_repo_name="/$clone_dir/" - local input_path="$root_path/scripts/clang-tools/compile_commands.json" + local input_path="$root_path/scripts/change-impact-tools/build_data/compile_commands.json" local output_path="$root_path/scripts/change-impact-tools/build_data/sourcefile.txt" - display_file_head "$root_path/scripts/clang-tools" "compile_commands.json" 3 python3 "$python_path" "$cloned_repo_name" "$input_path" "$output_path" display_file_head "$root_path/scripts/change-impact-tools/build_data" "sourcefile.txt" 3 } @@ -62,7 +61,7 @@ parse_source_json_file() { # generate the build file list after building the kernel generate_compiled_file_lists() { # Generate compiled source files list - local json_output_path="$root_path/scripts/clang-tools/compile_commands.json" + local json_output_path="$root_path/scripts/change-impact-tools/build_data/compile_commands.json" echo "path check: $(pwd)" python3 scripts/clang-tools/gen_compile_commands.py -o "$json_output_path" diff --git a/build_scripts/git_shortlog.sh b/build_scripts/git_shortlog.sh index b04c9dd..4ee16c8 100755 --- a/build_scripts/git_shortlog.sh +++ b/build_scripts/git_shortlog.sh @@ -2,7 +2,7 @@ # # Fetch name email information for linux kernel contributors -DEFAULT_TAG="v6.10" +DEFAULT_TAG="v5.15.100" TAG="${1:-$DEFAULT_TAG}" git checkout "$TAG" diff --git a/generate_build_filelists.sh b/generate_build_filelists.sh index c3a7b20..046da2d 100755 --- a/generate_build_filelists.sh +++ b/generate_build_filelists.sh @@ -5,11 +5,12 @@ show_help() { echo "Usage: ./generate_build_filelists.sh [tag1] [tag2]" - echo "Example: ./generate_rugenerate_build_filelistsntime_web_reports.sh tag1 tag2" + echo "Example: ./generate_rugenerate_build_filelists.h tag1 tag2 clone_path" echo echo "Options:" - echo " tag1 Optional. Specify tag1 (default: $DEFAULT_TAG1)" - echo " tag2 Optional. Specify tag2 (default: $DEFAULT_TAG2)" + echo " tag1 Required. old_tag." + echo " tag2 Required. new_tag." + echo " clone_path Requred. the path to clone the linux repo to apply change diff analysis." echo " -h Display this help message" exit 0 } @@ -62,12 +63,15 @@ if [[ "$1" == "-h" ]]; then show_help fi -DEFAULT_TAG1="v6.9" -DEFAULT_TAG2="v6.10" -TAG1="${1:-$DEFAULT_TAG1}" -TAG2="${2:-$DEFAULT_TAG2}" -DEFAULT_CLONE_PATH="linux-clone" -CLONE_PATH="${3:-$DEFAULT_CLONE_PATH}" +# Ensure required arguments (tag1 and tag2) are provided +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 " + exit 1 +fi + +TAG1="$1" +TAG2="$2" +CLONE_PATH="$3" mkdir -p "build_data" diff --git a/generate_web_reports.sh b/generate_web_reports.sh index 4c0b57d..91b1005 100755 --- a/generate_web_reports.sh +++ b/generate_web_reports.sh @@ -3,12 +3,12 @@ # Generate the web report source codes (*.html and *.js) for the web report function show_help { - echo "Usage: ./generate_build_web_reports.sh [tag1] [tag2]" - echo "Example: ./generate_build_web_reports.sh tag1 tag2" + echo "Usage: ./generate_build_web_reports.sh [tag1] [tag2] [linux_repo_root_url]" + echo "Example: ./generate_build_web_reports.sh tag1 tag2 linux_url" echo echo "Options:" - echo " tag1 Optional. Specify tag1 (default: $DEFAULT_TAG1)" - echo " tag2 Optional. Specify tag2 (default: $DEFAULT_TAG2)" + echo " tag1 Required: old_linux_version [e.g. v6.8]" + echo " tag2 Required: new_linux_version [e.g. v6.9]" echo " -h Display this help message" exit 0 } @@ -17,11 +17,16 @@ if [[ "$1" == "-h" ]]; then show_help fi -DEFAULT_TAG1="v6.9" -DEFAULT_TAG2="v6.10" -TAG1="${1:-$DEFAULT_TAG1}" -TAG2="${2:-$DEFAULT_TAG2}" +# Ensure required arguments (tag1 and tag2) are provided +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 " + exit 1 +fi + +TAG1="$1" +TAG2="$2" +REPO_URL="$3" -./web_scripts/generate_data.sh "$TAG1" "$TAG2" +./web_scripts/generate_data.sh "$TAG1" "$TAG2" "$REPO_URL" echo "finishing generating webpage, report can be viewed in web_source_code/index.html" diff --git a/run_tool.sh b/run_tool.sh index dec21cc..002081b 100755 --- a/run_tool.sh +++ b/run_tool.sh @@ -2,34 +2,53 @@ # # A script to run the tool with all stages -show_help() { - echo "Usage: ./run_tool.sh [tag1] [tag2]" - echo "Example: ./run_tool.sh v6.9 v6.10" - echo - echo "Options:" - echo " tag1 Optional. Specify old_linux_version (default: $DEFAULT_TAG1)" - echo " tag2 Optional. Specify new_linux_version (default: $DEFAULT_TAG2)" - echo " -h Display this help message" - exit 0 -} - -if [[ "$1" == "-h" ]]; then - show_help +# Ensure required arguments (tag1 and tag2) are provided +if [ -z "$1" ] || [ -z "$2" ]; then + echo "Usage: $0 [clone_path] [linux_repo_root_url]" + exit 1 fi -DEFAULT_TAG1="v6.9" -DEFAULT_TAG2="v6.10" -TAG1="${1:-$DEFAULT_TAG1}" -TAG2="${2:-$DEFAULT_TAG2}" +TAG1="$1" +TAG2="$2" + +# Optional arguments +# Default values DEFAULT_CLONE_PATH="linux-clone" -CLONE_PATH="${3:-$DEFAULT_CLONE_PATH}" +DEFAULT_LINUX_REPO_URL="https://github.com/gregkh/linux" + +# Check the number of arguments and set optional parameters accordingly +if [ -n "$3" ] && [ -n "$4" ]; then + CLONE_PATH="$3" + REPO_URL="$4" +elif [ -n "$3" ] && [[ "$3" == *"github.com"* || "$3" == *"gitlab.com"* ]]; then + CLONE_PATH="$DEFAULT_CLONE_PATH" + REPO_URL="$3" +elif [ -n "$3" ]; then + CLONE_PATH="$3" + REPO_URL="$DEFAULT_LINUX_REPO_URL" +else + CLONE_PATH="$DEFAULT_CLONE_PATH" + REPO_URL="$DEFAULT_LINUX_REPO_URL" +fi + +# Determine if REPO_URL is a GitHub or GitLab URL +if [[ "$REPO_URL" == "https://github"* ]]; then + REPO_TYPE="github" +elif [[ "$REPO_URL" == "https://gitlab"* ]]; then + REPO_TYPE="gitlab" +else + REPO_TYPE="Unknown" + echo "Error: The repository URL ($REPO_URL) is neither a GitHub nor a GitLab URL." + exit 1 +fi + +echo "Repository URL ($REPO_URL) is identified as: $REPO_TYPE" -# Define the repository URL and the target directory -REPO_URL="https://github.com/gregkh/linux" TARGET_DIR="linux/scripts/change-impact-tools" # Clone the repository -git clone $REPO_URL +git clone "$REPO_URL" + # Create the target directory if it doesn't exist mkdir -p $TARGET_DIR @@ -50,10 +69,11 @@ CUR_DIR=$(pwd) # Navigate to the target directory and execute the scripts cd "$TARGET_DIR" || exit ./generate_build_filelists.sh "$TAG1" "$TAG2" "$CLONE_PATH" -./generate_web_reports.sh "$TAG1" "$TAG2" +./generate_web_reports.sh "$TAG1" "$TAG2" "$REPO_URL" # fix later # Return to the original directory cd "$CUR_DIR" || exit # Copy the web source code to the current directory cp -r $TARGET_DIR/web_source_code . +cp -r $TARGET_DIR/build_data . diff --git a/web_scripts/generate_data.sh b/web_scripts/generate_data.sh index af9ec1d..27cdbde 100755 --- a/web_scripts/generate_data.sh +++ b/web_scripts/generate_data.sh @@ -3,12 +3,26 @@ # Generate metadata *.js files for web report echo "Generating metadata js files ..." -DEFAULT_TAG1="v6.9" -DEFAULT_TAG2="v6.10" -# shellcheck disable=SC2034 -TAG1="${1:-$DEFAULT_TAG1}" -# shellcheck disable=SC2034 -TAG2="${2:-$DEFAULT_TAG2}" + +# Ensure required arguments (tag1 and tag2) are provided +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 " + exit 1 +fi + +TAG1="$1" +TAG2="$2" +REPO_URL="$3" + +# Determine if REPO_URL is a GitHub or GitLab URL +if [[ "$REPO_URL" == "https://github"* ]]; then + REPO_TYPE="github" +else + REPO_TYPE="gitlab" +fi + +echo "Repository URL ($REPO_URL) is identified as: $REPO_TYPE" + # Generate front-end commit info metadata echo "Generating source_data.js ..." @@ -17,7 +31,7 @@ if [ ! -f "build_data/tokenize_source.json" ]; then exit 1 fi TOKEN_DATA_SOURCE=$(cat build_data/tokenize_source.json) -echo "let token_data = $TOKEN_DATA_SOURCE;" > web_source_code/source_data.js +echo "let token_data_source = $TOKEN_DATA_SOURCE;" > web_source_code/source_data.js echo "Generating header_data.js ..." if [ ! -f "build_data/tokenize_header.json" ]; then @@ -25,7 +39,7 @@ if [ ! -f "build_data/tokenize_header.json" ]; then exit 1 fi TOKEN_DATA_HEADER=$(cat build_data/tokenize_header.json) -echo "let token_data = $TOKEN_DATA_HEADER;" > web_source_code/header_data.js +echo "let token_data_header = $TOKEN_DATA_HEADER;" > web_source_code/header_data.js # Generate front-end git diff report metadata echo "Generating git_diff_source.js ..." @@ -33,7 +47,7 @@ DIFF_OUTPUT_SOURCE=$(cat build_data/filtered_diff_source_replace.txt) SOURCE_TRIM=${DIFF_OUTPUT_SOURCE//\`/\\\`} # shellcheck disable=SC2001 SOURCE_TRIM_ESCAPED=$(echo "$SOURCE_TRIM" | sed 's/\$/\\$/g') -echo "let diffs = \` +echo "let diffs_source = \` $SOURCE_TRIM_ESCAPED \`.trim();" > web_source_code/git_diff_source.js @@ -42,10 +56,16 @@ DIFF_OUTPUT_HEADER=$(cat build_data/filtered_diff_header_replace.txt) HEADER_TRIM=${DIFF_OUTPUT_HEADER//\`/\\\`} # shellcheck disable=SC2001 HEADER_TRIM_ESCAPED=$(echo "$HEADER_TRIM" | sed 's/\$/\\$/g') -echo "let diffs = \` +echo "let diffs_header = \` $HEADER_TRIM_ESCAPED \`.trim();" > web_source_code/git_diff_header.js -echo "let tag1 = \"$TAG1\"; let tag2 = \"$TAG2\";" > web_source_template/versions.js + +cat < web_source_code/versions.js +let tag1 = "$TAG1"; +let tag2 = "$TAG2"; +let root_linux_url = "$REPO_URL"; +let repo_type = "$REPO_TYPE"; +EOF echo "SUCCESS generated metadata js files" diff --git a/web_source_template/header.html b/web_source_template/header.html deleted file mode 100644 index 44d889a..0000000 --- a/web_source_template/header.html +++ /dev/null @@ -1,369 +0,0 @@ - - - - - - - - - -

Header File Git Diff Visualization

-

Comparing tags v5.15 and v5.15.100 in the linux source code repository.

- - - - - - - - -
- - - - - - - - diff --git a/web_source_template/index.html b/web_source_template/index.html index c18e136..ab6e85f 100644 --- a/web_source_template/index.html +++ b/web_source_template/index.html @@ -1,40 +1,474 @@ - - - - - Index Page - + + +

Linux Kernel Git Diff Report

+

+ + + + + + + + + + + +
+ + + + + + + + + + + + + + diff --git a/web_source_template/sourcecode.html b/web_source_template/sourcecode.html deleted file mode 100644 index 0620c0e..0000000 --- a/web_source_template/sourcecode.html +++ /dev/null @@ -1,369 +0,0 @@ - - - - - - - - - -

Source File Git Diff Visualization

-

Comparing tags v5.15 and v5.15.100 in the linux source code repository.

- - - - - - - - -
- - - - - - - - From a498a18e73d419713984efc5a993e9cfd5bce306 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Thu, 25 Jul 2024 13:56:47 +0000 Subject: [PATCH 04/22] ready for review added subsystem flag Signed-off-by: Yizhen Zhang --- README.md | 7 ++- build_scripts/build_collect_diff.sh | 42 ++++++++------- build_scripts/git_shortlog.sh | 4 +- build_scripts/tokenize.py | 4 +- generate_build_filelists.sh | 6 ++- generate_web_reports.sh | 2 + run_tool.sh | 81 +++++++++++++++++------------ web_scripts/generate_data.sh | 20 ++----- web_source_template/index.html | 10 ++-- 9 files changed, 97 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index fb19ebc..ff5ec02 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,13 @@ cd Execute the tool by specifying the old and new tags: ```bash -./run_tool.sh +./run_tool.sh [-c clone_path] [-u repo_link] [-s subsystem] ``` +- ``: Specifies the old version tag. +- ``: Specifies the new version tag. +- `-c `: Optional. Defines the user-specified path to clone the Linux source code repository. +- `-u `: Optional. Provides the URL for the Linux source code repository. +- `-s `: Optional. Specifies the subsystem for analysis (e.g., -s security). ## Files Generated diff --git a/build_scripts/build_collect_diff.sh b/build_scripts/build_collect_diff.sh index 9343283..0e490fb 100755 --- a/build_scripts/build_collect_diff.sh +++ b/build_scripts/build_collect_diff.sh @@ -5,11 +5,6 @@ set -e -DEFAULT_TAG1="v5.15" -DEFAULT_TAG2="v5.15.100" -TAG1="${TAG1_ENV:-$DEFAULT_TAG1}" -TAG2="${TAG2_ENV:-$DEFAULT_TAG2}" - # check and install gcc-11 if not already installed install_package_safe() { if ! command -v gcc-11 &> /dev/null; then @@ -102,19 +97,23 @@ generate_git_diff() { else echo "[generate_git_diff] Generating source git diff report ..." fi - + echo "parsing for subsys: $SUBSYS" while IFS= read -r file do - if git show "$TAG2:$file" &> /dev/null; then - local diff_result - diff_result=$(git diff "$TAG1" "$TAG2" -- "$file") - if [[ -n "$diff_result" ]]; then - { - echo "Diff for $file" - echo "$diff_result" - echo "" - } >> "$diff_output" - + if [[ "$file" == $SUBSYS/* ]]; then + echo "now generating git diff for $file" + if git show "$TAG2:$file" &> /dev/null; then + local diff_result + echo "$file suitable for parse" + diff_result=$(git diff "$TAG1" "$TAG2" -- "$file") + if [[ -n "$diff_result" ]]; then + { + echo "Diff for $file" + echo "$diff_result" + echo "" + } >> "$diff_output" + + fi fi fi done < "$diff_input" @@ -123,10 +122,10 @@ generate_git_diff() { } -if [ $# -eq 2 ]; then - TAG1="$1" - TAG2="$2" -fi +TAG1="$1" +TAG2="$2" +SUBSYS="$3" +echo "build and collect kernel for subsystem: $SUBSYS" # Fetch tags from the repository git fetch --tags @@ -151,10 +150,13 @@ echo "finished compile kernel using gcc 11" # Collect build metadata +echo "starting on preparing compiled file list" generate_compiled_file_lists # Generate git diff report +echo "starting on generating git diff report on source" generate_git_diff source +echo "starting on generating git diff report on header" generate_git_diff header # Clean up the working directory diff --git a/build_scripts/git_shortlog.sh b/build_scripts/git_shortlog.sh index 4ee16c8..6175fec 100755 --- a/build_scripts/git_shortlog.sh +++ b/build_scripts/git_shortlog.sh @@ -1,9 +1,9 @@ #!/bin/bash # # Fetch name email information for linux kernel contributors +set -e -DEFAULT_TAG="v5.15.100" -TAG="${1:-$DEFAULT_TAG}" +TAG="$1" git checkout "$TAG" echo "Starting to generate the email name list ..." diff --git a/build_scripts/tokenize.py b/build_scripts/tokenize.py index 60d4c4d..d9b0966 100755 --- a/build_scripts/tokenize.py +++ b/build_scripts/tokenize.py @@ -24,7 +24,7 @@ diff_info = {} -def run_git_command(arguments): +def run_command(arguments): """Run git command in python.""" result = subprocess.run(arguments, capture_output=True, text=True, check=True) @@ -70,7 +70,7 @@ def get_log_line(tag1, tag2, line_num, file_path, git_blame): else: command = ['git', 'log', f'{tag1}..{tag2}', f'-L{line_num},{line_num}:{file_path}'] - log_commit_info = run_git_command(command) + log_commit_info = run_command(command) return log_commit_info.splitlines() # Split the output into lines diff --git a/generate_build_filelists.sh b/generate_build_filelists.sh index 046da2d..dfdcfa7 100755 --- a/generate_build_filelists.sh +++ b/generate_build_filelists.sh @@ -3,6 +3,8 @@ # This script compiles kernel and collects a list of complite-time files and related Git metadata # for each newly added line from tag 1 to tag 2. +set -e + show_help() { echo "Usage: ./generate_build_filelists.sh [tag1] [tag2]" echo "Example: ./generate_rugenerate_build_filelists.h tag1 tag2 clone_path" @@ -72,7 +74,7 @@ fi TAG1="$1" TAG2="$2" CLONE_PATH="$3" - +SUBSYS="$4" mkdir -p "build_data" curr_dir=$(pwd) @@ -81,7 +83,7 @@ export curr_dir clone_and_goto_cloned_repo "$CLONE_PATH" -bash "$curr_dir"/build_scripts/build_collect_diff.sh "$TAG1" "$TAG2" +bash "$curr_dir"/build_scripts/build_collect_diff.sh "$TAG1" "$TAG2" "$SUBSYS" echo "Finishing generating build source file lists" # Fetch email and name pairing information for linux contributor diff --git a/generate_web_reports.sh b/generate_web_reports.sh index 91b1005..949f462 100755 --- a/generate_web_reports.sh +++ b/generate_web_reports.sh @@ -2,6 +2,8 @@ # # Generate the web report source codes (*.html and *.js) for the web report +set -e + function show_help { echo "Usage: ./generate_build_web_reports.sh [tag1] [tag2] [linux_repo_root_url]" echo "Example: ./generate_build_web_reports.sh tag1 tag2 linux_url" diff --git a/run_tool.sh b/run_tool.sh index 002081b..01839f4 100755 --- a/run_tool.sh +++ b/run_tool.sh @@ -2,35 +2,47 @@ # # A script to run the tool with all stages -# Ensure required arguments (tag1 and tag2) are provided +set -e + + +# Ensure at least two positional parameters are provided if [ -z "$1" ] || [ -z "$2" ]; then - echo "Usage: $0 [clone_path] [linux_repo_root_url]" + echo "Usage: $0 [-c clone_path] [-u repo_link] [-s subsystem]" + echo "error" exit 1 fi TAG1="$1" TAG2="$2" -# Optional arguments -# Default values -DEFAULT_CLONE_PATH="linux-clone" -DEFAULT_LINUX_REPO_URL="https://github.com/gregkh/linux" - -# Check the number of arguments and set optional parameters accordingly -if [ -n "$3" ] && [ -n "$4" ]; then - CLONE_PATH="$3" - REPO_URL="$4" -elif [ -n "$3" ] && [[ "$3" == *"github.com"* || "$3" == *"gitlab.com"* ]]; then - CLONE_PATH="$DEFAULT_CLONE_PATH" - REPO_URL="$3" -elif [ -n "$3" ]; then - CLONE_PATH="$3" - REPO_URL="$DEFAULT_LINUX_REPO_URL" -else - CLONE_PATH="$DEFAULT_CLONE_PATH" - REPO_URL="$DEFAULT_LINUX_REPO_URL" -fi - +CLONE_PATH="linux-clone" +REPO_URL="https://github.com/gregkh/linux" +SUBSYS="" + +print_usage() { + echo "Usage: $0 [-c clone_path] [-u repo_link] [-s subsystem]" +} + +# Shift positional parameters before processing options +shift 2 + +# Parse optional arguments +while getopts 'c:u:s:' flag; do + case "${flag}" in + c) CLONE_PATH="${OPTARG}" ;; + u) REPO_URL="${OPTARG}" ;; + s) SUBSYS="${OPTARG}" ;; + *) print_usage + exit 1 ;; + esac +done + +# Debug information +echo "TAG1: $TAG1" +echo "TAG2: $TAG2" +echo "CLONE_PATH: $CLONE_PATH" +echo "REPO_URL: $REPO_URL" +echo "SUBSYS: $SUBSYS" # Determine if REPO_URL is a GitHub or GitLab URL if [[ "$REPO_URL" == "https://github"* ]]; then REPO_TYPE="github" @@ -47,33 +59,34 @@ echo "Repository URL ($REPO_URL) is identified as: $REPO_TYPE" TARGET_DIR="linux/scripts/change-impact-tools" # Clone the repository -git clone "$REPO_URL" +if [ ! -d linux/.git ]; then + git clone "$REPO_URL" +fi # Create the target directory if it doesn't exist mkdir -p $TARGET_DIR # Copy necessary scripts to the target directory -cp -r build_scripts $TARGET_DIR/ -cp -r web_scripts $TARGET_DIR/ -cp fixdep-patch.file $TARGET_DIR/ -cp generate_build_filelists.sh $TARGET_DIR/ -cp generate_web_reports.sh $TARGET_DIR/ +cp -rf build_scripts $TARGET_DIR/ +cp -rf web_scripts $TARGET_DIR/ +cp -f fixdep-patch.file $TARGET_DIR/ +cp -f generate_build_filelists.sh $TARGET_DIR/ +cp -f generate_web_reports.sh $TARGET_DIR/ mkdir -p $TARGET_DIR/web_source_code -mv web_source_template/* $TARGET_DIR/web_source_code/ - +cp web_source_template/* $TARGET_DIR/web_source_code/ # Save the current directory CUR_DIR=$(pwd) # Navigate to the target directory and execute the scripts cd "$TARGET_DIR" || exit -./generate_build_filelists.sh "$TAG1" "$TAG2" "$CLONE_PATH" -./generate_web_reports.sh "$TAG1" "$TAG2" "$REPO_URL" # fix later +./generate_build_filelists.sh "$TAG1" "$TAG2" "$CLONE_PATH" "$SUBSYS" +./generate_web_reports.sh "$TAG1" "$TAG2" "$REPO_URL" # Return to the original directory cd "$CUR_DIR" || exit # Copy the web source code to the current directory -cp -r $TARGET_DIR/web_source_code . -cp -r $TARGET_DIR/build_data . +cp -rf $TARGET_DIR/web_source_code . +cp -rf $TARGET_DIR/build_data . diff --git a/web_scripts/generate_data.sh b/web_scripts/generate_data.sh index 27cdbde..0667596 100755 --- a/web_scripts/generate_data.sh +++ b/web_scripts/generate_data.sh @@ -26,18 +26,10 @@ echo "Repository URL ($REPO_URL) is identified as: $REPO_TYPE" # Generate front-end commit info metadata echo "Generating source_data.js ..." -if [ ! -f "build_data/tokenize_source.json" ]; then - echo "Error: build_data/tokenize_source.json not found!" - exit 1 -fi TOKEN_DATA_SOURCE=$(cat build_data/tokenize_source.json) echo "let token_data_source = $TOKEN_DATA_SOURCE;" > web_source_code/source_data.js echo "Generating header_data.js ..." -if [ ! -f "build_data/tokenize_header.json" ]; then - echo "Error: build_data/tokenize_header.json not found!" - exit 1 -fi TOKEN_DATA_HEADER=$(cat build_data/tokenize_header.json) echo "let token_data_header = $TOKEN_DATA_HEADER;" > web_source_code/header_data.js @@ -60,12 +52,10 @@ echo "let diffs_header = \` $HEADER_TRIM_ESCAPED \`.trim();" > web_source_code/git_diff_header.js - -cat < web_source_code/versions.js -let tag1 = "$TAG1"; -let tag2 = "$TAG2"; -let root_linux_url = "$REPO_URL"; -let repo_type = "$REPO_TYPE"; -EOF +echo "tag1=\"$TAG1\"" > web_source_code/versions.js +# shellcheck disable=SC2129 +echo "tag2=\"$TAG2\"" >> web_source_code/versions.js +echo "root_linux_url=\"$REPO_URL\"" >> web_source_code/versions.js +echo "repo_type=\"$REPO_TYPE\"" >> web_source_code/versions.js echo "SUCCESS generated metadata js files" diff --git a/web_source_template/index.html b/web_source_template/index.html index ab6e85f..c8d70d8 100644 --- a/web_source_template/index.html +++ b/web_source_template/index.html @@ -216,12 +216,16 @@

Linux Kernel Git Diff Report

fileCell.colSpan = 2; // Extract GitHub URL for the file - const fileUrl = `https://github.com/gregkh/linux/blob/master/${diff.fileName}`; - + let fileUrl; + if (repo_type == "gitlab") { + fileUrl = `${root_linux_url}/-/blob/${tag2}/${diff.fileName}`; + } else { + fileUrl = `${root_linux_url}/blob/${tag2}/${diff.fileName}`; + } // Update the file row with URL fileCell.innerHTML = ` ${diff.fileName} - View on GitHub + Link From 586c2c38d8a5432b52dd9cca4fb122bc1dbf09b4 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Thu, 25 Jul 2024 19:07:38 +0000 Subject: [PATCH 05/22] describe delta-kernel innovation in README.md Signed-off-by: Yizhen Zhang --- README.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ff5ec02..b579753 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,7 @@ # Change-impact-analysis Tool - -The Change Impact Analysis Tool generates a comprehensive visual report detailing changes in both header files and source code between two Linux versions (tags in the Linux kernel repository: old_tag and new_tag). This tool helps developers view updates from the old version. - -The diff report includes a subset of files from the Linux repository that are included in building the kernel, contributing to a focused and detailed report on the compile-time source code in Linux. - ## Table of Content - +- [Introduction](#introduction) +- [Innovation](#innovation) - [How to Use](#how-to-use) - [Files Generated](#files-generated) - [Structure of the Tool](#structure-of-the-tool) @@ -14,6 +10,20 @@ The diff report includes a subset of files from the Linux repository that are in - [III. Commit Metadata Retrieval](#iii-commit-metadata-retrieval) - [IV. Web Script Generation](#iv-web-script-generation) +## Introduction + +The Change Impact Analysis Tool generates a comprehensive visual report detailing changes in both header files and source code between two Linux versions (tags in the Linux kernel repository: old_tag and new_tag). This tool helps developers view updates from the old version. + +The diff report includes a subset of files from the Linux repository that are included in building the kernel, contributing to a focused and detailed report on the compile-time source code in Linux. + +## Innovation +The idea of generating a web display for Linux kernel version change impact analysis is inspired by [Cregit](https://github.com/cregit/cregit). This tool innovates on Cregit by: + +- Considering the extensive code space the Linux kernel deals with, it provides a compile-time analysis instead of a static analysis of the commit history of the Linux kernel, presenting changes only in files used during compilation. +- Generating not only web source files but also lists of all source files and dependencies/header files used in kernel compilation, facilitating additional analysis purposes. (More details in [Files Generated](#files-generated)) +- Enabling comparison between two specific tags/releases in the Linux kernel, highlighting all newly added and deleted lines. This provides a clear layout of differences between the tags. While Cregit organizes information by files and embeds the latest commit details in each line/token, it does not support direct comparison between two tags. +- User customization: allows users to define the URL of the Linux root repository and specify the subsystem for analysis. (More details in [How to Use](#how-to-use)) + ## How to use To utilize this tool in your Linux environment (compatible with Ubuntu and Debian), follow these steps: From a25241387f18bc8060708cd8e8fcdc376537bdf3 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Fri, 26 Jul 2024 15:50:52 +0000 Subject: [PATCH 06/22] fix comments Signed-off-by: Yizhen Zhang --- README.md | 16 +++++++++++++--- build_scripts/build_collect_diff.sh | 8 +------- build_scripts/tokenize.py | 11 +++++++---- generate_build_filelists.sh | 4 ++-- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b579753..ae3b305 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ -# Change-impact-analysis Tool +# DeltaKernel Change Impact Analysis Tool + ## Table of Content + - [Introduction](#introduction) - [Innovation](#innovation) - [How to Use](#how-to-use) @@ -12,11 +14,12 @@ ## Introduction -The Change Impact Analysis Tool generates a comprehensive visual report detailing changes in both header files and source code between two Linux versions (tags in the Linux kernel repository: old_tag and new_tag). This tool helps developers view updates from the old version. +DeltaKernel Change Impact Analysis Tool generates a comprehensive visual report detailing changes in both header files and source code between two Linux versions (tags in the Linux kernel repository: old_tag and new_tag). This tool helps developers view updates from the old version. The diff report includes a subset of files from the Linux repository that are included in building the kernel, contributing to a focused and detailed report on the compile-time source code in Linux. ## Innovation + The idea of generating a web display for Linux kernel version change impact analysis is inspired by [Cregit](https://github.com/cregit/cregit). This tool innovates on Cregit by: - Considering the extensive code space the Linux kernel deals with, it provides a compile-time analysis instead of a static analysis of the commit history of the Linux kernel, presenting changes only in files used during compilation. @@ -45,6 +48,13 @@ Execute the tool by specifying the old and new tags: ```bash ./run_tool.sh [-c clone_path] [-u repo_link] [-s subsystem] ``` + +Example Usage: +```bash +./run_tool.sh "v6.8" "v6.9" -c "linux-clone" -u "https://github.com/torvalds/linux" -s "security" +# the tool will generate web update report on linux kernel v6.9 from v6.8 for security subsystem. +# the linux repository will be cloned during tool execution and will be cloned into a folder named linux-clone. +``` - ``: Specifies the old version tag. - ``: Specifies the new version tag. - `-c `: Optional. Defines the user-specified path to clone the Linux source code repository. @@ -76,7 +86,7 @@ The tool operates through a structured process to generate a comprehensive chang During linux kernel compilation, `Makefile.build` calls `$K/scripts/basic/fixdep.c` to generate a .cmd file for each source that collects dependency information during compilation. -This tool incorporates a modification that applies a patch (`patch.file`) to `scripts/basic/fixdep.c`, enabling it to output dependency information into a **list of header files** when building the kernel. +The `scripts/basic/fixdep.c` file generates a `.cmd` file containing dependency information for each source file that the kernel compiles. This tool includes a modification that applies a patch (fixdep-patch.file) to `fixdep.c`, enabling it to collect dependency files for each source file and output a comprehensive list of all source files and their dependencies for the entire kernel compilation. The resulting `dependency_list.txt`` is generated after kernel compilation. #### Source code diff --git a/build_scripts/build_collect_diff.sh b/build_scripts/build_collect_diff.sh index 0e490fb..98c67e3 100755 --- a/build_scripts/build_collect_diff.sh +++ b/build_scripts/build_collect_diff.sh @@ -7,12 +7,6 @@ set -e # check and install gcc-11 if not already installed install_package_safe() { - if ! command -v gcc-11 &> /dev/null; then - sudo apt update - sudo apt install gcc-11 - else - echo "GCC-11 is already installed." - fi if ! command -v libssl-dev &> /dev/null; then sudo apt-get update sudo apt-get install -y libssl-dev @@ -145,7 +139,7 @@ echo "the current os-linux version: " cat /etc/os-release echo "start running make" -make HOSTCC=gcc-11 CC=gcc-11 +make echo "finished compile kernel using gcc 11" diff --git a/build_scripts/tokenize.py b/build_scripts/tokenize.py index d9b0966..b065e7a 100755 --- a/build_scripts/tokenize.py +++ b/build_scripts/tokenize.py @@ -15,8 +15,9 @@ import json import argparse from datetime import datetime +import tempfile -CHUNK_HEADER_PATTERN = r'^@@ -\d+,\d+ \+(\d+),\d+ @@' +CHUNK_HEADER_PATTERN = r'^@@ -\d+,\d+ \+(\d+),\d+ f@@' COMMIT_REGEX = r'^commit ([0-9a-f]{40})$' AUTHOR_REGEX = r'^Author: (.+)$' DATE_REGEX = r'^Date:\s+(.+)$' @@ -301,8 +302,6 @@ def highlight_substring(commit_list): parser = argparse.ArgumentParser(description='Process some files.') parser.add_argument('git_diff_report', type=str, help='Path to git_diff_report.txt') - parser.add_argument('intermediate_file', type=str, - help='Path to intermediate_file') parser.add_argument('output_file', type=str, help='Path to output_file') parser.add_argument('tag1', type=str, help='old version tag') parser.add_argument('tag2', type=str, help='new verison tag') @@ -310,11 +309,14 @@ def highlight_substring(commit_list): args = parser.parse_args() git_diff_report_path = args.git_diff_report - intermediate_file_path = args.intermediate_file output_file_path = args.output_file TAG1 = args.tag1 TAG2 = args.tag2 + # Create a temporary file and get its path + temp_file = tempfile.NamedTemporaryFile(delete=False) + intermediate_file_path = temp_file.name + # Identify all the (line number in git diff report) added_lines = parse(git_diff_report_path) @@ -329,3 +331,4 @@ def highlight_substring(commit_list): print("starting tokenize function test", flush=True) tokenize(intermediate_file_path, output_file_path) print("Result exported to tokenize.json", flush=True) + os.remove(intermediate_file_path) diff --git a/generate_build_filelists.sh b/generate_build_filelists.sh index dfdcfa7..58dc39e 100755 --- a/generate_build_filelists.sh +++ b/generate_build_filelists.sh @@ -107,8 +107,8 @@ display_file_head "$curr_dir/build_data" "filtered_diff_source_replace.txt" 3 # Retrieve and tokenize commit info per added line echo "tokenizing each commit line ..." git checkout "$TAG2" -python3 "$curr_dir"/build_scripts/tokenize.py "$curr_dir/build_data/filtered_diff_header.txt" "$curr_dir/build_data/parse_git_header.json" "$curr_dir/build_data/tokenize_header.json" "$TAG1" "$TAG2" -python3 "$curr_dir"/build_scripts/tokenize.py "$curr_dir/build_data/filtered_diff_source.txt" "$curr_dir/build_data/parse_git_source.json" "$curr_dir/build_data/tokenize_source.json" "$TAG1" "$TAG2" +python3 "$curr_dir"/build_scripts/tokenize.py "$curr_dir/build_data/filtered_diff_header.txt" "$curr_dir/build_data/tokenize_header.json" "$TAG1" "$TAG2" +python3 "$curr_dir"/build_scripts/tokenize.py "$curr_dir/build_data/filtered_diff_source.txt" "$curr_dir/build_data/tokenize_source.json" "$TAG1" "$TAG2" display_file_head "$curr_dir/build_data" "tokenize_source.json" 3 display_file_head "$curr_dir/build_data" "tokenize_header.json" 3 echo "finished tokenization" From 31d39b5f010b42ecf3b05f9c06966935bf59dd53 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang <98562104+LuminaScript@users.noreply.github.com> Date: Mon, 29 Jul 2024 11:28:00 -0400 Subject: [PATCH 07/22] Update README.md Co-authored-by: Chuck Wolber Signed-off-by: Yizhen Zhang --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae3b305..309f783 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ ## Introduction -DeltaKernel Change Impact Analysis Tool generates a comprehensive visual report detailing changes in both header files and source code between two Linux versions (tags in the Linux kernel repository: old_tag and new_tag). This tool helps developers view updates from the old version. +DeltaKernel Change Impact Analysis Tool generates a visual report detailing changes in both header files and source code between two Linux kernel versions (tags in the Linux kernel repository: old_tag and new_tag). This tool helps developers compare updates between versions. The diff report includes a subset of files from the Linux repository that are included in building the kernel, contributing to a focused and detailed report on the compile-time source code in Linux. From 0f0d5b93c8be26f8ad6afaeb97027974b3491365 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Mon, 29 Jul 2024 16:04:28 +0000 Subject: [PATCH 08/22] remove file extension & modified readme Signed-off-by: Yizhen Zhang --- README.md | 51 ++++++++++++++----- ...ild_collect_diff.sh => build_collect_diff} | 10 ++-- .../{git_shortlog.sh => git_shortlog} | 0 .../{parse_dep_list.py => parse_dep_list} | 3 +- build_scripts/{parse_json.py => parse_json} | 5 +- build_scripts/{tokenize.py => tokenize} | 3 +- ...d_filelists.sh => generate_build_filelists | 10 ++-- ...ate_web_reports.sh => generate_web_reports | 6 +-- run_tool.sh => run_tool | 8 +-- .../{generate_data.sh => generate_data} | 0 10 files changed, 63 insertions(+), 33 deletions(-) rename build_scripts/{build_collect_diff.sh => build_collect_diff} (94%) rename build_scripts/{git_shortlog.sh => git_shortlog} (100%) rename build_scripts/{parse_dep_list.py => parse_dep_list} (97%) mode change 100644 => 100755 rename build_scripts/{parse_json.py => parse_json} (94%) rename build_scripts/{tokenize.py => tokenize} (99%) rename generate_build_filelists.sh => generate_build_filelists (85%) rename generate_web_reports.sh => generate_web_reports (75%) rename run_tool.sh => run_tool (90%) rename web_scripts/{generate_data.sh => generate_data} (100%) diff --git a/README.md b/README.md index 309f783..73bd9a0 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The diff report includes a subset of files from the Linux repository that are in The idea of generating a web display for Linux kernel version change impact analysis is inspired by [Cregit](https://github.com/cregit/cregit). This tool innovates on Cregit by: -- Considering the extensive code space the Linux kernel deals with, it provides a compile-time analysis instead of a static analysis of the commit history of the Linux kernel, presenting changes only in files used during compilation. +- Considering the extensive code space the Linux kernel deals with, it provides a compile-time analysis instead of a static analysis of the commit history of the Linux kernel, presenting changes only in files used during compilation. - Generating not only web source files but also lists of all source files and dependencies/header files used in kernel compilation, facilitating additional analysis purposes. (More details in [Files Generated](#files-generated)) - Enabling comparison between two specific tags/releases in the Linux kernel, highlighting all newly added and deleted lines. This provides a clear layout of differences between the tags. While Cregit organizes information by files and embeds the latest commit details in each line/token, it does not support direct comparison between two tags. - User customization: allows users to define the URL of the Linux root repository and specify the subsystem for analysis. (More details in [How to Use](#how-to-use)) @@ -31,35 +31,62 @@ The idea of generating a web display for Linux kernel version change impact anal To utilize this tool in your Linux environment (compatible with Ubuntu and Debian), follow these steps: -Clone the repository: +**Clone the repository**: ```bash git clone ``` -Navigate to the cloned repository: +**Navigate to the cloned repository**: ```bash cd ``` -Execute the tool by specifying the old and new tags: +**Execute the tool by specifying the old and new tags**: ```bash -./run_tool.sh [-c clone_path] [-u repo_link] [-s subsystem] +./run_tool [-c clone_path] [-u repo_link] [-s subsystem] ``` -Example Usage: +**Example Usage**: + ```bash -./run_tool.sh "v6.8" "v6.9" -c "linux-clone" -u "https://github.com/torvalds/linux" -s "security" +./run_tool "v6.8" "v6.9" -c "linux-clone" -u "https://github.com/torvalds/linux" -s "security" # the tool will generate web update report on linux kernel v6.9 from v6.8 for security subsystem. # the linux repository will be cloned during tool execution and will be cloned into a folder named linux-clone. ``` + +**Command Line Arguments**: - ``: Specifies the old version tag. - ``: Specifies the new version tag. -- `-c `: Optional. Defines the user-specified path to clone the Linux source code repository. -- `-u `: Optional. Provides the URL for the Linux source code repository. -- `-s `: Optional. Specifies the subsystem for analysis (e.g., -s security). +- `c `: Optional. Defines the user-specified repo name to clone the Linux source code repository. Default is linux-clone. +- `u `: Optional. Provides the URL for the Linux source code repository. +- `s `: Optional. Specifies the subsystem for analysis (e.g., -s security). + +**What the Tool Does**: +- Clones the Linux repository from `repo_link`. +- Copies `/delta-kernel/*` into `linux/scripts/change-impact-tools/`. +- Clones the Linux repository into another repository named `clone_name`, defaulting to `linux-clone`. +- Navigates to `linux-clone`: + - Checks out `tag1` and applies `fixdep-patch.file`. + - Configures and compiles the kernel. + - Collects compile-time source files list and dependency files list. + - Generates diff reports based on the file lists. + - Cleans up the working directory in `linux-clone`. + - Retrieves git metadata for each file inside the lists. + - Copies file lists, git diff reports, and git metadata (`build_data/`) to `/delta-kernel`. + +**Usage of `linux-clone`**: +`delta-kernel` is assumed to be a directory located in `$K/scripts/change-impact-tools`. To avoid removing the `change-impact-tools/` directory while checking out different tags, linux-clone is created to simulate changes without affecting `change-impact-tools/``. + +**Clean Up (Optional)**: + +```bash +rm -r linux +rm -r linux-clone # or where you place the cloned dir +rm -r build_data +``` ## Files Generated @@ -90,9 +117,9 @@ The `scripts/basic/fixdep.c` file generates a `.cmd` file containing dependency #### Source code -This tool leverages the `$K/scripts/clang-tools/gen_compile_commands.py` script to generate a `compile_commands.json` file. This file documents all source files involved in the compilation process. The `gen_compile_commands.py` script traverses each `.cmd` file to aggregate the list of source files. +This tool leverages the `$K/scripts/clang-tools/gen_compile_commands` script to generate a `compile_commands.json` file. This file documents all source files involved in the compilation process. The `gen_compile_commands` script traverses each `.cmd` file to aggregate the list of source files. -Then, the tool invokes `parse_json.py` to parse `compile_commands.json`, generating **a list of source files**. +Then, the tool invokes `parse_json` to parse `compile_commands.json`, generating **a list of source files**. ### II. Git Diff Report Generation diff --git a/build_scripts/build_collect_diff.sh b/build_scripts/build_collect_diff similarity index 94% rename from build_scripts/build_collect_diff.sh rename to build_scripts/build_collect_diff index 98c67e3..94a83d3 100755 --- a/build_scripts/build_collect_diff.sh +++ b/build_scripts/build_collect_diff @@ -37,13 +37,13 @@ apply_patch() { # parse the JSON file parse_source_json_file() { - local python_path="$root_path/scripts/change-impact-tools/build_scripts/parse_json.py" + local python_path="$root_path/scripts/change-impact-tools/build_scripts/parse_json" # shellcheck disable=SC2154 local cloned_repo_name="/$clone_dir/" local input_path="$root_path/scripts/change-impact-tools/build_data/compile_commands.json" local output_path="$root_path/scripts/change-impact-tools/build_data/sourcefile.txt" - python3 "$python_path" "$cloned_repo_name" "$input_path" "$output_path" + ./"$python_path" "$cloned_repo_name" "$input_path" "$output_path" display_file_head "$root_path/scripts/change-impact-tools/build_data" "sourcefile.txt" 3 } @@ -52,7 +52,7 @@ generate_compiled_file_lists() { # Generate compiled source files list local json_output_path="$root_path/scripts/change-impact-tools/build_data/compile_commands.json" echo "path check: $(pwd)" - python3 scripts/clang-tools/gen_compile_commands.py -o "$json_output_path" + ./scripts/clang-tools/gen_compile_commands -o "$json_output_path" parse_source_json_file echo "source compiled filelist generated to sourcefile.txt" @@ -62,9 +62,9 @@ generate_compiled_file_lists() { local output_list="$root_path/scripts/change-impact-tools/build_data/headerfile.txt" local output_json="$root_path/scripts/change-impact-tools/build_data/source_dep.json" local dep_path="dependency_file.txt" - local python_tool_path="$root_path/scripts/change-impact-tools/build_scripts/parse_dep_list.py" + local python_tool_path="$root_path/scripts/change-impact-tools/build_scripts/parse_dep_list" - python3 "$python_tool_path" "$dep_path" "$output_json" "$output_list" + ./"$python_tool_path" "$dep_path" "$output_json" "$output_list" echo "dependency compiled filelist generated to headerfile.txt$" } diff --git a/build_scripts/git_shortlog.sh b/build_scripts/git_shortlog similarity index 100% rename from build_scripts/git_shortlog.sh rename to build_scripts/git_shortlog diff --git a/build_scripts/parse_dep_list.py b/build_scripts/parse_dep_list old mode 100644 new mode 100755 similarity index 97% rename from build_scripts/parse_dep_list.py rename to build_scripts/parse_dep_list index 2c5a08a..9d30a57 --- a/build_scripts/parse_dep_list.py +++ b/build_scripts/parse_dep_list @@ -1,3 +1,4 @@ +#!/usr/bin/python3 """ The script parses the dependency list generated by patching `fixdep.c`. @@ -7,7 +8,7 @@ 3. The output path for the list of header files. Usage: - parse_json.py + parse_json """ import re diff --git a/build_scripts/parse_json.py b/build_scripts/parse_json similarity index 94% rename from build_scripts/parse_json.py rename to build_scripts/parse_json index f597c00..e014ccf 100755 --- a/build_scripts/parse_json.py +++ b/build_scripts/parse_json @@ -1,7 +1,8 @@ +#!/usr/bin/python3 """ The script collects a list of compile-time source files. -Running `$K/scripts/clang-tools/gen_compile_commands.py` post kernel build +Running `$K/scripts/clang-tools/gen_compile_commands` post kernel build generates `compile_commands.json`. This file is created by parsing `.cmd` files to collect all metadata for the @@ -16,7 +17,7 @@ 3. The output path for the list of source files. Usage: - parse_json.py + parse_json """ diff --git a/build_scripts/tokenize.py b/build_scripts/tokenize similarity index 99% rename from build_scripts/tokenize.py rename to build_scripts/tokenize index b065e7a..1e77ff4 100755 --- a/build_scripts/tokenize.py +++ b/build_scripts/tokenize @@ -1,3 +1,4 @@ +#!/usr/bin/python3 """ This script parses a git diff report, fetches commit details, and tokenizes the results. @@ -7,7 +8,7 @@ 3. The path to the output file for storing the tokenized results. Usage: - tokenize.py + tokenize """ import subprocess diff --git a/generate_build_filelists.sh b/generate_build_filelists similarity index 85% rename from generate_build_filelists.sh rename to generate_build_filelists index 58dc39e..2e37af6 100755 --- a/generate_build_filelists.sh +++ b/generate_build_filelists @@ -6,7 +6,7 @@ set -e show_help() { - echo "Usage: ./generate_build_filelists.sh [tag1] [tag2]" + echo "Usage: ./generate_build_filelists [tag1] [tag2]" echo "Example: ./generate_rugenerate_build_filelists.h tag1 tag2 clone_path" echo echo "Options:" @@ -83,11 +83,11 @@ export curr_dir clone_and_goto_cloned_repo "$CLONE_PATH" -bash "$curr_dir"/build_scripts/build_collect_diff.sh "$TAG1" "$TAG2" "$SUBSYS" +bash "$curr_dir"/build_scripts/build_collect_diff "$TAG1" "$TAG2" "$SUBSYS" echo "Finishing generating build source file lists" # Fetch email and name pairing information for linux contributor -bash "$curr_dir"/build_scripts/git_shortlog.sh "$TAG2" +bash "$curr_dir"/build_scripts/git_shortlog "$TAG2" display_file_head "$curr_dir/build_data" "name_list.txt" 3 echo "finished generating email and name list" @@ -107,8 +107,8 @@ display_file_head "$curr_dir/build_data" "filtered_diff_source_replace.txt" 3 # Retrieve and tokenize commit info per added line echo "tokenizing each commit line ..." git checkout "$TAG2" -python3 "$curr_dir"/build_scripts/tokenize.py "$curr_dir/build_data/filtered_diff_header.txt" "$curr_dir/build_data/tokenize_header.json" "$TAG1" "$TAG2" -python3 "$curr_dir"/build_scripts/tokenize.py "$curr_dir/build_data/filtered_diff_source.txt" "$curr_dir/build_data/tokenize_source.json" "$TAG1" "$TAG2" +./"$curr_dir"/build_scripts/tokenize "$curr_dir/build_data/filtered_diff_header.txt" "$curr_dir/build_data/tokenize_header.json" "$TAG1" "$TAG2" +./"$curr_dir"/build_scripts/tokenize "$curr_dir/build_data/filtered_diff_source.txt" "$curr_dir/build_data/tokenize_source.json" "$TAG1" "$TAG2" display_file_head "$curr_dir/build_data" "tokenize_source.json" 3 display_file_head "$curr_dir/build_data" "tokenize_header.json" 3 echo "finished tokenization" diff --git a/generate_web_reports.sh b/generate_web_reports similarity index 75% rename from generate_web_reports.sh rename to generate_web_reports index 949f462..c1f077c 100755 --- a/generate_web_reports.sh +++ b/generate_web_reports @@ -5,8 +5,8 @@ set -e function show_help { - echo "Usage: ./generate_build_web_reports.sh [tag1] [tag2] [linux_repo_root_url]" - echo "Example: ./generate_build_web_reports.sh tag1 tag2 linux_url" + echo "Usage: ./generate_build_web_reports [tag1] [tag2] [linux_repo_root_url]" + echo "Example: ./generate_build_web_reports tag1 tag2 linux_url" echo echo "Options:" echo " tag1 Required: old_linux_version [e.g. v6.8]" @@ -29,6 +29,6 @@ TAG1="$1" TAG2="$2" REPO_URL="$3" -./web_scripts/generate_data.sh "$TAG1" "$TAG2" "$REPO_URL" +./web_scripts/generate_data "$TAG1" "$TAG2" "$REPO_URL" echo "finishing generating webpage, report can be viewed in web_source_code/index.html" diff --git a/run_tool.sh b/run_tool similarity index 90% rename from run_tool.sh rename to run_tool index 01839f4..8f53d49 100755 --- a/run_tool.sh +++ b/run_tool @@ -71,8 +71,8 @@ mkdir -p $TARGET_DIR cp -rf build_scripts $TARGET_DIR/ cp -rf web_scripts $TARGET_DIR/ cp -f fixdep-patch.file $TARGET_DIR/ -cp -f generate_build_filelists.sh $TARGET_DIR/ -cp -f generate_web_reports.sh $TARGET_DIR/ +cp -f generate_build_filelists $TARGET_DIR/ +cp -f generate_web_reports $TARGET_DIR/ mkdir -p $TARGET_DIR/web_source_code cp web_source_template/* $TARGET_DIR/web_source_code/ @@ -81,8 +81,8 @@ CUR_DIR=$(pwd) # Navigate to the target directory and execute the scripts cd "$TARGET_DIR" || exit -./generate_build_filelists.sh "$TAG1" "$TAG2" "$CLONE_PATH" "$SUBSYS" -./generate_web_reports.sh "$TAG1" "$TAG2" "$REPO_URL" +./generate_build_filelists "$TAG1" "$TAG2" "$CLONE_PATH" "$SUBSYS" +./generate_web_reports "$TAG1" "$TAG2" "$REPO_URL" # Return to the original directory cd "$CUR_DIR" || exit diff --git a/web_scripts/generate_data.sh b/web_scripts/generate_data similarity index 100% rename from web_scripts/generate_data.sh rename to web_scripts/generate_data From 740022077d4fc6f4eabf723c157633b8d00aa4ac Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Mon, 29 Jul 2024 16:10:36 +0000 Subject: [PATCH 09/22] update README Signed-off-by: Yizhen Zhang --- README.md | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 73bd9a0..4de2600 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ - [Introduction](#introduction) - [Innovation](#innovation) - [How to Use](#how-to-use) -- [Files Generated](#files-generated) -- [Structure of the Tool](#structure-of-the-tool) +- [Intermediate Files Generated](#intermediate-files-generated) +- [Operation Stages of the Tool](#operation-stages-of-the-tool) - [I. Compilation File List Generation](#i-compilation-file-list-generation) - [II. Git Diff Report Generation](#ii-git-diff-report-generation) - [III. Commit Metadata Retrieval](#iii-commit-metadata-retrieval) @@ -23,7 +23,7 @@ The diff report includes a subset of files from the Linux repository that are in The idea of generating a web display for Linux kernel version change impact analysis is inspired by [Cregit](https://github.com/cregit/cregit). This tool innovates on Cregit by: - Considering the extensive code space the Linux kernel deals with, it provides a compile-time analysis instead of a static analysis of the commit history of the Linux kernel, presenting changes only in files used during compilation. -- Generating not only web source files but also lists of all source files and dependencies/header files used in kernel compilation, facilitating additional analysis purposes. (More details in [Files Generated](#files-generated)) +- Generating not only web source files but also lists of all source files and dependencies/header files used in kernel compilation, facilitating additional analysis purposes. (More details in [Intermediate Files Generated](#intermediate-files-generated)) - Enabling comparison between two specific tags/releases in the Linux kernel, highlighting all newly added and deleted lines. This provides a clear layout of differences between the tags. While Cregit organizes information by files and embeds the latest commit details in each line/token, it does not support direct comparison between two tags. - User customization: allows users to define the URL of the Linux root repository and specify the subsystem for analysis. (More details in [How to Use](#how-to-use)) @@ -53,18 +53,20 @@ cd ```bash ./run_tool "v6.8" "v6.9" -c "linux-clone" -u "https://github.com/torvalds/linux" -s "security" -# the tool will generate web update report on linux kernel v6.9 from v6.8 for security subsystem. -# the linux repository will be cloned during tool execution and will be cloned into a folder named linux-clone. +# the tool will generate web update report on linux kernel v6.9 from v6.8 for security subsystem. +cd web_source_code # click on index.html to view the result ``` **Command Line Arguments**: + - ``: Specifies the old version tag. - ``: Specifies the new version tag. -- `c `: Optional. Defines the user-specified repo name to clone the Linux source code repository. Default is linux-clone. +- `c `: Optional. Defines the user-specified repo name to clone the Linux source code repository. Default is linux-clone. `delta-kernel` should be located in `$K/scripts/change-impact-tools`. To preserve `change-impact-tools/` while checking out different tags, `linux-clone` simulates changes without affecting `change-impact-tools/`. - `u `: Optional. Provides the URL for the Linux source code repository. - `s `: Optional. Specifies the subsystem for analysis (e.g., -s security). **What the Tool Does**: + - Clones the Linux repository from `repo_link`. - Copies `/delta-kernel/*` into `linux/scripts/change-impact-tools/`. - Clones the Linux repository into another repository named `clone_name`, defaulting to `linux-clone`. @@ -76,19 +78,28 @@ cd - Cleans up the working directory in `linux-clone`. - Retrieves git metadata for each file inside the lists. - Copies file lists, git diff reports, and git metadata (`build_data/`) to `/delta-kernel`. +**linux-clone** +After execution, `linux-clone` will be in the branch of `tag2`. -**Usage of `linux-clone`**: -`delta-kernel` is assumed to be a directory located in `$K/scripts/change-impact-tools`. To avoid removing the `change-impact-tools/` directory while checking out different tags, linux-clone is created to simulate changes without affecting `change-impact-tools/``. +If a runtime git conflict is encountered, resolve it with the following steps: + +```bash +cd linux-clone # or user-defined clone name +git reset --hard +git checkout master +cd .. # return to delta-kernel +./run_tool # the cloned repository will not be re-cloned +``` **Clean Up (Optional)**: ```bash rm -r linux -rm -r linux-clone # or where you place the cloned dir +rm -r linux-clone # or how you name the cloned dir rm -r build_data ``` -## Files Generated +## Intermediate Files Generated **/build_data:** @@ -99,11 +110,7 @@ rm -r build_data - `tokenize_header.json` - Metadata for commit git diff for dependency files - `tokenize_source.json` - Metadata for commit git diff for source files -**/web_source_codes:** - -- `index.html` - Click on to view the result - -## Structure of the Tool +## Operation Stages of the Tool The tool operates through a structured process to generate a comprehensive change impact analysis report. Here's a detailed breakdown of its operation: From 8d06c26439981e7ee1fd5392923e11990193a1ef Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Mon, 29 Jul 2024 16:13:31 +0000 Subject: [PATCH 10/22] update docs Signed-off-by: Yizhen Zhang --- build_scripts/build_collect_diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_scripts/build_collect_diff b/build_scripts/build_collect_diff index 94a83d3..4e44355 100755 --- a/build_scripts/build_collect_diff +++ b/build_scripts/build_collect_diff @@ -69,7 +69,7 @@ generate_compiled_file_lists() { } -# clean up the working directory +# clean up the working directory in linux-clone cleanup_working_directory() { git reset --hard git clean -fdx From 840beab2ca5454037b0e73b2be461c9712aca845 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Mon, 29 Jul 2024 16:23:31 +0000 Subject: [PATCH 11/22] fix usage manual Signed-off-by: Yizhen Zhang --- generate_build_filelists | 5 ++--- generate_web_reports | 3 +-- run_tool | 12 +++++------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/generate_build_filelists b/generate_build_filelists index 2e37af6..0875c9a 100755 --- a/generate_build_filelists +++ b/generate_build_filelists @@ -6,7 +6,7 @@ set -e show_help() { - echo "Usage: ./generate_build_filelists [tag1] [tag2]" + echo "Usage: ./generate_build_filelists [tag1] [tag2] [clone-path]" echo "Example: ./generate_rugenerate_build_filelists.h tag1 tag2 clone_path" echo echo "Options:" @@ -67,8 +67,7 @@ fi # Ensure required arguments (tag1 and tag2) are provided if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage: $0 " - exit 1 + show_help fi TAG1="$1" diff --git a/generate_web_reports b/generate_web_reports index c1f077c..54acf3f 100755 --- a/generate_web_reports +++ b/generate_web_reports @@ -21,8 +21,7 @@ fi # Ensure required arguments (tag1 and tag2) are provided if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage: $0 " - exit 1 + show_help fi TAG1="$1" diff --git a/run_tool b/run_tool index 8f53d49..22ae4ac 100755 --- a/run_tool +++ b/run_tool @@ -4,12 +4,14 @@ set -e +print_usage() { + echo "Usage: $0 [-c clone_path] [-u repo_link] [-s subsystem]" + exit 1 +} # Ensure at least two positional parameters are provided if [ -z "$1" ] || [ -z "$2" ]; then - echo "Usage: $0 [-c clone_path] [-u repo_link] [-s subsystem]" - echo "error" - exit 1 + print_usage fi TAG1="$1" @@ -19,10 +21,6 @@ CLONE_PATH="linux-clone" REPO_URL="https://github.com/gregkh/linux" SUBSYS="" -print_usage() { - echo "Usage: $0 [-c clone_path] [-u repo_link] [-s subsystem]" -} - # Shift positional parameters before processing options shift 2 From 14cab6bd01d4bbcbf348fcfd7e0b58d29702317b Mon Sep 17 00:00:00 2001 From: Yizhen Zhang <98562104+LuminaScript@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:25:05 -0400 Subject: [PATCH 12/22] Update README.md Co-authored-by: Chuck Wolber Signed-off-by: Yizhen Zhang --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4de2600..6314411 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ DeltaKernel Change Impact Analysis Tool generates a visual report detailing changes in both header files and source code between two Linux kernel versions (tags in the Linux kernel repository: old_tag and new_tag). This tool helps developers compare updates between versions. -The diff report includes a subset of files from the Linux repository that are included in building the kernel, contributing to a focused and detailed report on the compile-time source code in Linux. +The diff report includes a subset of files from the Linux kernel repository that are included in building the kernel, contributing to a focused and detailed report on the compile-time source code in Linux. ## Innovation From 119a1a65d7640545e6b5449efe4a499310ad4f1f Mon Sep 17 00:00:00 2001 From: Yizhen Zhang <98562104+LuminaScript@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:25:34 -0400 Subject: [PATCH 13/22] Update README.md Co-authored-by: Chuck Wolber Signed-off-by: Yizhen Zhang --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6314411..409b3a9 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ The diff report includes a subset of files from the Linux kernel repository that ## Innovation -The idea of generating a web display for Linux kernel version change impact analysis is inspired by [Cregit](https://github.com/cregit/cregit). This tool innovates on Cregit by: +The idea of generating a web display for Linux kernel version change impact analysis is inspired by [Cregit](https://github.com/cregit/cregit). This tool improves on Cregit by: - Considering the extensive code space the Linux kernel deals with, it provides a compile-time analysis instead of a static analysis of the commit history of the Linux kernel, presenting changes only in files used during compilation. - Generating not only web source files but also lists of all source files and dependencies/header files used in kernel compilation, facilitating additional analysis purposes. (More details in [Intermediate Files Generated](#intermediate-files-generated)) From 6e11c0e40907cf5279e155319d282fff6fb2a1b7 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang <98562104+LuminaScript@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:28:03 -0400 Subject: [PATCH 14/22] Update README.md Co-authored-by: Chuck Wolber Signed-off-by: Yizhen Zhang --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 409b3a9..8b929fb 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ Based on the git diff reports, the tool retrieves commit metadata for each newly Using the git diff reports and metadata stored in JSON files, the tool generates a web report displaying the changes. -The web report contains three source html: +The web report contains three html files: - `index.html`: with on-click directions to: - `sourcecode.html`: renders the content in source diff report, with embedded url and on-hover metadata box for each newly added lines/tokens in new_tag. From 474155dfbbfd6b32c2fa7ca51739f35eec0fe816 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang <98562104+LuminaScript@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:28:42 -0400 Subject: [PATCH 15/22] Update README.md Co-authored-by: Chuck Wolber Signed-off-by: Yizhen Zhang --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b929fb..ffd64a1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The idea of generating a web display for Linux kernel version change impact anal - Considering the extensive code space the Linux kernel deals with, it provides a compile-time analysis instead of a static analysis of the commit history of the Linux kernel, presenting changes only in files used during compilation. - Generating not only web source files but also lists of all source files and dependencies/header files used in kernel compilation, facilitating additional analysis purposes. (More details in [Intermediate Files Generated](#intermediate-files-generated)) - Enabling comparison between two specific tags/releases in the Linux kernel, highlighting all newly added and deleted lines. This provides a clear layout of differences between the tags. While Cregit organizes information by files and embeds the latest commit details in each line/token, it does not support direct comparison between two tags. -- User customization: allows users to define the URL of the Linux root repository and specify the subsystem for analysis. (More details in [How to Use](#how-to-use)) +- User customization: allows users to define the URL of the Linux kernel repository and specify the specific subsystem for analysis. (More details in [How to Use](#how-to-use)) ## How to use From c0d40b515493ff3c6450e8949f3e38ffe2696368 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Tue, 30 Jul 2024 16:28:30 +0000 Subject: [PATCH 16/22] fix Signed-off-by: Yizhen Zhang --- README.md | 8 ++++--- build_scripts/build_collect_diff | 37 ++++++++++++++++---------------- build_scripts/git_shortlog | 5 +++++ generate_build_filelists | 22 +++++++++---------- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index ffd64a1..e34be8a 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,16 @@ The diff report includes a subset of files from the Linux kernel repository that The idea of generating a web display for Linux kernel version change impact analysis is inspired by [Cregit](https://github.com/cregit/cregit). This tool improves on Cregit by: -- Considering the extensive code space the Linux kernel deals with, it provides a compile-time analysis instead of a static analysis of the commit history of the Linux kernel, presenting changes only in files used during compilation. +- The tool performs a version analysis between two tags to identify updated files, similar to git diff, but focuses only on files used to compile the kernel rather than the entire Linux source code. + - Generating not only web source files but also lists of all source files and dependencies/header files used in kernel compilation, facilitating additional analysis purposes. (More details in [Intermediate Files Generated](#intermediate-files-generated)) - Enabling comparison between two specific tags/releases in the Linux kernel, highlighting all newly added and deleted lines. This provides a clear layout of differences between the tags. While Cregit organizes information by files and embeds the latest commit details in each line/token, it does not support direct comparison between two tags. - User customization: allows users to define the URL of the Linux kernel repository and specify the specific subsystem for analysis. (More details in [How to Use](#how-to-use)) +- Kernel Configuration: The linux kernel is configured with `make olddefconfig` in `build_scripts/build_collect_diff`: Updates the configuration using the existing `.config` file and applies default values for new options. ## How to use -To utilize this tool in your Linux environment (compatible with Ubuntu and Debian), follow these steps: +To utilize this tool in your Linux environment (tested successfully on `Ubuntu 22.04`), follow these steps: **Clone the repository**: @@ -52,7 +54,7 @@ cd **Example Usage**: ```bash -./run_tool "v6.8" "v6.9" -c "linux-clone" -u "https://github.com/torvalds/linux" -s "security" +./run_tool "v5.15" "v5.15.100" -c "linux-clone" -u "https://github.com/torvalds/linux" -s "security" # the tool will generate web update report on linux kernel v6.9 from v6.8 for security subsystem. cd web_source_code # click on index.html to view the result ``` diff --git a/build_scripts/build_collect_diff b/build_scripts/build_collect_diff index 4e44355..c55f5fd 100755 --- a/build_scripts/build_collect_diff +++ b/build_scripts/build_collect_diff @@ -18,16 +18,18 @@ install_package_safe() { # safely apply a patch to linux kernel apply_patch() { # shellcheck disable=SC2154 - local patch_path="$root_path/scripts/change-impact-tools/fixdep-patch.file" + local patch_path="$kernel_repo_path/scripts/change-impact-tools/fixdep-patch.file" # Stash any changes if there is any if ! git diff --quiet; then - git stash + echo "linux-clone has unstashed change. Please stash them and run the script again." + exit 1 fi # Abort `git am` only if there is a patch being applied if git am --show-current-patch &> /dev/null; then - git am --abort + echo "linux-clone has a patch being applied already. Should run git am --abort and try run the tool again." + exit 1 fi echo "path check: $(pwd)" git apply < "$patch_path" @@ -35,22 +37,21 @@ apply_patch() { echo "path check: $(pwd)" } -# parse the JSON file parse_source_json_file() { - local python_path="$root_path/scripts/change-impact-tools/build_scripts/parse_json" + local python_path="$kernel_repo_path/scripts/change-impact-tools/build_scripts/parse_json" # shellcheck disable=SC2154 local cloned_repo_name="/$clone_dir/" - local input_path="$root_path/scripts/change-impact-tools/build_data/compile_commands.json" - local output_path="$root_path/scripts/change-impact-tools/build_data/sourcefile.txt" + local input_path="$kernel_repo_path/scripts/change-impact-tools/build_data/compile_commands.json" + local output_path="$kernel_repo_path/scripts/change-impact-tools/build_data/sourcefile.txt" ./"$python_path" "$cloned_repo_name" "$input_path" "$output_path" - display_file_head "$root_path/scripts/change-impact-tools/build_data" "sourcefile.txt" 3 + display_file_head "$kernel_repo_path/scripts/change-impact-tools/build_data" "sourcefile.txt" 3 } # generate the build file list after building the kernel generate_compiled_file_lists() { # Generate compiled source files list - local json_output_path="$root_path/scripts/change-impact-tools/build_data/compile_commands.json" + local json_output_path="$kernel_repo_path/scripts/change-impact-tools/build_data/compile_commands.json" echo "path check: $(pwd)" ./scripts/clang-tools/gen_compile_commands -o "$json_output_path" @@ -59,10 +60,10 @@ generate_compiled_file_lists() { # Generate compiled header files list - local output_list="$root_path/scripts/change-impact-tools/build_data/headerfile.txt" - local output_json="$root_path/scripts/change-impact-tools/build_data/source_dep.json" + local output_list="$kernel_repo_path/scripts/change-impact-tools/build_data/headerfile.txt" + local output_json="$kernel_repo_path/scripts/change-impact-tools/build_data/source_dep.json" local dep_path="dependency_file.txt" - local python_tool_path="$root_path/scripts/change-impact-tools/build_scripts/parse_dep_list" + local python_tool_path="$kernel_repo_path/scripts/change-impact-tools/build_scripts/parse_dep_list" ./"$python_tool_path" "$dep_path" "$output_json" "$output_list" echo "dependency compiled filelist generated to headerfile.txt$" @@ -80,7 +81,7 @@ generate_git_diff() { # collect and setup input & output file file_type=${1:-source} - local root_build_data_path="$root_path/scripts/change-impact-tools/build_data" + local root_build_data_path="$kernel_repo_path/scripts/change-impact-tools/build_data" local diff_input="$root_build_data_path/sourcefile.txt" local diff_output="$root_build_data_path/filtered_diff_source.txt" @@ -125,22 +126,22 @@ echo "build and collect kernel for subsystem: $SUBSYS" git fetch --tags echo "Generating source file list for $TAG1" git checkout "$TAG1" +# Preparation before running make +apply_patch +install_package_safe echo "starting to run make olddefconfig" make olddefconfig echo "finished make olddefconfig" -# Preparation before running make -apply_patch -install_package_safe # Build linux kernel echo "the current os-linux version: " cat /etc/os-release echo "start running make" -make -echo "finished compile kernel using gcc 11" +make -j $(nproc) +echo "finished compile kernel" # Collect build metadata diff --git a/build_scripts/git_shortlog b/build_scripts/git_shortlog index 6175fec..114fa3b 100755 --- a/build_scripts/git_shortlog +++ b/build_scripts/git_shortlog @@ -3,7 +3,12 @@ # Fetch name email information for linux kernel contributors set -e +usage() { + echo "Usage: $0 tag" + exit 1 +} TAG="$1" +[[ -z ${TAG} ]] && usage git checkout "$TAG" echo "Starting to generate the email name list ..." diff --git a/generate_build_filelists b/generate_build_filelists index 0875c9a..4c4495d 100755 --- a/generate_build_filelists +++ b/generate_build_filelists @@ -21,21 +21,21 @@ clone_and_goto_cloned_repo() { clone_dir=${1:-"linux-clone"} # Capture the absolute path of the root directory - root_path=$(cd ../.. && pwd) - parent_root_path=$(cd ../../.. && pwd) - clone_root_path="$parent_root_path/$clone_dir" - echo "root_path: $root_path" - echo "clone root path: $clone_root_path" - export clone_root_path - export root_path + kernel_repo_path=$(cd ../.. && pwd) + parent_kernel_repo_path=$(cd ../../.. && pwd) + clone_kernel_repo_path="$parent_kernel_repo_path/$clone_dir" + echo "kernel_repo_path: $kernel_repo_path" + echo "clone root path: $clone_kernel_repo_path" + export clone_kernel_repo_path + export kernel_repo_path export clone_dir - mkdir -p "$clone_root_path" + mkdir -p "$clone_kernel_repo_path" - if [ ! -d "$clone_root_path/.git" ]; then - git clone "$root_path" "$clone_root_path" + if [ ! -d "$clone_kernel_repo_path/.git" ]; then + git clone "$kernel_repo_path" "$clone_kernel_repo_path" fi - cd "$clone_root_path" || exit + cd "$clone_kernel_repo_path" || exit } check_and_create_dir() { From b364ae37e57d0bd6a7ba528ca8635744796ac5b4 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang <98562104+LuminaScript@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:26:16 -0400 Subject: [PATCH 17/22] Update build_scripts/build_collect_diff Co-authored-by: Chuck Wolber Signed-off-by: Yizhen Zhang --- build_scripts/build_collect_diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_scripts/build_collect_diff b/build_scripts/build_collect_diff index c55f5fd..34cecbf 100755 --- a/build_scripts/build_collect_diff +++ b/build_scripts/build_collect_diff @@ -53,7 +53,7 @@ generate_compiled_file_lists() { # Generate compiled source files list local json_output_path="$kernel_repo_path/scripts/change-impact-tools/build_data/compile_commands.json" echo "path check: $(pwd)" - ./scripts/clang-tools/gen_compile_commands -o "$json_output_path" + ./scripts/clang-tools/gen_compile_commands.py -o "$json_output_path" parse_source_json_file echo "source compiled filelist generated to sourcefile.txt" From c9524a386b026296a6dd2459af6bbeb53e426cd6 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang <98562104+LuminaScript@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:27:16 -0400 Subject: [PATCH 18/22] Update build_scripts/build_collect_diff Co-authored-by: Chuck Wolber Signed-off-by: Yizhen Zhang --- build_scripts/build_collect_diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_scripts/build_collect_diff b/build_scripts/build_collect_diff index 34cecbf..607281d 100755 --- a/build_scripts/build_collect_diff +++ b/build_scripts/build_collect_diff @@ -44,7 +44,7 @@ parse_source_json_file() { local input_path="$kernel_repo_path/scripts/change-impact-tools/build_data/compile_commands.json" local output_path="$kernel_repo_path/scripts/change-impact-tools/build_data/sourcefile.txt" - ./"$python_path" "$cloned_repo_name" "$input_path" "$output_path" + "$python_path" "$cloned_repo_name" "$input_path" "$output_path" display_file_head "$kernel_repo_path/scripts/change-impact-tools/build_data" "sourcefile.txt" 3 } From a3a25d9a1c56a93ae8f031215666ebbe438608a8 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang <98562104+LuminaScript@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:27:29 -0400 Subject: [PATCH 19/22] Update build_scripts/build_collect_diff Co-authored-by: Chuck Wolber Signed-off-by: Yizhen Zhang --- build_scripts/build_collect_diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_scripts/build_collect_diff b/build_scripts/build_collect_diff index 607281d..0cd4ede 100755 --- a/build_scripts/build_collect_diff +++ b/build_scripts/build_collect_diff @@ -65,7 +65,7 @@ generate_compiled_file_lists() { local dep_path="dependency_file.txt" local python_tool_path="$kernel_repo_path/scripts/change-impact-tools/build_scripts/parse_dep_list" - ./"$python_tool_path" "$dep_path" "$output_json" "$output_list" + "$python_tool_path" "$dep_path" "$output_json" "$output_list" echo "dependency compiled filelist generated to headerfile.txt$" } From b186f878d99b23a2b25d04f3f18ccdf5982c1e04 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang <98562104+LuminaScript@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:27:44 -0400 Subject: [PATCH 20/22] Update generate_build_filelists Co-authored-by: Chuck Wolber Signed-off-by: Yizhen Zhang --- generate_build_filelists | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate_build_filelists b/generate_build_filelists index 4c4495d..89b04a2 100755 --- a/generate_build_filelists +++ b/generate_build_filelists @@ -106,8 +106,8 @@ display_file_head "$curr_dir/build_data" "filtered_diff_source_replace.txt" 3 # Retrieve and tokenize commit info per added line echo "tokenizing each commit line ..." git checkout "$TAG2" -./"$curr_dir"/build_scripts/tokenize "$curr_dir/build_data/filtered_diff_header.txt" "$curr_dir/build_data/tokenize_header.json" "$TAG1" "$TAG2" -./"$curr_dir"/build_scripts/tokenize "$curr_dir/build_data/filtered_diff_source.txt" "$curr_dir/build_data/tokenize_source.json" "$TAG1" "$TAG2" +"$curr_dir"/build_scripts/tokenize "$curr_dir/build_data/filtered_diff_header.txt" "$curr_dir/build_data/tokenize_header.json" "$TAG1" "$TAG2" +"$curr_dir"/build_scripts/tokenize "$curr_dir/build_data/filtered_diff_source.txt" "$curr_dir/build_data/tokenize_source.json" "$TAG1" "$TAG2" display_file_head "$curr_dir/build_data" "tokenize_source.json" 3 display_file_head "$curr_dir/build_data" "tokenize_header.json" 3 echo "finished tokenization" From 5ad20abff38720078c44807e0268820ec046dbd2 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Wed, 31 Jul 2024 17:34:53 +0000 Subject: [PATCH 21/22] remove install package Signed-off-by: Yizhen Zhang --- build_scripts/build_collect_diff | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/build_scripts/build_collect_diff b/build_scripts/build_collect_diff index 0cd4ede..483c54f 100755 --- a/build_scripts/build_collect_diff +++ b/build_scripts/build_collect_diff @@ -5,16 +5,6 @@ set -e -# check and install gcc-11 if not already installed -install_package_safe() { - if ! command -v libssl-dev &> /dev/null; then - sudo apt-get update - sudo apt-get install -y libssl-dev - else - echo "libssl-dev is already installed." - fi -} - # safely apply a patch to linux kernel apply_patch() { # shellcheck disable=SC2154 @@ -128,9 +118,8 @@ echo "Generating source file list for $TAG1" git checkout "$TAG1" # Preparation before running make apply_patch -install_package_safe echo "starting to run make olddefconfig" -make olddefconfig +make defconfig echo "finished make olddefconfig" From 83231842c6c4b24c327720f0f7c31e3e85163f67 Mon Sep 17 00:00:00 2001 From: Yizhen Zhang Date: Wed, 31 Jul 2024 17:37:05 +0000 Subject: [PATCH 22/22] versio Signed-off-by: Yizhen Zhang --- README.md | 2 +- generate_web_reports | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e34be8a..99e802b 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ cd ```bash ./run_tool "v5.15" "v5.15.100" -c "linux-clone" -u "https://github.com/torvalds/linux" -s "security" -# the tool will generate web update report on linux kernel v6.9 from v6.8 for security subsystem. +# the tool will generate web update report on linux kernel v5.15.100 from v5.15 for security subsystem. cd web_source_code # click on index.html to view the result ``` diff --git a/generate_web_reports b/generate_web_reports index 54acf3f..8b40309 100755 --- a/generate_web_reports +++ b/generate_web_reports @@ -9,8 +9,8 @@ function show_help { echo "Example: ./generate_build_web_reports tag1 tag2 linux_url" echo echo "Options:" - echo " tag1 Required: old_linux_version [e.g. v6.8]" - echo " tag2 Required: new_linux_version [e.g. v6.9]" + echo " tag1 Required: old_linux_version [e.g. v5.15]" + echo " tag2 Required: new_linux_version [e.g. v5.15.100]" echo " -h Display this help message" exit 0 }