From 2f0aa541e7bf3adbe19f8e7f72e1978335c438b6 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 7 Aug 2024 00:02:58 +0300 Subject: [PATCH] Review and Format Output (#59) - Add support for GitHub automatic annotations. --- .github/dependabot.yml | 27 ++++++++ .github/workflows/python-tests.yaml | 63 +++++++++---------- .github/workflows/python-versions-tests.yaml | 16 ----- CHANGELOG.md | 5 +- README.md | 2 +- docs/source/advanced.md | 2 +- docs/source/index.md | 2 +- src/markdown_checker/__init__.py | 37 ++++++++--- .../reports/md_reports/generator.py | 2 +- src/markdown_checker/utils/format_output.py | 28 ++++++--- 10 files changed, 113 insertions(+), 71 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5ff9ba2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,27 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + github-actions: + patterns: + - "*" + + # Maintain dependencies for pip + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + groups: + python-requirements: + patterns: + - "*" diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml index e3dd072..34321b0 100644 --- a/.github/workflows/python-tests.yaml +++ b/.github/workflows/python-tests.yaml @@ -13,8 +13,8 @@ permissions: contents: read jobs: - tests-genai-repo: - name: Python Tests on generative-ai-for-beginners + tests-openai-repo: + name: Python Tests on azure-search-openai-demo runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -29,26 +29,27 @@ jobs: python3 -m pip install -e . - name: Clone Repo - run: git clone https://github.com/microsoft/generative-ai-for-beginners + run: git clone https://github.com/Azure-Samples/azure-search-openai-demo - - name: Test Check Broken Paths in generative-ai-for-beginners + - name: Test Check Broken Paths in azure-search-openai-demo if: always() run: | - cd generative-ai-for-beginners + cd azure-search-openai-demo markdown-checker -d . -f check_broken_paths -gu '' - - name: Test Check URLs Country locale in generative-ai-for-beginners + - name: Test Check URLs Country locale in azure-search-openai-demo if: always() run: | - cd generative-ai-for-beginners + cd azure-search-openai-demo markdown-checker -d . -f check_urls_locale -gu '' - - name: Test Check Broken URLs in generative-ai-for-beginners + + - name: Test Check Broken URLs in azure-search-openai-demo if: always() run: | - cd generative-ai-for-beginners + cd azure-search-openai-demo markdown-checker -d . -f check_broken_urls -gu '' - - tests-openai-repo: - name: Python Tests on azure-search-openai-demo + + tests-phicookbook-repo: + name: Python Tests on Phi-3CookBook runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -63,27 +64,26 @@ jobs: python3 -m pip install -e . - name: Clone Repo - run: git clone https://github.com/Azure-Samples/azure-search-openai-demo + run: git clone https://github.com/microsoft/Phi-3CookBook - - name: Test Check Broken Paths in azure-search-openai-demo + - name: Test Check Broken Paths in Phi-3CookBook if: always() run: | - cd azure-search-openai-demo + cd Phi-3CookBook markdown-checker -d . -f check_broken_paths -gu '' - - name: Test Check URLs Country locale in azure-search-openai-demo + - name: Test Check URLs Country locale in Phi-3CookBook if: always() run: | - cd azure-search-openai-demo + cd Phi-3CookBook markdown-checker -d . -f check_urls_locale -gu '' - - - name: Test Check Broken URLs in azure-search-openai-demo + - name: Test Check Broken URLs in Phi-3CookBook if: always() run: | - cd azure-search-openai-demo + cd Phi-3CookBook markdown-checker -d . -f check_broken_urls -gu '' - - tests-phicookbook-repo: - name: Python Tests on Phi-3CookBook + + tests-markdown-checker-repo: + name: Python Tests on markdown-checker runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -97,21 +97,20 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install -e . - - name: Clone Repo - run: git clone https://github.com/microsoft/Phi-3CookBook - - - name: Test Check Broken Paths in Phi-3CookBook + - name: Test Check Broken Paths in markdown-checker if: always() run: | - cd Phi-3CookBook markdown-checker -d . -f check_broken_paths -gu '' - - name: Test Check URLs Country locale in Phi-3CookBook + - name: Test Check URLs Country locale in markdown-checker if: always() run: | - cd Phi-3CookBook markdown-checker -d . -f check_urls_locale -gu '' - - name: Test Check Broken URLs in Phi-3CookBook + - name: Test Check Broken URLs in markdown-checker if: always() run: | - cd Phi-3CookBook markdown-checker -d . -f check_broken_urls -gu '' + + - name: Set job summary + if: ${{ always() && hashFiles('./comment.md') != '' }} + shell: bash + run: cat ./comment.md >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/python-versions-tests.yaml b/.github/workflows/python-versions-tests.yaml index 8f96e8f..da08206 100644 --- a/.github/workflows/python-versions-tests.yaml +++ b/.github/workflows/python-versions-tests.yaml @@ -31,7 +31,6 @@ jobs: - name: Clone Testing Repositories run: | git clone https://github.com/Azure-Samples/azure-search-openai-demo - git clone https://github.com/microsoft/generative-ai-for-beginners git clone https://github.com/microsoft/Phi-3CookBook - name: Test Check Broken Paths in azure-search-openai-demo @@ -39,11 +38,6 @@ jobs: run: | cd azure-search-openai-demo markdown-checker -d . -f check_broken_paths -gu '' - - name: Test Check Broken Paths in generative-ai-for-beginners - if: always() - run: | - cd generative-ai-for-beginners - markdown-checker -d . -f check_broken_paths -gu '' - name: Test Check Broken Paths in Phi-3CookBook if: always() run: | @@ -55,11 +49,6 @@ jobs: run: | cd azure-search-openai-demo markdown-checker -d . -f check_urls_locale -gu '' - - name: Test Check URLs Country locale in generative-ai-for-beginners - if: always() - run: | - cd generative-ai-for-beginners - markdown-checker -d . -f check_urls_locale -gu '' - name: Test Check URLs Country locale in Phi-3CookBook if: always() run: | @@ -71,11 +60,6 @@ jobs: run: | cd azure-search-openai-demo markdown-checker -d . -f check_broken_urls -gu '' - - name: Test Check Broken URLs in generative-ai-for-beginners - if: always() - run: | - cd generative-ai-for-beginners - markdown-checker -d . -f check_broken_urls -gu '' - name: Test Check Broken URLs in Phi-3CookBook if: always() run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 4639ce7..f9168db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ All notable changes to this project will be documented in this file. ### Other Changes -## [v0.2.0] 3 Aug 2024 +## [v0.2.0] 7 Aug 2024 - Redesign the package. - Port to using Click instead of arg_parser. - Expose options for external users to allow for more customization. @@ -22,6 +22,9 @@ All notable changes to this project will be documented in this file. - Analyze all web URLs except the ones in skip_domains list. - Change Syntax of terminal comments to improve readability. - Add Spinner to indicate that the tool is working (Not compatible with all terminals) +- Add documentation for the new features. +- Use multiprocessing for checking broken urls reducing the execution time by 50%. +- Add support for GitHub automatic annotations. ## [v0.1.5] 8 Jul 2024 - Increase timeout for requests to check web urls alive or not. https://github.com/john0isaac/markdown-checker/pull/52 diff --git a/README.md b/README.md index 7a274cd..73ab296 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ pip install markdown-checker 1. Run `pip install markdown-checker`. 2. Run `markdown-checker -d {src} -f {func} -gu {url}`. Replace `{src}` with the directory you want to analyze, `{func}` with the available functions like `check_broken_paths`, `{gu}` with your contribution guidance full URL. -3. The output will be displayed in the terminal and in a `comments.md` file. +3. The output will be displayed in the terminal and in a `comment.md` file. For more customizations read the docs. diff --git a/docs/source/advanced.md b/docs/source/advanced.md index 3f93bec..786abea 100644 --- a/docs/source/advanced.md +++ b/docs/source/advanced.md @@ -70,7 +70,7 @@ To further customize your experience with the Markdown Checker, you can utilize ### `-o`, `--output-file-name` - **Type**: `str` - **Description**: Name of the output file. -- **Default**: `comments` +- **Default**: `comment` - **Required**: Yes diff --git a/docs/source/index.md b/docs/source/index.md index eba97a8..7e0214f 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -19,4 +19,4 @@ pip install markdown-checker 1. Run `pip install markdown-checker`. 2. Run `markdown-checker -d {src} -f {func} -gu {url}`. Replace `{src}` with the directory you want to analyze, `{func}` with the available functions like `check_broken_paths`, `{gu}` with your contribution guidance full URL. -3. The output will be displayed in the terminal and in a `comments.md` file. +3. The output will be displayed in the terminal and in a `comment.md` file. diff --git a/src/markdown_checker/__init__.py b/src/markdown_checker/__init__.py index 7c1bb78..40e9e27 100644 --- a/src/markdown_checker/__init__.py +++ b/src/markdown_checker/__init__.py @@ -5,6 +5,7 @@ """ import concurrent.futures +import os import platform import sys from pathlib import Path @@ -242,7 +243,7 @@ def detect_issues( "-o", "--output-file-name", type=str, - default="comments", + default="comment", help="Name of the output file.", required=True, ) @@ -282,6 +283,7 @@ def main( formatted_output = "" all_files_issues: list[Union[MarkdownPath, MarkdownURL]] = [] links_checked_count = 0 + github_ci = os.getenv("CI", "false") # iterate over the files to validate the content for file_path in files_paths: @@ -296,7 +298,10 @@ def main( ) links_checked_count += links_count if len(detected_issues) > 0: - formatted_output += f"| [`{file_path}`]({file_path}) |" + format_links(detected_issues) + if github_ci == "true": + formatted_output += f"| `{file_path}` |" + format_links(detected_issues) + else: + formatted_output += f"| [`{file_path}`]({file_path}) |" + format_links(detected_issues) all_files_issues.extend(detected_issues) click.echo( click.style(f"\nšŸ” Checked {links_checked_count} links in {len(files_paths)} files.", fg="blue"), err=False @@ -307,14 +312,26 @@ def main( generator.generate(func, formatted_output) click.echo(click.style(f"šŸ˜­ Found {len(all_files_issues)} issues in the following files:", fg="red"), err=True) for markdown_path in all_files_issues: - click.echo( - click.style( - f"\tFile '{markdown_path.file_path.resolve()}', line {markdown_path.line_number}" - f"\n{markdown_path} {markdown_path.issue}.\n", - fg="red", - ), - err=True, - ) + if github_ci == "true": + # Ref: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-warning-message + click.echo( + click.style( + f"::error file={markdown_path.file_path},line={markdown_path.line_number}::" + f"File {markdown_path.file_path}, line {markdown_path.line_number}, " + f"Link {markdown_path} {markdown_path.issue}.", + fg="red", + ), + err=True, + ) + else: + click.echo( + click.style( + f"\tFile '{markdown_path.file_path.resolve()}', line {markdown_path.line_number}" + f"\n{markdown_path} {markdown_path.issue}.\n", + fg="red", + ), + err=True, + ) sys.exit(1) click.echo(click.style("All files are compliant with the guidelines. šŸŽ‰", fg="green"), err=False) sys.exit(0) diff --git a/src/markdown_checker/reports/md_reports/generator.py b/src/markdown_checker/reports/md_reports/generator.py index 3ac3250..e673705 100644 --- a/src/markdown_checker/reports/md_reports/generator.py +++ b/src/markdown_checker/reports/md_reports/generator.py @@ -20,7 +20,7 @@ class MarkdownGenerator(GeneratorBase): def __init__( self, - output_file_name: str = "comments", + output_file_name: str = "comment", contributing_guide_url: str = "https://github.com/john0isaac/markdown-checker/blob/main/CONTRIBUTING.md", ) -> None: self.output_file_name = output_file_name diff --git a/src/markdown_checker/utils/format_output.py b/src/markdown_checker/utils/format_output.py index cb28af2..90e1a9f 100644 --- a/src/markdown_checker/utils/format_output.py +++ b/src/markdown_checker/utils/format_output.py @@ -2,6 +2,7 @@ Module to format the output of the markdown checker. """ +import os from typing import Union from markdown_checker.paths import MarkdownPath @@ -18,13 +19,24 @@ def format_links(links: list[Union[MarkdownPath, MarkdownURL]]) -> str: Returns: formatted_links (str): The formatted string with numbered bullets. """ - formatted_links = "" - i = 1 - for link in links: - if i == len(links): - formatted_links += f" {i}. `{link}` |\n" - else: - formatted_links += f" {i}. `{link}`
" - i += 1 + formatted_links = "" + # Ref: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/variables#default-environment-variables + github_ci = os.getenv("CI", "false") + # repo_name = os.getenv("GITHUB_REPOSITORY", "../tree/") + # branch_name = os.getenv("GITHUB_HEAD_REF", os.getenv("GITHUB_REF_NAME", "main")) + # repo name is always target repo in PR not source + # need more work in upcoming release + if github_ci == "true": + for link in range(len(links)): + formatted_links += ( + f"" + ) + else: + for link in range(len(links)): + formatted_links += ( + f"" + f"" + ) + formatted_links += "
#LinkLine Number
{link + 1}`{links[link].link}``{links[link].line_number}`
{link + 1}`{links[link].link}`[`{links[link].line_number}`]({links[link].file_path}#L{links[link].line_number})
|\n" return formatted_links