Revamp (#56)

- Port to using Click instead of arg_parser. - Expose options for external users to allow for more customization. - Increase coverage for paths by including paths that start with `/` or nothing. - Add retires for URLs before flagging them as broken. - Preform head request on URL which falls back to get if both not working flag as broken after retries count finishes. - Analyze all web URLs except the ones in skip_domains list. - Change Syntax of terminal comments to improve readability. - Add Spinner to indicate that the tool is working (Not compatible with all terminals) Signed-off-by: John Aziz <[email protected]>
john0isaac · Aug 3, 2024 · b2bd651 · b2bd651
1 parent 52036a4
commit b2bd651
Show file tree

Hide file tree

Showing 29 changed files with 1,056 additions and 459 deletions.
diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml
@@ -0,0 +1,117 @@
+name: Run Tests on Repositories
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    branches: [ main ]
+    paths:
+      - '**.py'
+      - ".github/workflows/python-tests.yaml"
+
+permissions:
+  contents: read
+
+jobs:
+  tests-genai-repo:
+    name: Python Tests on generative-ai-for-beginners
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      id: python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install -e .
+
+    - name: Clone Repo
+      run: git clone https://github.com/microsoft/generative-ai-for-beginners
+
+    - name: Test Check Broken Paths in generative-ai-for-beginners
+      if: always()
+      run: |
+        cd generative-ai-for-beginners
+        markdown-checker -d . -f check_broken_paths -gu ''
+    - name: Test Check URLs Country locale in generative-ai-for-beginners
+      if: always()
+      run: |
+        cd generative-ai-for-beginners
+        markdown-checker -d . -f check_urls_locale -gu ''
+    - name: Test Check Broken URLs in generative-ai-for-beginners
+      if: always()
+      run: |
+        cd generative-ai-for-beginners
+        markdown-checker -d . -f check_broken_urls -gu ''
+
+  tests-openai-repo:
+    name: Python Tests on azure-search-openai-demo
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      id: python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install -e .
+
+    - name: Clone Repo
+      run: git clone https://github.com/Azure-Samples/azure-search-openai-demo
+
+    - name: Test Check Broken Paths in azure-search-openai-demo
+      if: always()
+      run: |
+        cd azure-search-openai-demo
+        markdown-checker -d . -f check_broken_paths -gu ''
+    - name: Test Check URLs Country locale in azure-search-openai-demo
+      if: always()
+      run: |
+        cd azure-search-openai-demo
+        markdown-checker -d . -f check_urls_locale -gu ''
+ 
+    - name: Test Check Broken URLs in azure-search-openai-demo
+      if: always()
+      run: |
+        cd azure-search-openai-demo
+        markdown-checker -d . -f check_broken_urls -gu ''
+ 
+  tests-phicookbook-repo:
+    name: Python Tests on Phi-3CookBook
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      id: python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install -e .
+
+    - name: Clone Repo
+      run: git clone https://github.com/microsoft/Phi-3CookBook
+
+    - name: Test Check Broken Paths in Phi-3CookBook
+      if: always()
+      run: |
+        cd Phi-3CookBook
+        markdown-checker -d . -f check_broken_paths -gu ''
+    - name: Test Check URLs Country locale in Phi-3CookBook
+      if: always()
+      run: |
+        cd Phi-3CookBook
+        markdown-checker -d . -f check_urls_locale -gu ''
+    - name: Test Check Broken URLs in Phi-3CookBook
+      if: always()
+      run: |
+        cd Phi-3CookBook
+        markdown-checker -d . -f check_broken_urls -gu ''
diff --git a/.github/workflows/python-versions-tests.yaml b/.github/workflows/python-versions-tests.yaml
@@ -0,0 +1,83 @@
+name: Run Python Matrix Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+
+permissions:
+  contents: read
+
+jobs:
+  python-matrix-tests:
+    if: ${{ always() }} # Always run even if a matrix candidate fails
+    name: Python ${{ matrix.python_version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python_version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python_version }}
+        architecture: x64
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install -e .
+
+    - name: Clone Testing Repositories
+      run: |
+        git clone https://github.com/Azure-Samples/azure-search-openai-demo
+        git clone https://github.com/microsoft/generative-ai-for-beginners
+        git clone https://github.com/microsoft/Phi-3CookBook
+
+    - name: Test Check Broken Paths in azure-search-openai-demo
+      if: always()
+      run: |
+        cd azure-search-openai-demo
+        markdown-checker -d . -f check_broken_paths -gu ''
+    - name: Test Check Broken Paths in generative-ai-for-beginners
+      if: always()
+      run: |
+        cd generative-ai-for-beginners
+        markdown-checker -d . -f check_broken_paths -gu ''
+    - name: Test Check Broken Paths in Phi-3CookBook
+      if: always()
+      run: |
+        cd Phi-3CookBook
+        markdown-checker -d . -f check_broken_paths -gu ''
+
+    - name: Test Check URLs Country locale in azure-search-openai-demo
+      if: always()
+      run: |
+        cd azure-search-openai-demo
+        markdown-checker -d . -f check_urls_locale -gu ''
+    - name: Test Check URLs Country locale in generative-ai-for-beginners
+      if: always()
+      run: |
+        cd generative-ai-for-beginners
+        markdown-checker -d . -f check_urls_locale -gu ''
+    - name: Test Check URLs Country locale in Phi-3CookBook
+      if: always()
+      run: |
+        cd Phi-3CookBook
+        markdown-checker -d . -f check_urls_locale -gu ''
+
+    - name: Test Check Broken URLs in azure-search-openai-demo
+      if: always()
+      run: |
+        cd azure-search-openai-demo
+        markdown-checker -d . -f check_broken_urls -gu ''
+    - name: Test Check Broken URLs in generative-ai-for-beginners
+      if: always()
+      run: |
+        cd generative-ai-for-beginners
+        markdown-checker -d . -f check_broken_urls -gu ''
+    - name: Test Check Broken URLs in Phi-3CookBook
+      if: always()
+      run: |
+        cd Phi-3CookBook
+        markdown-checker -d . -f check_broken_urls -gu ''
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Set up Python ${{ env.DEFAULT_PYTHON }}
         id: python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.DEFAULT_PYTHON }}
       - name: Install requirements

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,7 +1,3 @@
-[build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta"
-
 [project]
 name = "markdown-checker"
 description= "A markdown link validation reporting tool"
@@ -13,19 +9,23 @@ readme = "README.md"
 
 requires-python = ">=3.9"
 dependencies = [
+    "click",
     "requests",
-    "types-requests",
-    ]
+]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
-    "Programming Language :: Python :: 3",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
-    ]
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Software Development :: Quality Assurance",
+]
 
 [project.urls]
 Homepage = "https://github.com/john0isaac/markdown-checker"
@@ -40,7 +40,8 @@ Releases = "https://github.com/john0isaac/markdown-checker/releases"
 [project.optional-dependencies]
 dev = [
     "pre-commit",
-    "mypy"
+    "mypy",
+    "types-requests",
 ]
 lint = [
     "ruff"
@@ -61,7 +62,17 @@ lint.select = ["E", "F", "I", "UP"]
 line-length = 120
 target-version = "py39"
 src = ["src"]
+lint.isort.known-first-party = ["markdown_checker"]
 output-format = "full"
 
+[tool.mypy]
+check_untyped_defs = true
+python_version = 3.9
+exclude = [".venv/*"]
+
 [project.scripts]
-markdown-checker = "markdown_checker.validate_markdown:main"
+markdown-checker = "markdown_checker:main_with_spinner"
+
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"