Merge pull request #216 from tokk-nv/feature/html-postprocess

HTML post-process to close some of the nav sections

.github/workflows/ci.yml

@@ -22,6 +22,43 @@ jobs:
           path: .cache
           restore-keys: |
             mkdocs-material-
-      - run: pip install mkdocs-material
-      - run: pip install mkdocs-redirects
-      - run: mkdocs gh-deploy --force
+         
+      - name: Install dependencies
+        run: |   
+          pip install mkdocs-material
+          pip install mkdocs-redirects
+          pip install beautifulsoup4
+          pip install lxml
+
+      - name: Get mkdocs-material version
+        run: |
+          MKDOCS_MATERIAL_VERSION=$(pip show mkdocs-material | grep Version | cut -d ' ' -f 2)
+          echo "MKDOCS_MATERIAL_VERSION=$MKDOCS_MATERIAL_VERSION" >> $GITHUB_ENV
+
+      - name: Build the MkDocs site
+        run: |
+          mkdocs build
+            # Custom step to modify HTML files
+
+      - name: Duplicate the generated site dir with postprocessing
+        run: |
+          python ./scripts/duplicate_site_with_postprocess.py ./site ../site_postprocessed
+
+      - name: Deploy to GitHub Pages
+        run: |
+          # Initialize gh-pages branch if it doesn't exist
+          git config --global user.name "GitHub Actions"
+          git config --global user.email "[email protected]"
+          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git
+          git fetch origin gh-pages
+          git checkout gh-pages || git checkout --orphan gh-pages
+          ls -lh
+          git status
+          
+          # Copy the site content into the current directory
+          cp -r ../site_postprocessed/* .
+
+          # Commit and push the changes
+          git add .
+          git commit -m "Deployed ${GITHUB_SHA} with mkdocs-material v${{ env.MKDOCS_MATERIAL_VERSION }} and HTML post-processing"
+          git push origin gh-pages --force

.gitignore

@@ -1 +1,165 @@
 docs/overrides/plugins_backup/
+site*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

README.md

@@ -26,9 +26,19 @@ sudo docker pull squidfunk/mkdocs-material
 docker run --rm -it -p 8000:8000 -v ${PWD}:/docs squidfunk/mkdocs-material
 ```
 
+### Test the post-processing
+
+```bash
+docker run --rm -it -p 8000:8000 -v ${PWD}:/docs squidfunk/mkdocs-material build
+pip install beautifulsoup4
+pip install lxml
+python3 ./scripts/duplicate_site_with_postprocess.py ./site ./site_postprocessed
+sudo apt install python3-livereload
+livereload ./site_postprocessed
+```
+
+## Troubleshooting
+
 > If you get "docker: Got permission denied while trying to connect to the Docker daemon socket at ..." error, 
 > issue `sudo usermod -aG docker $USER; newgrp docker` to get around with the issue.
 
-## How to also publish on GitHub
-
-(to be documented)

mkdocs.yml

@@ -54,8 +54,8 @@ markdown_extensions:
   - md_in_html
   - tables
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
   - pymdownx.critic
   - pymdownx.caret
   - pymdownx.keys

scripts/duplicate_site_with_postprocess.py

@@ -0,0 +1,52 @@
+import os
+import shutil
+import argparse
+from bs4 import BeautifulSoup
+from postprocess_html_file import postprocess_html_file  # Import the function from the local file
+
+# Function to duplicate site directory with post-processing on HTML files
+def duplicate_and_process_html(src_dir, dest_dir):
+    if not os.path.exists(src_dir):
+        print(f"Source directory {src_dir} does not exist.")
+        return
+
+    # Remove destination directory if it already exists, then create a new one
+    if os.path.exists(dest_dir):
+        shutil.rmtree(dest_dir)
+    os.makedirs(dest_dir)
+
+    # Walk through the source directory
+    for root, dirs, files in os.walk(src_dir):
+        # Create corresponding directory in the destination
+        rel_path = os.path.relpath(root, src_dir)
+        dest_subdir = os.path.join(dest_dir, rel_path)
+        os.makedirs(dest_subdir, exist_ok=True)
+
+        for file_name in files:
+            src_file_path = os.path.join(root, file_name)
+            dest_file_path = os.path.join(dest_subdir, file_name)
+
+            # Process only HTML files, copy others directly
+            if file_name.endswith(".html"):
+                # ### HTML file ### 
+                #  - Make certain nav items collapse by removing `md-toggle--indeterminate` class.
+                postprocess_html_file(src_file_path, dest_file_path)
+            else:
+                # Copy non-HTML files as-is
+                shutil.copy2(src_file_path, dest_file_path)
+
+    print(f"Site duplicated with post-processing to: {dest_dir}")
+
+# Main function to handle argument parsing
+def main():
+    parser = argparse.ArgumentParser(description="Duplicate a site directory and post-process HTML files.")
+    parser.add_argument("src_dir", help="The source directory containing the site files.")
+    parser.add_argument("dest_dir", help="The destination directory to output the post-processed site.")
+
+    args = parser.parse_args()
+
+    # Call the duplicate and post-process function with the provided arguments
+    duplicate_and_process_html(args.src_dir, args.dest_dir)
+
+if __name__ == "__main__":
+    main()

scripts/postprocess_html_file.py

@@ -0,0 +1,42 @@
+import os
+import sys
+from bs4 import BeautifulSoup
+
+# Specify the section name exactly as spelled in mkdocs.yml file
+sections_to_keep_open = ["Text (LLM)", 
+                         "Text + Vision (VLM)", 
+                         "Vision Transformers (ViT)",
+                         "Robotics & Embodiment"
+                         ]
+
+# Function to process each HTML file
+def postprocess_html_file(src_file_path, dest_file_path):
+    with open(src_file_path, "r", encoding="utf-8") as file:
+        # Parse the HTML file using BeautifulSoup
+        soup = BeautifulSoup(file, "lxml")
+
+    # Find and print all <li> elements with specific classes
+    li_items = soup.find_all("li", class_="md-nav__item md-nav__item--nested")
+    input_tags_modified = 0
+    for li in li_items:
+        span = li.find("span", class_="md-ellipsis")
+        section_name = ""
+        if span:
+            section_name = span.get_text(strip=True)
+            print(f"Text found: {section_name}")
+
+        if section_name not in sections_to_keep_open:
+            input_tag = li.find("input", class_="md-nav__toggle")  # Make sure the class is correct
+            if not input_tag:
+                print(" NO <input> found under <li>")
+            # Remove the class "md-toggle--indeterminate"
+            if input_tag and "md-toggle--indeterminate" in input_tag['class']:
+                input_tag['class'].remove("md-toggle--indeterminate")
+                input_tags_modified = input_tags_modified + 1
+                print(" --> removed `md-toggle--indeterminate` class")
+
+    # Overwrite the modified content to the file
+    with open(dest_file_path, "w", encoding="utf-8") as file:
+        file.write(str(soup.prettify()))
+
+    print(f"Modified {input_tags_modified} <input> elements and saved to {dest_file_path}")