diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a22dda6e..5524bd02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,43 @@ jobs: path: .cache restore-keys: | mkdocs-material- - - run: pip install mkdocs-material - - run: pip install mkdocs-redirects - - run: mkdocs gh-deploy --force + + - name: Install dependencies + run: | + pip install mkdocs-material + pip install mkdocs-redirects + pip install beautifulsoup4 + pip install lxml + + - name: Get mkdocs-material version + run: | + MKDOCS_MATERIAL_VERSION=$(pip show mkdocs-material | grep Version | cut -d ' ' -f 2) + echo "MKDOCS_MATERIAL_VERSION=$MKDOCS_MATERIAL_VERSION" >> $GITHUB_ENV + + - name: Build the MkDocs site + run: | + mkdocs build + # Custom step to modify HTML files + + - name: Duplicate the generated site dir with postprocessing + run: | + python ./scripts/duplicate_site_with_postprocess.py ./site ../site_postprocessed + + - name: Deploy to GitHub Pages + run: | + # Initialize gh-pages branch if it doesn't exist + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git + git fetch origin gh-pages + git checkout gh-pages || git checkout --orphan gh-pages + ls -lh + git status + + # Copy the site content into the current directory + cp -r ../site_postprocessed/* . + + # Commit and push the changes + git add . + git commit -m "Deployed ${GITHUB_SHA} with mkdocs-material v${{ env.MKDOCS_MATERIAL_VERSION }} and HTML post-processing" + git push origin gh-pages --force \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4cbcc053..54a0d0f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,165 @@ docs/overrides/plugins_backup/ +site* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/README.md b/README.md index 1aac95e3..358dcc69 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,19 @@ sudo docker pull squidfunk/mkdocs-material docker run --rm -it -p 8000:8000 -v ${PWD}:/docs squidfunk/mkdocs-material ``` +### Test the post-processing + +```bash +docker run --rm -it -p 8000:8000 -v ${PWD}:/docs squidfunk/mkdocs-material build +pip install beautifulsoup4 +pip install lxml +python3 ./scripts/duplicate_site_with_postprocess.py ./site ./site_postprocessed +sudo apt install python3-livereload +livereload ./site_postprocessed +``` + +## Troubleshooting + > If you get "docker: Got permission denied while trying to connect to the Docker daemon socket at ..." error, > issue `sudo usermod -aG docker $USER; newgrp docker` to get around with the issue. -## How to also publish on GitHub - -(to be documented) diff --git a/mkdocs.yml b/mkdocs.yml index 6261c14b..7436b0a9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -54,8 +54,8 @@ markdown_extensions: - md_in_html - tables - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.critic - pymdownx.caret - pymdownx.keys diff --git a/scripts/duplicate_site_with_postprocess.py b/scripts/duplicate_site_with_postprocess.py new file mode 100644 index 00000000..009cfd43 --- /dev/null +++ b/scripts/duplicate_site_with_postprocess.py @@ -0,0 +1,52 @@ +import os +import shutil +import argparse +from bs4 import BeautifulSoup +from postprocess_html_file import postprocess_html_file # Import the function from the local file + +# Function to duplicate site directory with post-processing on HTML files +def duplicate_and_process_html(src_dir, dest_dir): + if not os.path.exists(src_dir): + print(f"Source directory {src_dir} does not exist.") + return + + # Remove destination directory if it already exists, then create a new one + if os.path.exists(dest_dir): + shutil.rmtree(dest_dir) + os.makedirs(dest_dir) + + # Walk through the source directory + for root, dirs, files in os.walk(src_dir): + # Create corresponding directory in the destination + rel_path = os.path.relpath(root, src_dir) + dest_subdir = os.path.join(dest_dir, rel_path) + os.makedirs(dest_subdir, exist_ok=True) + + for file_name in files: + src_file_path = os.path.join(root, file_name) + dest_file_path = os.path.join(dest_subdir, file_name) + + # Process only HTML files, copy others directly + if file_name.endswith(".html"): + # ### HTML file ### + # - Make certain nav items collapse by removing `md-toggle--indeterminate` class. + postprocess_html_file(src_file_path, dest_file_path) + else: + # Copy non-HTML files as-is + shutil.copy2(src_file_path, dest_file_path) + + print(f"Site duplicated with post-processing to: {dest_dir}") + +# Main function to handle argument parsing +def main(): + parser = argparse.ArgumentParser(description="Duplicate a site directory and post-process HTML files.") + parser.add_argument("src_dir", help="The source directory containing the site files.") + parser.add_argument("dest_dir", help="The destination directory to output the post-processed site.") + + args = parser.parse_args() + + # Call the duplicate and post-process function with the provided arguments + duplicate_and_process_html(args.src_dir, args.dest_dir) + +if __name__ == "__main__": + main() diff --git a/scripts/postprocess_html_file.py b/scripts/postprocess_html_file.py new file mode 100644 index 00000000..26bb2416 --- /dev/null +++ b/scripts/postprocess_html_file.py @@ -0,0 +1,42 @@ +import os +import sys +from bs4 import BeautifulSoup + +# Specify the section name exactly as spelled in mkdocs.yml file +sections_to_keep_open = ["Text (LLM)", + "Text + Vision (VLM)", + "Vision Transformers (ViT)", + "Robotics & Embodiment" + ] + +# Function to process each HTML file +def postprocess_html_file(src_file_path, dest_file_path): + with open(src_file_path, "r", encoding="utf-8") as file: + # Parse the HTML file using BeautifulSoup + soup = BeautifulSoup(file, "lxml") + + # Find and print all
  • elements with specific classes + li_items = soup.find_all("li", class_="md-nav__item md-nav__item--nested") + input_tags_modified = 0 + for li in li_items: + span = li.find("span", class_="md-ellipsis") + section_name = "" + if span: + section_name = span.get_text(strip=True) + print(f"Text found: {section_name}") + + if section_name not in sections_to_keep_open: + input_tag = li.find("input", class_="md-nav__toggle") # Make sure the class is correct + if not input_tag: + print(" NO found under
  • ") + # Remove the class "md-toggle--indeterminate" + if input_tag and "md-toggle--indeterminate" in input_tag['class']: + input_tag['class'].remove("md-toggle--indeterminate") + input_tags_modified = input_tags_modified + 1 + print(" --> removed `md-toggle--indeterminate` class") + + # Overwrite the modified content to the file + with open(dest_file_path, "w", encoding="utf-8") as file: + file.write(str(soup.prettify())) + + print(f"Modified {input_tags_modified} elements and saved to {dest_file_path}")