From e34fbe3c5822191a6f2161457e0434687392ac74 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 27 Nov 2023 20:59:30 +0300 Subject: [PATCH] Update hugo/docsy version (#6791) Newer hugo/docsy versions will allow to use several new features, such as tab groups and tags. - Updated Docsy to v0.7.1 - Updated Hugo to v0.110.0 - Fixed broken layout in the footer - Removed unnecessary breadcrumbs on the main page - Fixed some links in the footer - Updated the build pipeline to build both old and new releases correctly - Added VS Code tasks for docs build steps --- .github/workflows/main.yml | 18 +- .vscode/launch.json | 13 +- site/README.md | 140 +- site/assets/scss/_custom.scss | 14 +- site/build_docs.py | 95 +- site/config.toml | 17 +- site/content/en/docs/_index.md | 10 +- site/layouts/home.html | 6 +- site/layouts/partials/footer.html | 39 +- site/layouts/partials/head.html | 69 +- site/layouts/partials/navbar.html | 95 +- site/package-lock.json | 2721 ++++++++++++++++++++++++++++- site/package.json | 4 +- site/themes/docsy | 2 +- 14 files changed, 3027 insertions(+), 216 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 11629305970c..0618135bbd43 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -398,15 +398,21 @@ jobs: path: /tmp/cvat_sdk/ - name: Setup Hugo - uses: peaceiris/actions-hugo@v2 - with: - hugo-version: '0.83.1' - extended: true + run: | + wget https://github.com/gohugoio/hugo/releases/download/v0.110.0/hugo_extended_0.110.0_Linux-64bit.tar.gz + (mkdir hugo_extended_0.110.0_Linux-64bit && tar -xf hugo_extended_0.110.0_Linux-64bit.tar.gz -C hugo_extended_0.110.0_Linux-64bit) + + wget https://github.com/gohugoio/hugo/releases/download/v0.83.0/hugo_extended_0.83.0_Linux-64bit.tar.gz + (mkdir hugo_extended_0.83.0_Linux-64bit && tar -xf hugo_extended_0.83.0_Linux-64bit.tar.gz -C hugo_extended_0.83.0_Linux-64bit) + + mkdir hugo + cp hugo_extended_0.110.0_Linux-64bit/hugo hugo/hugo-0.110 + cp hugo_extended_0.83.0_Linux-64bit/hugo hugo/hugo-0.83 - name: Setup Node uses: actions/setup-node@v3 with: - node-version: '16.x' + node-version: '18.x' - name: Install npm packages working-directory: ./site @@ -417,7 +423,7 @@ jobs: run: | pip install -r site/requirements.txt python site/process_sdk_docs.py --input-dir /tmp/cvat_sdk/docs/ --site-root site/ - python site/build_docs.py + PATH="$PWD/hugo:$PATH" python site/build_docs.py env: HUGO_ENV: production diff --git a/.vscode/launch.json b/.vscode/launch.json index 3fe869936663..fd0a35f45438 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -455,7 +455,7 @@ "console": "integratedTerminal" }, { - "name": "sdk docs: Postprocess generated docs", + "name": "docs: Postprocess SDK docs", "type": "python", "request": "launch", "justMyCode": false, @@ -469,6 +469,17 @@ "cwd": "${workspaceFolder}", "console": "integratedTerminal" }, + { + "name": "docs: Build docs", + "type": "python", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/site/build_docs.py", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, { "name": "server: Generate REST API Schema", "type": "python", diff --git a/site/README.md b/site/README.md index c57535405080..2de143a93b75 100644 --- a/site/README.md +++ b/site/README.md @@ -1,6 +1,6 @@ -## Basic manual for website editing +## Basic manual -### Edit or add documentation pages +### How to edit or add documentation pages online To edit and/or add documentation, you need to have a [GitHub](https://github.com/login) account. To change documentation files or add a documentation page, @@ -15,65 +15,129 @@ Read how to edit files for github ([GitHub docs](https://docs.github.com/en/gith Please note that files have a markup for correct display on the site: the title, the title of the link, the weight (affects the order of files display on the sidebar) and description (optional): - --- - title: "Title" - linkTitle: "Link Title" - weight: 1 - description: > - Description - --- +``` +--- +title: "Title" +linkTitle: "Link Title" +weight: 1 +description: > + Description +--- +``` -### Start site localy -To start the site locally, you need a recent [extended version hugo](https://github.com/gohugoio/hugo/releases) -(recommend version 0.83.1). -Open the most recent release and scroll down until you find a list of Extended versions. [Read more](https://gohugo.io/getting-started/installing/#quick-install) +### How to start the site locally -Add a path to "hugo" in the "Path" environment variable. +#### Installation -Clone a repository branch containing the site. For example, using a git command: +1. Install Hugo - git clone --branch +Get the [v110.0-extended release of hugo](https://github.com/gohugoio/hugo/releases/tag/v0.110.0). +Expand the Assets section of the release on GitHub and and scroll down +until you find a list of **Extended** versions. +[Read more](https://gohugo.io/getting-started/installing/#quick-install) -If you want to build and/or serve your site locally, you also need to get local copies of the theme’s own submodules: +Add a path to `hugo` in the `PATH` environment variable. - git submodule update --init --recursive +2. Get the Docsy theme code -To build and preview your site locally, use: +```bash +git submodule update --init --recursive +``` - cd /cvat/site/ - hugo server +3. Install the Docsy theme dependencies + +To build or update your site’s CSS resources, you will need several packages installed. +To install them, you must have a recent version of [NodeJS](https://nodejs.org/en/) +installed on your machine (tested with v18.0). For this, you can use +[npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) directly +or via an environment manager like [nvm](https://github.com/nvm-sh/nvm). +By default `npm` installs tools under the directory where you run `npm install`. + +```bash +(cd site/ && npm ci) +``` + +The full documentation is available +[here](https://www.docsy.dev/docs/get-started/other-options/#for-an-existing-site). + +4. To preview your site locally, use: + +```bash +(cd site/ && hugo server) +``` By default, your site will be available at . -Instead of a "hugo server" command, you can use the "hugo" command that generates the site into a "public" folder. +#### How to build for production deployment + +1. Install dependencies + +```bash +cd site/ +pip -m venv venv +. venv/bin/activate +pip install -r requirements.txt +``` -To build or update your site’s CSS resources you will need [PostCSS](https://postcss.org/) to create final assets. -To install it you must have a recent version of [NodeJS](https://nodejs.org/en/) installed on your machine, -so you can use npm, the Node package manager. -By default npm installs tools under the directory where you run [npm install](https://docs.npmjs.com/cli/v6/commands/npm-install#description): +The documentation site includes both old and new releases. Because of this, +you will need tooling for all these releases. Currently, it means you need +in your environment: +- `hugo-0.110` - for new docs +- `hugo-0.83` - for older docs - cd /cvat/site/ - npm ci +Please download these hugo releases (both extended), and make such binaries +available in your `PATH` environment variable. -Then you can build a website in the "public" folder: +On Linux, you can install it this way: - hugo +```bash +wget https://github.com/gohugoio/hugo/releases/download/v0.110.0/hugo_extended_0.110.0_Linux-64bit.tar.gz +(mkdir hugo_extended_0.110.0_Linux-64bit && tar -xf hugo_extended_0.110.0_Linux-64bit.tar.gz -C hugo_extended_0.110.0_Linux-64bit) +cd hugo_extended_0.110.0_Linux-64bit +sudo cp hugo /usr/local/bin/hugo-0.110 -[Read more](https://www.docsy.dev/docs/getting-started/) +wget https://github.com/gohugoio/hugo/releases/download/v0.83.0/hugo_extended_0.83.0_Linux-64bit.tar.gz +(mkdir hugo_extended_0.83.0_Linux-64bit && tar -xf hugo_extended_0.83.0_Linux-64bit.tar.gz -C hugo_extended_0.83.0_Linux-64bit) +cd hugo_extended_0.83.0_Linux-64bit +sudo cp hugo /usr/local/bin/hugo-0.83 +``` -### Update the submodule of the docsy theme +2. Use the commands that generate a static site in the `public/` folder: -To update the submodule of the docsy theme you need to have a repository clone. While in the repository folder, -use the git command: +Make sure to generate the SDK code first. - git submodule update --remote +```bash +python process_sdk_docs.py --input-dir ../cvat-sdk/docs/ --site-root . + +python build_docs.py +``` + +The resulting folder contains the whole site, which can be published by a server like Apache. +Read more [here](https://www.docsy.dev/docs/getting-started/) +and [here](https://gohugo.io/hosting-and-deployment/). + +### How to update the submodule of the Docsy theme + +To update the submodule of the docsy theme, you need to have a repository clone. +While in the repository folder, use the git command: + +```bash +git submodule update --remote +``` Add and then commit the change to project: - git add themes/ - git commit -m "Updating theme submodule" +```bash +git add themes/ +git commit -m "Updating theme submodule" +``` Push the commit to project repo. For example, run: - git push +```bash +git push +``` + +Make sure to update the corresponding configuration files and +theme overrides (`layouts/`, `i18n/`, etc.). diff --git a/site/assets/scss/_custom.scss b/site/assets/scss/_custom.scss index 62f6deb455b2..dac648f675d8 100644 --- a/site/assets/scss/_custom.scss +++ b/site/assets/scss/_custom.scss @@ -2,6 +2,9 @@ // // SPDX-License-Identifier: MIT +/* doesn't work in hugo */ +/* stylelint-disable media-feature-range-notation */ + /* Code blocks */ div.code-toolbar .toolbar { @@ -42,7 +45,7 @@ details > .code-toolbar { } /* Increased left padding on the sidebar of documentation */ - +/* stylelint-disable-next-line selector-class-pattern */ .td-sidebar-nav__section .ul-1 ul { padding-left: 0.6rem !important; padding-right: 0.1rem !important; @@ -168,15 +171,22 @@ details > .code-toolbar { } /* Display version menu on mobile version of the site */ - +/* stylelint-disable-next-line selector-id-pattern */ #dropdown_ver { right: 5px !important; left: auto; } @media (min-width: 991.98px) { + /* stylelint-disable-next-line selector-id-pattern */ #dropdown_ver { right: 245px !important; left: auto; } } + +/* Hide breadcrumbs, when on the root level + https://github.com/google/docsy/issues/944 +*/ +/* stylelint-disable-next-line selector-class-pattern */ +.td-breadcrumbs__single { display: none !important; } \ No newline at end of file diff --git a/site/build_docs.py b/site/build_docs.py index 7a41699126d2..0d160d65eca6 100755 --- a/site/build_docs.py +++ b/site/build_docs.py @@ -5,11 +5,12 @@ # # SPDX-License-Identifier: MIT +import os import shutil import subprocess -import tarfile import tempfile from pathlib import Path +from typing import Dict, Optional import git import toml @@ -18,8 +19,15 @@ # the initial version for the documentation site MINIMUM_VERSION = version.Version("1.5.0") +# Start the name with HUGO_ for Hugo default security checks +VERSION_URL_ENV_VAR = "HUGO_VERSION_REL_URL" -def prepare_tags(repo): +# Hugo binaries for different versions +hugo110 = "hugo-0.110" # required for new docs +hugo83 = "hugo-0.83" # required for older docs + + +def prepare_tags(repo: git.Repo): tags = {} for tag in repo.tags: tag_version = version.parse(tag.name) @@ -45,19 +53,26 @@ def write_version_item(file_object, version, url): write_version_item(f, v, "{}/{}".format(url_prefix, v)) -def git_checkout(tagname, repo, temp_dir): - subdirs = ["site/content/en/docs", "site/content/en/images"] +def git_checkout(ref: str, temp_repo: git.Repo, temp_dir: Path): + # We need to checkout with submodules, recursively - for subdir in subdirs: - shutil.rmtree(temp_dir / subdir) + subdirs = [ + "site/content/en/docs", + "site/content/en/images", + "site/assets", + "site/layouts/partials", + "site/layouts/shortcodes", + "site/themes", + ] - with tempfile.TemporaryFile() as archive: - # `git checkout` doesn't work for this, as it modifies the index. - # `git restore` would work, but it's only available since Git 2.23. - repo.git.archive(tagname, "--", subdir, output_stream=archive) - archive.seek(0) - with tarfile.open(fileobj=archive) as tar: - tar.extractall(temp_dir) + temp_repo.git.checkout(ref, recurse_submodules=True, force=True) + temp_repo.git.submodule("update", "--init", "--recursive") + tmp_repo_root = Path(temp_repo.working_tree_dir) + + for subdir in subdirs: + dst_dir = temp_dir / subdir + shutil.rmtree(dst_dir) + shutil.copytree(tmp_repo_root / subdir, dst_dir, symlinks=True) def change_version_menu_toml(filename, version): @@ -68,17 +83,31 @@ def change_version_menu_toml(filename, version): toml.dump(data, f) -def generate_docs(repo, output_dir, tags): +def generate_docs(repo: git.Repo, output_dir: os.PathLike, tags): repo_root = Path(repo.working_tree_dir) with tempfile.TemporaryDirectory() as temp_dir: content_loc = Path(temp_dir, "site") shutil.copytree(repo_root / "site", content_loc, symlinks=True) - def run_hugo(destination_dir): + def run_npm_install(): + subprocess.run(["npm", "install"], cwd=content_loc) # nosec + + def run_hugo( + destination_dir: os.PathLike, + *, + extra_env_vars: Dict[str, str] = None, + executable: Optional[str] = "hugo", + ): + extra_kwargs = {} + + if extra_env_vars: + extra_kwargs["env"] = os.environ.copy() + extra_kwargs["env"].update(extra_env_vars) + subprocess.run( # nosec [ - "hugo", + executable, "--destination", str(destination_dir), "--config", @@ -86,26 +115,52 @@ def run_hugo(destination_dir): ], cwd=content_loc, check=True, + **extra_kwargs, ) versioning_toml_path = content_loc / "versioning.toml" - # Handle the develop version + # Process the develop version generate_versioning_config(versioning_toml_path, (t.name for t in tags)) change_version_menu_toml(versioning_toml_path, "develop") - run_hugo(output_dir) + run_hugo(output_dir, executable=hugo110) + + # Create a temp repo for checkouts + temp_repo_path = Path(temp_dir) / "tmp_repo" + shutil.copytree(repo_root, temp_repo_path, symlinks=True) + temp_repo = git.Repo(temp_repo_path) + temp_repo.git.reset(hard=True, recurse_submodules=True) + # Process older versions generate_versioning_config(versioning_toml_path, (t.name for t in tags), "/..") for tag in tags: - git_checkout(tag.name, repo, Path(temp_dir)) + git_checkout(tag.name, temp_repo, Path(temp_dir)) change_version_menu_toml(versioning_toml_path, tag.name) - run_hugo(output_dir / tag.name) + run_npm_install() + run_hugo( + output_dir / tag.name, + # Docsy doesn't forward the current version url to templates + extra_env_vars={VERSION_URL_ENV_VAR: f"/cvat/{tag.name}/docs"}, + executable=hugo83, + ) + + +def validate_env(): + for hugo in [hugo83, hugo110]: + try: + subprocess.run([hugo, "version"], capture_output=True) # nosec + except (subprocess.CalledProcessError, FileNotFoundError) as ex: + raise Exception( + f"Failed to run '{hugo}', please make sure it exists." + ) from ex if __name__ == "__main__": repo_root = Path(__file__).resolve().parents[1] output_dir = repo_root / "public" + validate_env() + with git.Repo(repo_root) as repo: tags = prepare_tags(repo) generate_docs(repo, output_dir, tags) diff --git a/site/config.toml b/site/config.toml index da20a36b2381..57c65160f03f 100644 --- a/site/config.toml +++ b/site/config.toml @@ -88,7 +88,7 @@ section = ["HTML", "print"] [params] cvat_ai_terms_of_use = "https://www.cvat.ai/terms-of-use" -cvat_privacy_notice = "https://www.cvat.ai/privacy" +cvat_ai_privacy_notice = "https://www.cvat.ai/privacy" # First one is picked as the Twitter card image if not set on page. # images = ["images/project-illustration.png"] @@ -179,22 +179,23 @@ enable = false desc = "Practical questions and curated answers" [[params.links.user]] name = "YouTube" - url = "https://www.youtube.com/user/nmanovic" + url = "https://www.youtube.com/@cvat-ai" icon = "fab fa-youtube" - desc = "Practical questions and curated answers" + desc = "YouTube channel of CVAT.ai" + # Developer relevant links. These will show up on right side of footer and in the community page if you have one. [[params.links.developer]] name = "GitHub" url = "https://github.com/cvat-ai/cvat" icon = "fab fa-github" desc = "Development takes place here!" -[[params.links.developer]] - name = "Forum on Intel Developer Zone" - url = "https://community.intel.com/t5/Intel-Distribution-of-OpenVINO/bd-p/distribution-openvino-toolkit" - icon = "fas fa-envelope" - desc = "Development takes place here!" [[params.links.developer]] name ="Gitter developers chat" url = "https://gitter.im/opencv-cvat/dev" icon = "fab fa-gitter" desc = "Join our Gitter channel for community support." +[[params.links.developer]] + name = "Discord chat" + url = "https://discord.gg/fNR3eXfk6C" + icon = "fab fa-discord" + desc = "Join our Discord server for community support." diff --git a/site/content/en/docs/_index.md b/site/content/en/docs/_index.md index 7d7be595a9b4..403869e20a15 100644 --- a/site/content/en/docs/_index.md +++ b/site/content/en/docs/_index.md @@ -9,16 +9,18 @@ menu: --- CVAT is a free, online, interactive video and image annotation tool for computer vision. -It is being developed and used by Intel to annotate millions of objects with different properties. +It is being developed and used by CVAT.ai to annotate millions of objects with different properties. Many UI and UX decisions are based on feedbacks from professional data annotation team. Try it online [app.cvat.ai](https://app.cvat.ai). -Our documentation provides information for AI researchers, system administrators, developers, simple and advanced users. -The documentation is divided into three sections, and each section is divided into subsections `basic` and `advanced`. +Our documentation provides information for annotators, AI researchers, system administrators, +and developers. +The documentation is divided into three sections, and each section is divided into +subsections `basic` and `advanced`.
-{{< blocks/section color="docs" >}} +{{< blocks/section color="docs" type="row" >}} {{% blocks/feature icon="fa-sign-in-alt" title="[Getting started](/docs/getting_started/)" %}} diff --git a/site/layouts/home.html b/site/layouts/home.html index 7a6f2a7f7f05..a1590cdde657 100644 --- a/site/layouts/home.html +++ b/site/layouts/home.html @@ -1,6 +1,10 @@ - + {{ $version_rel_url := os.Getenv "HUGO_VERSION_REL_URL" }} + {{ if not $version_rel_url }} + {{ $version_rel_url = "/cvat/docs" }} + {{ end }} + \ No newline at end of file diff --git a/site/layouts/partials/footer.html b/site/layouts/partials/footer.html index 51293ad20d02..c2bc4afad51d 100644 --- a/site/layouts/partials/footer.html +++ b/site/layouts/partials/footer.html @@ -1,37 +1,42 @@ -{{ $links := .Site.Params.links }} -