process_sdk_docs.py

#!/usr/bin/env python3

# Copyright (C) CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT

import argparse
import os
import os.path as osp
import re
import shutil
import sys
import textwrap
from glob import iglob
from typing import Callable

from inflection import underscore


class Processor:
    _reference_files: list[str]

    def __init__(self, *, input_dir: str, site_root: str) -> None:
        self._input_dir = input_dir
        self._site_root = site_root

        self._content_dir = osp.join(self._site_root, "content")
        self._sdk_reference_dir = osp.join(self._content_dir, "en/docs/api_sdk/sdk/reference")
        self._templates_dir = osp.join(self._site_root, "templates")

    @staticmethod
    def _copy_files(src_dir: str, glob_pattern: str, dst_dir: str) -> list[str]:
        copied_files = []

        for src_path in iglob(osp.join(src_dir, glob_pattern), recursive=True):
            src_filename = osp.relpath(src_path, src_dir)
            dst_path = osp.join(dst_dir, src_filename)
            # assume dst dir exists
            shutil.copy(src_path, dst_path, follow_symlinks=True)

            copied_files.append(dst_path)

        return copied_files

    def _copy_pages(self):
        self._reference_files = self._copy_files(
            self._input_dir, "*/**/*.md", self._sdk_reference_dir
        )

    def _add_page_headers(self):
        """
        Adds headers required by hugo to docs pages
        """

        HEADER_SEPARATOR = "---"

        for p in self._reference_files:
            with open(p) as f:
                contents = f.read()

            assert not contents.startswith(HEADER_SEPARATOR), p

            lines = contents.splitlines()

            assert lines[0].startswith("#")
            classname = lines[0][1:].strip()

            header = textwrap.dedent(
                """\
                %(header_separator)s
                title: '%(classname)s class reference'
                linkTitle: '%(classname)s'
                weight: 10
                description: ''
                %(header_separator)s
            """
                % {"header_separator": HEADER_SEPARATOR, "classname": classname}
            )

            contents = header + "\n".join(lines[1:])

            with open(p, "w") as f:
                f.write(contents)

    def _move_api_summary(self):
        """
        Moves API summary section from README to apis/_index
        """

        SUMMARY_REPLACE_TOKEN = "{{REPLACEME:apis_summary}}"  # nosec

        with open(osp.join(self._input_dir, "api_summary.md")) as f:
            apis_summary = f.read()

        apis_index_filename = osp.join(
            osp.relpath(self._sdk_reference_dir, self._content_dir), "apis/_index.md"
        )
        apis_index_path = osp.join(self._templates_dir, apis_index_filename + ".template")
        with open(apis_index_path) as f:
            contents = f.read()

        contents = contents.replace(SUMMARY_REPLACE_TOKEN, apis_summary)

        with open(osp.join(self._content_dir, apis_index_filename), "w") as f:
            f.write(contents)

    def _fix_page_links_and_references(self):
        """
        Replaces reference page links from full lowercase (which is generated by hugo from the
        orignal camelcase and creates broken links) ('authapi') to the minus-case ('auth-api'),
        which is more readable and works.
        Adds an extra parent directory part to links ('../') as hugo requires, even for neighbor
        files.
        """

        mapping = {}

        for src_path in self._reference_files:
            src_filename = osp.relpath(src_path, self._sdk_reference_dir)
            dst_filename = underscore(src_filename).replace("_", "-")
            dst_path = osp.join(self._sdk_reference_dir, dst_filename)
            os.rename(src_path, dst_path)
            mapping[src_filename] = dst_filename

        self._reference_files = [osp.join(self._sdk_reference_dir, p) for p in mapping.values()]

        for p in iglob(self._sdk_reference_dir + "/**/*.md", recursive=True):
            with open(p) as f:
                contents = f.read()

            for src_filename, dst_filename in mapping.items():
                src_dir, src_filename = osp.split(osp.splitext(src_filename)[0])
                dst_filename = osp.basename(osp.splitext(dst_filename)[0])
                contents = re.sub(
                    rf"(\[.*?\]\()((?:\.\./)?(?:{src_dir}/)?){src_filename}((?:#[^\)]*?)?\))",
                    rf"\1../\2{dst_filename}\3",
                    contents,
                )

            with open(p, "w") as f:
                f.write(contents)

    def _process_non_code_blocks(self, text: str, handlers: list[Callable[[str], str]]) -> str:
        """
        Allows to process Markdown documents with passed callbacks. Callbacks are only
        executed outside code blocks.
        """

        used_quotes = ""
        block_start_pos = 0
        inside_code_block = False
        while block_start_pos < len(text):
            pattern = re.compile(used_quotes or "```|`")
            next_code_block_quote = pattern.search(text, pos=block_start_pos)
            if next_code_block_quote is not None:
                if not used_quotes:
                    inside_code_block = False
                    block_end_pos = next_code_block_quote.start(0)
                    used_quotes = next_code_block_quote.group(0)
                else:
                    inside_code_block = True
                    block_end_pos = next_code_block_quote.end(0)
                    used_quotes = None
            else:
                block_end_pos = len(text)

            if not inside_code_block:
                block = text[block_start_pos:block_end_pos]

                for handler in handlers:
                    block = handler(block)

                text = text[:block_start_pos] + block + text[block_end_pos:]
                block_end_pos = block_start_pos + len(block) + len(used_quotes)

            block_start_pos = block_end_pos

        return text

    def _escape_free_square_brackets(self, text: str) -> str:
        return re.sub(r"\[([^\[\]]*?)\]([^\(])", r"\[\1\]\2", text)

    def _add_angle_brackets_to_free_links(self, text: str) -> str:
        # Adapted from https://stackoverflow.com/a/31952097
        URL_REGEX = (
            # Scheme (HTTP, HTTPS):
            r"(?:https?:\/\/)"
            r"(?:"
            # www:
            r"(?:www\.)?"
            # Host and domain (including ccSLD):
            r"(?:(?:[a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]\.)+)"
            # TLD:
            r"(?:[a-zA-Z]{2,6})"
            # IP Address:
            r"|(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"
            r")"
            # Port:
            r"(?::\d{1,5})?"
            # Query path:
            r"(?:(?:\/\S+)*|\/)"
        )

        text = re.sub(
            r"(\A|[\.\s])(" + URL_REGEX + r")([\.\s]|\Z)",
            r"\1<\2>\3",
            text,
            flags=re.MULTILINE,
        )

        return text

    def _fix_parsing_problems(self):
        """
        Adds angle brackets to freestanding links, as the linter requires. Such links can appear
        from the generated model and api descriptions.
        Adds escapes to freestanding square brackets to make parsing correct.
        """

        for p in iglob(self._sdk_reference_dir + "/**/*.md", recursive=True):
            with open(p) as f:
                contents = f.read()

            contents = self._process_non_code_blocks(
                contents,
                [
                    self._add_angle_brackets_to_free_links,
                    self._escape_free_square_brackets,
                ],
            )

            with open(p, "w") as f:
                f.write(contents)

    def run(self):
        assert osp.isdir(self._input_dir), self._input_dir
        assert osp.isdir(self._site_root), self._site_root
        assert osp.isdir(self._sdk_reference_dir), self._sdk_reference_dir
        assert osp.isdir(self._templates_dir), self._templates_dir

        self._copy_pages()
        self._move_api_summary()
        self._add_page_headers()
        self._fix_page_links_and_references()
        self._fix_parsing_problems()


def parse_args(args=None):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--input-dir",
        type=osp.abspath,
        default="cvat-sdk/docs/",
        help="Path to the cvat-sdk/docs/ directory",
    )
    parser.add_argument(
        "--site-root",
        type=osp.abspath,
        default="site/",
    )

    return parser.parse_args(args)


def main(args=None):
    args = parse_args(args)
    processor = Processor(input_dir=args.input_dir, site_root=args.site_root)
    processor.run()
    return 0


if __name__ == "__main__":
    sys.exit(main())