add build scripts

.gitignore

@@ -49,3 +49,5 @@ build/
 
 # nodeenv
 env/
+
+__cloudpods-website*

Makefile

@@ -25,6 +25,9 @@ docker-start-en:
 build:
 	yarn build
 
+oneline-build:
+	./scripts/build.py --edition=ce --multi-versions --no-out-fetch
+
 docker-build:
 	./tools/run-in-docker.sh build
 

scripts/build.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+import argparse
+import textwrap
+
+from converter import processor
+from builder.consts import MODE_ONLINE, MODE_OFFLINE, MODE_OEM
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Use processor and hugo build document website.",
+        formatter_class=argparse.RawTextHelpFormatter)
+
+    bool_action = 'store_true'
+    if hasattr(argparse, 'BooleanOptionalAction'):
+        bool_action = argparse.BooleanOptionalAction
+    parser.add_argument('--mode',
+                        choices=[MODE_ONLINE, MODE_OFFLINE, MODE_OEM],
+                        default=MODE_ONLINE,
+                        help='''\
+                            Build mode
+                                online: Public website
+                                offline: Docs inside ee edition
+                                oem: Docs inside ee OEM edition
+                            ''')
+    # TODO: impl OEM build
+    parser.add_argument('--oem-name', help="OEM NAME")
+    parser.add_argument('--host', help="Hugo base url", default="")
+    parser.add_argument('--edition', help="Build edition",
+                        choices=[processor.EDITION_CE, processor.EDITION_EE],
+                        default=processor.EDITION_CE)
+    parser.add_argument('--multi-versions', help="Enable multi versions",
+                        action=bool_action)
+    parser.add_argument('--out-fetch', help="Not fetch upstream",
+                        action=bool_action)
+
+    args = parser.parse_args()
+
+    return args
+
+
+def start_build(args):
+    mode = args.mode
+    drv = get_build_driver(mode)
+    drv.start('./content', args)
+
+
+def get_build_driver(mode):
+    from builder.drivers import get_driver
+    return get_driver(mode)
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    start_build(args)

scripts/builder/consts.py

@@ -0,0 +1,25 @@
+import os
+
+MODE_ONLINE = 'online'
+MODE_OFFLINE = 'offline'
+MODE_OEM = 'oem'
+
+DEFAULT_VERSION_ARRAY = [
+    '3.11',
+    '3.10',
+    # '3.9',
+    # '3.8',
+    # '3.7',
+    # '3.6',
+    # '3.4',
+    # '3.3',
+    # '3.2'
+]
+
+
+def VERSION_ARRAY():
+    # VERSIONS='3.9,3.8'
+    versions = os.environ.get('VERSIONS', None)
+    if versions:
+        return versions.split(',')
+    return DEFAULT_VERSION_ARRAY

scripts/builder/docusaurus.py

@@ -0,0 +1,123 @@
+import os
+import tempfile
+
+
+def generate_config(original_config_path, version=None):
+    with open(original_config_path) as f:
+        content = f.readlines()
+        return update_config(content, version=version)
+
+
+def update_config(content_lines, version=None):
+    result = []
+    prev_line = ''
+    for line in content_lines:
+        new_line = update_config_line(line, prev_line, version=version)
+        prev_line = line
+        result.append(new_line)
+    return ''.join(result)
+
+
+def update_config_line(line, prev_line, version=None):
+    if line.startswith("  baseUrl: '/',"):
+        if version:
+            return f"  baseUrl: '/{version}/',"
+    return line
+
+
+class Docusaurus(object):
+
+    def __init__(self, content_dir):
+        self._host = ''
+        self._content_dir = content_dir
+        self._dest_dir = 'build'
+        self._current_branch = ''
+        self._current_version = ''
+        self._base_url_prefix = ''
+        self._versions = []
+        self._title = 'Cloudpods'
+
+    def set_host(self, host):
+        self._host = host
+        return self
+
+    def get_host(self):
+        return self._host
+
+    def get_title(self):
+        return self._title
+
+    def set_title(self, title):
+        self._title = title
+        return self
+
+    def set_dest_dir(self, dest):
+        self._dest_dir = dest
+        return self
+
+    def get_dest_dir(self):
+        return self._dest_dir
+
+    def set_current_branch(self, cur_br):
+        self._current_branch = cur_br
+        return self
+
+    def get_current_branch(self):
+        return self._current_branch
+
+    def set_current_version(self, ver):
+        self._current_version = ver
+        return self
+
+    def get_current_version(self):
+        return self._current_version
+
+    def set_versions(self, versions):
+        self._versions = versions
+        return self
+
+    def set_base_url_prefix(self, pre):
+        self._base_url_prefix = pre
+        return self
+
+    def get_base_url_prefix(self):
+        return self._base_url_prefix
+
+    def generate_config(self, version=None):
+        config_content = generate_config('./docusaurus.config.js', version=version)
+        fp = tempfile.NamedTemporaryFile(dir=os.curdir, delete=False, prefix='__cloudpods-website', suffix='.js')
+        fp.write(config_content.encode(encoding='utf-8'))
+        return fp.name
+
+    def execute(self):
+        from utils import run_process
+
+        ver_dir = ''
+        ver_title = self.get_title()
+        if self._versions:
+            if not self.get_current_version():
+                raise Exception("Current version not set when versions are %s" % self._versions)
+            if self.get_current_version() != self._versions[0]:
+                ver_dir = 'v' + self.get_current_version()
+                ver_title = ver_title + ' ' + self.get_current_version()
+        dest = self.get_dest_dir()
+        base_url = self.get_host()
+        base_url = os.path.join(base_url, self.get_base_url_prefix())
+        version = None
+        if not base_url.endswith("/"):
+            base_url += "/"
+        if ver_dir:
+            dest = os.path.join(dest, ver_dir)
+            base_url = base_url + '/' + ver_dir + '/'
+            version = ver_dir
+        temp_config_file = self.generate_config(version=version)
+        run_process(['rm', '-rf', dest])
+        cmd = ['yarn', 'docusaurus',
+               'build',
+               '--config', temp_config_file,
+               '--out-dir', dest]
+        run_process(cmd)
+
+
+if __name__ == '__main__':
+    print(generate_config('../../docusaurus.config.js', version="v3.10"))

scripts/builder/drivers.py

@@ -0,0 +1,153 @@
+import os
+
+from .consts import MODE_ONLINE, MODE_OFFLINE, MODE_OEM
+from .consts import VERSION_ARRAY
+from .docusaurus import Docusaurus
+from utils import run_process
+from converter import processor
+
+
+__drivers = {}
+
+
+def __register_driver(mode, drv):
+    __drivers[mode] = drv
+
+
+def get_driver(mode):
+    drv = __drivers.get(mode, None)
+    if drv:
+        return drv
+    raise Exception('Not found driver by mode: %s' % mode)
+
+
+class BaseDriver(object):
+
+    def __init__(self, mode):
+        self._mode = mode
+
+    def get_mode(self):
+        return self._mode
+
+    def is_oem(self):
+        return self.get_mode() == MODE_OEM
+
+    def get_scopes(self):
+        return [
+            [processor.SCOPE_SYSTEM, '', 'public'],
+        ]
+
+    def pre_process(self, content_dir, args, version=None):
+        ret = []
+        for scope_pair in self.get_scopes():
+            scope = scope_pair[0]
+            base_url_prefix = scope_pair[1]
+            dest_dir = scope_pair[2]
+            out_dir = os.path.join('./_output',
+                                   'content_' + args.edition + "_" + scope)
+            if version:
+                out_dir = out_dir + "_" + version
+
+            # clean out_dir first
+            run_process('rm -rf %s' % out_dir)
+
+            p = processor.DirProcess(content_dir, out_dir)
+            p.\
+                include_by_scope(scope).\
+                include_by_oem(self.is_oem()). \
+                include_by_edition(args.edition).\
+                start()
+            ret.append([out_dir, base_url_prefix, dest_dir])
+        return ret
+
+    def start(self, content_dir, args):
+        if args.multi_versions:
+            self.multi_versions_build(content_dir, args)
+        else:
+            self.singal_version_build(content_dir, args)
+
+    def singal_version_build(self, content_dir, args):
+        out_dirs = self.pre_process(content_dir, args)
+        for out_dir in out_dirs:
+            docs = self.new_docs(out_dir, args)
+            docs.execute()
+
+    def new_docs(self, out_dir_pair, args):
+        out_dir = out_dir_pair[0]
+        base_url_prefix = out_dir_pair[1]
+        dest_dir = out_dir_pair[2]
+        docs = Docusaurus(out_dir)
+        docs.set_dest_dir(dest_dir)
+        docs.set_base_url_prefix(base_url_prefix)
+        if args.host:
+            docs.set_host(args.host)
+        if args.edition == processor.EDITION_EE:
+            docs.set_base_config('config-ee.toml')
+        return docs
+
+    def update_branch(self, branch):
+        run_process('git checkout -q %s' % branch)
+        upstream_cmd = 'git rev-parse @{u}'
+        local_cmd = 'git rev-parse @'
+        upstream_out = run_process(upstream_cmd)
+        local_out = run_process(local_cmd)
+        if upstream_out != local_out:
+            run_process('git merge -q upstream/%s' % branch)
+
+    def check_and_update(self, version):
+        branch = 'master'
+        if version != 'master':
+            branch = 'release/%s' % version
+        self.update_branch(branch)
+
+    def multi_versions_build(self, content_dir, args):
+        versions = VERSION_ARRAY()
+        cur_branch = run_process('git rev-parse --abbrev-ref HEAD')
+        print("args.out_fetch: %s" % args.out_fetch)
+        if not args.out_fetch:
+            run_process('git remote update')
+        for version in versions:
+            self.check_and_update(version)
+            out_dirs = self.pre_process(content_dir, args, version=version)
+            for out_dir in out_dirs:
+                docs = self.new_docs(out_dir, args)
+                docs.set_current_version(version)
+                docs.set_versions(versions)
+                docs.execute()
+        run_process('git checkout %s' % cur_branch)
+
+
+class OnlineDriver(BaseDriver):
+
+    def __init__(self):
+        super(OnlineDriver, self).__init__(MODE_ONLINE)
+
+
+__register_driver(MODE_ONLINE, OnlineDriver())
+
+
+class OfflineDriver(BaseDriver):
+
+    def __init__(self):
+        super(OfflineDriver, self).__init__(MODE_OFFLINE)
+        run_process('rm -rf ./public')
+
+    def get_scopes(self):
+        return [
+            [processor.SCOPE_SYSTEM, '/docs/', 'public/docs'],
+            [processor.SCOPE_DOMAIN, '/docs/domain', 'public/docs/domain'],
+            [processor.SCOPE_PROJECT, '/docs/project', 'public/docs/project'],
+        ]
+
+
+__register_driver(MODE_OFFLINE, OfflineDriver())
+
+
+class OfflineOEMDriver(OfflineDriver):
+
+    def __init__(self):
+        super().__init__()
+        self._mode = MODE_OEM
+
+
+__register_driver(MODE_OEM, OfflineOEMDriver())