From 0893806f09df23b437a57679a0cb4a2e14831752 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Thu, 3 Oct 2024 16:39:28 -0400 Subject: [PATCH] docs overhaul: render flyteidl under the /api/ path (#5802) * fix flyteidl structure so it renders under /api/ Signed-off-by: Niels Bantilan * do not check in flyteidl docs Signed-off-by: Niels Bantilan * update gitignore and unneeded conf Signed-off-by: Niels Bantilan * add mock DOCSEARCH_API_KEY to docs test ci Signed-off-by: Niels Bantilan * add css styling (#5803) * add css styling Signed-off-by: Niels Bantilan * update logo height Signed-off-by: Niels Bantilan --------- Signed-off-by: Niels Bantilan --------- Signed-off-by: Niels Bantilan --- .github/workflows/tests.yml | 3 + .gitignore | 2 +- docs/_static/custom.css | 113 ++++++++++++++++++ docs/_static/custom.js | 10 ++ docs/_templates/layout.html | 6 +- docs/api/index.md | 13 +- docs/community/contribute_docs.md | 2 +- docs/conf.py | 49 ++++++-- docs/index.md | 80 ++++++------- docs/reference_flyteidl.md | 18 --- docs/user_guide/index.md | 1 - flyteidl/README.md | 69 +---------- flyteidl/protos/docs/contributing.md | 68 +++++++++++ flyteidl/protos/{index.rst => docs_index.rst} | 1 + 14 files changed, 293 insertions(+), 142 deletions(-) create mode 100644 docs/_static/custom.css create mode 100644 docs/_static/custom.js delete mode 100644 docs/reference_flyteidl.md create mode 100644 flyteidl/protos/docs/contributing.md rename flyteidl/protos/{index.rst => docs_index.rst} (96%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d69466464..0571e60eea 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,6 +69,9 @@ jobs: - name: Build the documentation working-directory: ${{ github.workspace }}/flyte shell: bash -el {0} + env: + # this is a fake key for algolia docsearch to make docs build process pass + DOCSEARCH_API_KEY: fake_docsearch_api_key run: | conda activate monodocs-env make docs diff --git a/.gitignore b/.gitignore index 35c73f38c9..6b280884f9 100644 --- a/.gitignore +++ b/.gitignore @@ -31,8 +31,8 @@ vendor/ **/bin/ docs/_tags/ docs/api/flytectl -docs/protos docs/api/flytekit +docs/api/flyteidl docs/flytesnacks docs/examples docs/_src diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000000..faa3b25f33 --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,113 @@ +html[data-theme=light], html[data-theme=dark] { + --pst-color-primary: #7652A2; + --pst-color-link: #7652A2; + --pst-color-link-hover: #7652A2; +} + + +/* Styling for page layout */ +.bd-page-width { + max-width: 100%; +} + +/* style the main title logo */ +.navbar-brand:hover, .navbar-brand:visited:hover { + text-decoration: none; +} + +.navbar-brand img { + height: 70%; +} + +/* +Center the main page content only when the sidebar has the .hide-on-wide +class, which is only on the root page +*/ +.bd-sidebar.hide-on-wide + main.bd-main .bd-content { + justify-content: center; +} + +/* left sidebar nav width */ +.bd-sidebar-primary { + width: 15%; + min-width: 250px; + padding: 1rem; +} + +/* make the scroll bar sleeker */ +.bd-sidebar::-webkit-scrollbar { + width: 3px; + height: 3px; + background: transparent; +} + +.bd-sidebar::-webkit-scrollbar-thumb { + background: var(--pst-color-border); +} + +/* make sure page content fills up the page */ +.bd-main .bd-content { + justify-content: left; +} + +.bd-main .bd-content .bd-article-container { + padding: 2.5rem; + max-width: 75em; +} + +/* aligh navbar items with the sidebar */ +.bd-header .navbar-header-items__start { + width: 15%; + min-width: 250px; +} + +/* styles for the main page subtitle and badges */ +div.sd-card.sd-shadow-sm.subtitle-and-badges { + border: none; + box-shadow: none !important; +} + +div.subtitle-and-badges .sd-card-body { + padding: 0; +} + +div.subtitle-and-badges .sd-card-title { + color: #808080; + font-weight: 350 !important; + font-size: 25px; + padding-top: 10px; + padding-bottom: 10px; +} + +div.subtitle-and-badges a { + text-decoration: none; +} + +div.subtitle-and-badges a img { + margin: 2px 0; +} + +/* update table hover color to neutral color */ +.table tbody tr:hover { + background-color: var(--pst-color-table-inner-border); +} + +/* custom footer style */ +div.custom-footer { + text-align: center; + background-color: var(--pst-color-surface); + padding: 15px 0; + color: var(--pst-color-text-muted); +} + +/* anchor link hover style */ +a:hover, +nav.bd-links li>a:active, +nav.bd-links li>a:hover, +.bd-header ul.navbar-nav>li.nav-item>.nav-link:hover { + color: var(--pst-color-primary); +} + +.bd-header ul.navbar-nav>li.nav-item>.nav-link:hover:before { + border-bottom: max(3px, .1875rem, .12em) solid var(--pst-color-primary); +} diff --git a/docs/_static/custom.js b/docs/_static/custom.js new file mode 100644 index 0000000000..573b21e6fd --- /dev/null +++ b/docs/_static/custom.js @@ -0,0 +1,10 @@ +window.addEventListener("DOMContentLoaded", function() { + // Select all elements with class "external" + var externalLinks = document.querySelectorAll("a.external"); + + // Loop through each element with class "external" + externalLinks.forEach(function(link) { + // Set the target attribute to "_blank" + link.setAttribute("target", "_blank"); + }); +}); diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a53f62a53f..fa6329208f 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -42,9 +42,11 @@ runllm-keyboard-shortcut="Mod+j" runllm-slack-community-url="https://flyte-org.slack.com/join/shared_invite/zt-2eq2fgs5f-UGUWnMYVB9agervilmlyaw#/shared-invite/email" runllm-name="RunLLM"> - + +
{% trans copyright=copyright|e -%} Copyright © {{ copyright }} - {%- endtrans %} + {%- endtrans %} +
{% endblock %} diff --git a/docs/api/index.md b/docs/api/index.md index 0db0bdad22..cde46b40a7 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -2,6 +2,7 @@ ::::{grid} 2 +:gutter: 2 :::{grid-item-card} {octicon}`rocket` Flytekit SDK :link: flytekit/docs_index @@ -10,13 +11,20 @@ Flytekit is the main Python SDK for building workflows. ::: -:::{grid-item-card} {octicon}`workflow` Flytectl SDK +:::{grid-item-card} {octicon}`workflow` Flytectl CLI :link: flytectl/docs_index :link-type: doc Flytectl is a Golang binary that can be installed on any platform supported by Golang. ::: +:::{grid-item-card} {octicon}`code` FlyteIDL +:link: flyteidl/docs_index +:link-type: doc + +Flyte’s core specification language. +::: + :::: ```{toctree} @@ -25,4 +33,5 @@ Flytectl is a Golang binary that can be installed on any platform supported by G flytekit flytectl -``` \ No newline at end of file +flyteidl +``` diff --git a/docs/community/contribute_docs.md b/docs/community/contribute_docs.md index 2e011b2d32..c54c442ca7 100644 --- a/docs/community/contribute_docs.md +++ b/docs/community/contribute_docs.md @@ -15,7 +15,7 @@ The Flyte documentation comprises the following types: * **API documentation:** * {doc}`flytekit <../api/flytekit/docs_index>` * {doc}`flytectl <../api/flytectl/docs_index>` - * {doc}`flyteidl <../reference_flyteidl>` + * {doc}`flyteidl <../api/flyteidl/docs_index>` * **{ref}`Tutorials `:** Longer, more advanced guides that use multiple Flyte features to solve real-world problems. Some tutorials may require extra setup, while others can only run on larger clusters. * **{ref}`Integrations examples `:** These examples showcase how to use the Flyte plugins that integrate with the broader data and machine learning ecosystem. * **{ref}`Deployment documentation `:** Guidance on deploying and configuring the Flyte backend. diff --git a/docs/conf.py b/docs/conf.py index df4f075bd3..ebd887f4a5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -288,6 +288,16 @@ "flytectl/gen/flytectl_demo_teardown": "../../api/flytectl/gen/flytectl_demo_teardown", "flytectl/gen/flytectl_demo_exec": "../../api/flytectl/gen/flytectl_demo_exec", "flytectl/gen/flytectl_demo_reload": "../../api/flytectl/gen/flytectl_demo_reload", + + # flyteidl + "reference_flyteidl": "../../api/flyteidl/docs_index.html", + "protos/docs/core/core": "../../api/flyteidl/docs/core/core.html", + "protos/docs/admin/admin": "../../api/flyteidl/docs/admin/admin.html", + "protos/docs/service/service": "../../api/flyteidl/docs/service/service.html", + "protos/docs/datacatalog/datacatalog": "../../api/flyteidl/docs/datacatalog/datacatalog.html", + "protos/docs/event/event": "../../api/flyteidl/docs/event/event.html", + "protos/docs/plugins/plugins": "../../api/flyteidl/docs/plugins/plugins.html", + "protos/README": "../../api/flyteidl/contributing.html", } @@ -375,11 +385,31 @@ html_theme_options = { # custom flyteorg pydata theme options - "github_url": "https://github.com/flyteorg/flyte", + # "github_url": "https://github.com/flyteorg/flyte", + "logo": { + "text": "Flyte", + }, + "external_links": [ + {"name": "Flyte", "url": "https://flyte.org"}, + ], "icon_links": [ { "name": "GitHub", + "icon": "fa-brands fa-github", + "type": "fontawesome", "url": "https://github.com/flyteorg/flyte", + }, + { + "name": "Slack", + "url": "https://slack.flyte.org", + "icon": "fa-brands fa-slack", + "type": "fontawesome", + }, + { + "name": "Flyte", + "url": "https://flyte.org", + "icon": "fa-solid fa-dragon", + "type": "fontawesome", } ], "use_edit_page_button": True, @@ -390,9 +420,9 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] -# html_css_files = [] -# html_js_files = [] +html_static_path = ["_static"] +html_css_files = ["custom.css"] +html_js_files = ["custom.js"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -476,9 +506,9 @@ tags_overview_title = "Pages by tags" # Algolia Docsearch credentials -docsearch_app_id = os.getenv("DOCSEARCH_APP_ID") +docsearch_app_id = "WLG0MZB58Q" docsearch_api_key = os.getenv("DOCSEARCH_API_KEY") -docsearch_index_name = os.getenv("DOCSEARCH_INDEX_NAME") +docsearch_index_name = "flyte" # -- Options for intersphinx extension --------------------------------------- @@ -529,7 +559,7 @@ # Pattern for replacing all ref/doc labels that point to protos/docs with /protos/docs PROTO_REF_PATTERN = r"([:<])(protos/docs)" -PROTO_REF_REPLACE = r"\1/protos/docs" +PROTO_REF_REPLACE = r"\1/api/flyteidl/docs" # These patterns are used to replace values in source files that are imported # from other repos. @@ -549,7 +579,7 @@ r"<_tags/DistributedComputing>": r"", r"{ref}`bioinformatics `": r"bioinformatics", PROTO_REF_PATTERN: PROTO_REF_REPLACE, - r"/protos/docs/service/index": r"/protos/docs/service/service" + r"/protos/docs/service/index": r"/api/flyteidl/docs/service/service" } # r"": r"", @@ -626,8 +656,7 @@ "name": "flyteidl", "source": "../flyteidl", "docs_path": "protos", - "dest": "protos", # to stay compatible with flyteidl docs path naming - "cmd": ["cp", "../flyteidl/README.md", "protos/README.md"], + "dest": "api/flyteidl", "local": True, } ] diff --git a/docs/index.md b/docs/index.md index 4c65791cb8..c93321f2cf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,54 +1,55 @@ # Welcome to Flyte! -```{eval-rst} -.. raw:: html +````{card} The highly scalable and flexible workflow orchestrator that unifies data, ML and analytics. +:class-card: subtitle-and-badges -

- The highly scalable and flexible workflow orchestrator that unifies data, ML and analytics. -

- -.. image:: https://img.shields.io/badge/Graduate%20Project-Linux%20Foundation-purple?style=for-the-badge - :target: https://lfaidata.foundation/projects/flyte/ - :alt: Linux Foundation - -.. image:: https://img.shields.io/github/stars/flyteorg/flyte?label=github&logo=github&style=for-the-badge - :target: https://github.com/flyteorg/flyte - :alt: GitHub Repo stars +```{image} https://img.shields.io/badge/Graduate%20Project-Linux%20Foundation-purple?style=for-the-badge +:target: https://lfaidata.foundation/projects/flyte/ +:alt: Linux Foundation +``` -.. image:: https://img.shields.io/github/release/flyteorg/flyte.svg?style=for-the-badge&color=blue - :target: https://github.com/flyteorg/flyte/releases/latest - :alt: Flyte Release +```{image} https://img.shields.io/github/stars/flyteorg/flyte?label=github&logo=github&style=for-the-badge +:target: https://github.com/flyteorg/flyte +:alt: GitHub Repo stars +``` -.. image:: https://img.shields.io/github/actions/workflow/status/flyteorg/flyte/tests.yml?label=tests&style=for-the-badge - :target: https://github.com/flyteorg/flyte/actions/workflows/tests.yml - :alt: GitHub Test Status +```{image} https://img.shields.io/github/release/flyteorg/flyte.svg?style=for-the-badge&color=blue +:target: https://github.com/flyteorg/flyte/releases/latest +:alt: Flyte Release +``` -.. image:: https://img.shields.io/github/actions/workflow/status/flyteorg/flyte/sandbox.yml?label=Sandbox%20docker%20image&style=for-the-badge - :target: https://github.com/flyteorg/flyte/actions/workflows/sandbox.yml - :alt: GitHub Sandbox Status +```{image} https://img.shields.io/github/actions/workflow/status/flyteorg/flyte/tests.yml?label=tests&style=for-the-badge +:target: https://github.com/flyteorg/flyte/actions/workflows/tests.yml +:alt: GitHub Test Status +``` -.. image:: https://img.shields.io/github/milestones/closed/flyteorg/flyte?style=for-the-badge - :target: https://github.com/flyteorg/flyte/milestones?state=closed - :alt: Completed Milestones +```{image} https://img.shields.io/github/actions/workflow/status/flyteorg/flyte/sandbox.yml?label=Sandbox%20docker%20image&style=for-the-badge +:target: https://github.com/flyteorg/flyte/actions/workflows/sandbox.yml +:alt: GitHub Sandbox Status +``` -.. image:: https://img.shields.io/pypi/dm/flytekit?color=blue&label=flytekit%20downloads&style=for-the-badge&logo=pypi&logoColor=white - :target: https://github.com/flyteorg/flytekit - :alt: Flytekit Downloads +```{image} https://img.shields.io/github/milestones/closed/flyteorg/flyte?style=for-the-badge +:target: https://github.com/flyteorg/flyte/milestones?state=closed +:alt: Completed Milestones +``` -.. image:: https://img.shields.io/badge/Slack-Chat-pink?style=for-the-badge&logo=slack - :target: https://slack.flyte.org - :alt: Flyte Slack +```{image} https://img.shields.io/pypi/dm/flytekit?color=blue&label=flytekit%20downloads&style=for-the-badge&logo=pypi&logoColor=white +:target: https://github.com/flyteorg/flytekit +:alt: Flytekit Downloads +``` -.. image:: https://img.shields.io/badge/LICENSE-Apache2.0-ff69b4.svg?style=for-the-badge - :target: http://www.apache.org/licenses/LICENSE-2.0.html - :alt: License +```{image} https://img.shields.io/badge/Slack-Chat-pink?style=for-the-badge&logo=slack +:target: https://slack.flyte.org +:alt: Flyte Slack +``` -.. |br| raw:: html +```{image} https://img.shields.io/badge/LICENSE-Apache2.0-ff69b4.svg?style=for-the-badge +:target: http://www.apache.org/licenses/LICENSE-2.0.html +:alt: License +``` -
-
+```` -``` [Flyte](https://github.com/flyteorg/flyte) is an open-source, Kubernetes-native workflow orchestrator implemented in [Go](https://go.dev/). It enables highly @@ -102,7 +103,7 @@ Below are the API reference to the different components of Flyte: - Flyte's official Python SDK. * - {doc}`FlyteCTL ` - Flyte's command-line interface for interacting with a Flyte cluster. -* - {doc}`FlyteIDL ` +* - {doc}`FlyteIDL ` - Flyte's core specification language. ``` @@ -135,6 +136,5 @@ flytesnacks/tutorials/index flytesnacks/integrations/index deployment/index api/index -reference_flyteidl community/index ``` diff --git a/docs/reference_flyteidl.md b/docs/reference_flyteidl.md deleted file mode 100644 index bfe9924c9e..0000000000 --- a/docs/reference_flyteidl.md +++ /dev/null @@ -1,18 +0,0 @@ -# FlyteIDL API reference - -The protocol buffers defined here provide a high level specification of various -entities in Flyte control plane and data plane. It provides detailed definitions -and documentation of all these entities. - -```{toctree} -:maxdepth: 2 -:name: flyteidltoc - -Flyte Core Language Specification -FlyteAdmin Service -FlyteAdmin REST and gRPC Interface -Data Catalog Service -Internal and External Eventing Interface -Flyte Task Plugin Specification -Contributing Guide -``` diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index 44c975f6ae..ae439c8760 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -94,7 +94,6 @@ concepts/component_architecture/index :hidden: flytekit-java -unionml pterodactyl latch sdk ``` diff --git a/flyteidl/README.md b/flyteidl/README.md index 67685f45b7..25a5badebc 100644 --- a/flyteidl/README.md +++ b/flyteidl/README.md @@ -1,6 +1,6 @@ # Flyteidl -This is one of the core repositories of Flyte. It contains the Specification of the Flyte Language using protobuf messages, the Backend API specification in gRPC, and Swagger REST. The repo contains the generated clients and protocol message structures in multiple languages. Along with the generated code, the repository also contains the Golang clients for Flyte's backend APIs (the services grouped under FlyteAdmin). +This is one of the core components of Flyte. It contains the Specification of the Flyte Language using protobuf messages, the Backend API specification in gRPC, and Swagger REST. The repo contains the generated clients and protocol message structures in multiple languages. Along with the generated code, the repository also contains the Golang clients for Flyte's backend APIs (the services grouped under FlyteAdmin). [![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://slack.flyte.org) @@ -11,69 +11,4 @@ This is one of the core repositories of Flyte. It contains the Specification of ## Contributing to Flyteidl -## Tooling for Flyteidl - -1. Run ``make download_tooling`` to install generator dependencies. - -```bash - make download_tooling -``` - -2. Ensure Docker is installed locally. -3. Run ``make generate`` to generate all the code, mock client, and docs for FlyteAdmin Service. - -```bash - make generate -``` - -4. To add new dependencies for documentation generation, modify ``doc-requirements.in`` and run - -```bash - make doc-requirements.txt -``` - -## Docs structure - -The index.rst files for protos are arranged in parallel under the ``docs`` folder. -All the proto definitions are within ``protos/flyteidl`` and their corresponding docs are in ``protos/docs``. - -``` -docs -├── admin -│   ├── admin.rst -│   └── index.rst -├── core -│   ├── core.rst -│   └── index.rst -├── datacatalog -│   ├── datacatalog.rst -│   └── index.rst -├── event -│   ├── event.rst -│   └── index.rst -├── plugins -│   ├── index.rst -│   └── plugins.rst -├── service -│   ├── index.rst -│   └── service.rst -``` - -Each module in protos has a module in docs with the same name. -For example: ``protos/flyteidl/core`` has a module ``protos/docs/core`` under the ``docs`` folder which has the corresponding index and documentation files. - - -## Generating Documentation - -* If a new module is to be introduced, follow the structure for core files in `generate_protos.sh` file which helps generate the core documentation from its proto files. -``` - core_proto_files=`ls protos/flyteidl/core/*.proto |xargs` - # Remove any currently generated file - ls -d protos/docs/core/* | grep -v index.rst | xargs rm - protoc --doc_out=protos/docs/core --doc_opt=restructuredtext,core.rst -I=protos `echo $core_proto_files` -``` - -* ``make generate`` generates the modified rst files. - -* ``make html`` generates the Sphinx documentation from the docs folder that uses the modified rst files. - +See the [contributing docs](protocs/contributing.md) for more information. diff --git a/flyteidl/protos/docs/contributing.md b/flyteidl/protos/docs/contributing.md new file mode 100644 index 0000000000..825ad7eeb6 --- /dev/null +++ b/flyteidl/protos/docs/contributing.md @@ -0,0 +1,68 @@ +## Contributing to FlyteIDL + +## Install Tooling + +1. Run ``make download_tooling`` to install generator dependencies. + +```bash + make download_tooling +``` + +2. Ensure Docker is installed locally. +3. Run ``make generate`` to generate all the code, mock client, and docs for FlyteAdmin Service. + +```bash + make generate +``` + +4. To add new dependencies for documentation generation, modify ``doc-requirements.in`` and run + +```bash + make doc-requirements.txt +``` + +## Docs structure + +The index.rst files for protos are arranged in parallel under the ``docs`` folder. +All the proto definitions are within ``protos/flyteidl`` and their corresponding docs are in ``protos/docs``. + +``` +docs +├── admin +│   ├── admin.rst +│   └── index.rst +├── core +│   ├── core.rst +│   └── index.rst +├── datacatalog +│   ├── datacatalog.rst +│   └── index.rst +├── event +│   ├── event.rst +│   └── index.rst +├── plugins +│   ├── index.rst +│   └── plugins.rst +├── service +│   ├── index.rst +│   └── service.rst +``` + +Each module in protos has a module in docs with the same name. +For example: ``protos/flyteidl/core`` has a module ``protos/docs/core`` under the ``docs`` folder which has the corresponding index and documentation files. + + +## Generating Documentation + +* If a new module is to be introduced, follow the structure for core files in `generate_protos.sh` file which helps generate the core documentation from its proto files. +``` + core_proto_files=`ls protos/flyteidl/core/*.proto |xargs` + # Remove any currently generated file + ls -d protos/docs/core/* | grep -v index.rst | xargs rm + protoc --doc_out=protos/docs/core --doc_opt=restructuredtext,core.rst -I=protos `echo $core_proto_files` +``` + +* ``make generate`` generates the modified rst files. + +* ``make html`` generates the Sphinx documentation from the docs folder that uses the modified rst files. + diff --git a/flyteidl/protos/index.rst b/flyteidl/protos/docs_index.rst similarity index 96% rename from flyteidl/protos/index.rst rename to flyteidl/protos/docs_index.rst index 39544bcc2e..27383673de 100644 --- a/flyteidl/protos/index.rst +++ b/flyteidl/protos/docs_index.rst @@ -16,3 +16,4 @@ and documentation of all these entities. docs/event/index docs/plugins/index docs/service/index + docs/contributing