Merge branch 'main' into collapse-my-toc

kedro-org · Apr 3, 2023 · 17dd1d5 · 17dd1d5
2 parents 93b9943 + ebfed0c
commit 17dd1d5
Show file tree

Hide file tree

Showing 59 changed files with 170 additions and 244 deletions.
diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml
@@ -354,34 +354,6 @@ jobs:
           name: Pip-compile requirements file
           command: conda activate kedro_builder; make pip-compile
 
-  build_docs:
-    executor:
-      name: docker
-      python_version: "3.7"
-    steps:
-      - setup
-      - run:
-          name: Build docs
-          command: make build-docs
-      - run:
-          name: Pip freeze including docs dependencies
-          command: pip freeze
-          when: always
-
-  docs_linkcheck:
-    executor:
-      name: docker
-      python_version: "3.8"
-    steps:
-      - setup
-      - run:
-          name: Check for broken links
-          command: make linkcheck
-      - run:
-          name: Pip freeze including docs dependencies
-          command: pip freeze
-          when: always
-
   sync:
     docker:
       # https://circleci.com/docs/2.0/circleci-images/#circleci-base-image
@@ -544,27 +516,6 @@ jobs:
 workflows:
   version: 2.1
 
-  build_docs_only:
-    when:
-      and:
-        - <<pipeline.parameters.docs_change>>
-        - not: <<pipeline.parameters.code_change>>
-        - not: <<pipeline.parameters.release_kedro>>
-        - not: <<pipeline.parameters.run_hourly>>
-        - not: <<pipeline.parameters.run_nightly>>
-    jobs:
-      - build_docs
-      - docs_linkcheck
-      - lint:
-          matrix:
-            parameters:
-              python_version: ["3.7", "3.8", "3.9", "3.10"]
-      - all_circleci_checks_succeeded:
-          requires:
-            - build_docs
-            - docs_linkcheck
-            - lint
-
   build_code_and_docs:
     when:
       and:
@@ -601,8 +552,6 @@ workflows:
           matrix:
             parameters:
               python_version: ["3.7", "3.8", "3.9", "3.10"]
-      - build_docs
-      - docs_linkcheck
       - all_circleci_checks_succeeded:
           requires:
             - e2e_tests
@@ -612,8 +561,6 @@ workflows:
             - lint
             - pip_compile
             - win_pip_compile
-            - build_docs
-            - docs_linkcheck
 
   main_updated:
     when:

diff --git a/.gitignore b/.gitignore
@@ -134,6 +134,11 @@ venv.bak/
 /site
 /kedro/framework/html
 
+# Sphinx documentation
+# Additional files created by sphinx.ext.autosummary
+# Some of them are actually tracked to control the output
+/docs/source/kedro.*
+
 # mypy
 .mypy_cache/
 

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -9,20 +9,22 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.7"
+    python: "3.8"
     nodejs: "19"
   apt_packages:
     - libasound2
   jobs:
     post_create_environment:
       - npm install -g @mermaid-js/mermaid-cli
       - ./docs/kedro-datasets-docs.sh
+    pre_build:
+      - python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   builder: html
-  configuration: docs/conf.py
-  fail_on_warning: false  # Turn back on soon
+  configuration: docs/source/conf.py
+  fail_on_warning: true
 
 # Build documentation with MkDocs
 # mkdocs:

diff --git a/Makefile b/Makefile
@@ -44,11 +44,8 @@ linkcheck:
 	pip install -e ".[docs]"
 	./docs/build-docs.sh "linkcheck"
 
-devserver: build-docs
-	cd docs && npm install && npm start
-
 package: clean install
-	python setup.py sdist bdist_wheel
+	python -m pip install build && python -m build
 
 install-test-requirements:
 	pip install -r test_requirements.txt
@@ -63,7 +60,7 @@ print-python-env:
 	@./tools/print_env.sh
 
 databricks-build:
-	python setup.py bdist_wheel
+	python -m pip install build && python -m build
 	python ./tools/databricks_build.py
 
 sign-off:

diff --git a/RELEASE.md b/RELEASE.md
@@ -15,6 +15,8 @@
 
 ## Bug fixes and other changes
 * Improvements to documentation about configuration.
+* Improvements to Jupyter E2E tests.
+* Improvements to documentation on visualising Kedro projects on Databricks.
 
 ## Breaking changes to the API
 

diff --git a/docs/build-docs.sh b/docs/build-docs.sh
@@ -7,35 +7,12 @@ set -o nounset
 
 action=$1
 
-# Move some files around. We need a separate build directory, which would
-# have all the files, build scripts would shuffle the files,
-# we don't want that happening on the actual code locally.
-# When running on ReadTheDocs, sphinx-build would run directly on the original files,
-# but we don't care about the code state there.
-rm -rf docs/build
+# Reinstall kedro-datasets locally
 rm -rf kedro/datasets
-mkdir docs/build/
-cp -r docs/_templates docs/conf.py docs/*.svg docs/*.json  docs/build/
-
 bash docs/kedro-datasets-docs.sh
 
 if [ "$action" == "linkcheck" ]; then
-  sphinx-build -c docs/ -ETan -j auto -D language=en -b linkcheck docs/build/ docs/build/html
+  sphinx-build -WETan -j auto -D language=en -b linkcheck -d docs/build/doctrees docs/source docs/build/linkcheck
 elif [ "$action" == "docs" ]; then
-  sphinx-build -c docs/ -ETa -j auto -D language=en docs/build/ docs/build/html
+  sphinx-build -WETa -j auto -D language=en -b html -d docs/build/doctrees docs/source docs/build/html
 fi
-
-# Clean up build artefacts
-rm -rf docs/build/html/_sources
-
-# Copy built HTML to temp directory, clean up build dir and replace with built docs only
-rm -rf docs/temp
-mkdir docs/temp/
-mkdir docs/temp/html
-cp -rf docs/build/html/* docs/temp/html
-
-rm -rf docs/build
-mkdir docs/build
-mkdir docs/build/html
-cp -rf docs/temp/html/* docs/build/html
-rm -rf docs/temp
diff --git a/docs/package.json b/docs/package.json
diff --git a/docs/source/css/qb1-sphinx-rtd.css → docs/source/_static/css/qb1-sphinx-rtd.css b/docs/source/css/qb1-sphinx-rtd.css → docs/source/_static/css/qb1-sphinx-rtd.css
diff --git a/docs/source/css/theme-overrides.css → docs/source/_static/css/theme-overrides.css b/docs/source/css/theme-overrides.css → docs/source/_static/css/theme-overrides.css
diff --git a/docs/_templates/autosummary/base.rst → docs/source/_templates/autosummary/base.rst b/docs/_templates/autosummary/base.rst → docs/source/_templates/autosummary/base.rst
diff --git a/docs/_templates/autosummary/class.rst → docs/source/_templates/autosummary/class.rst b/docs/_templates/autosummary/class.rst → docs/source/_templates/autosummary/class.rst
diff --git a/docs/_templates/autosummary/module.rst → .../source/_templates/autosummary/module.rst b/docs/_templates/autosummary/module.rst → .../source/_templates/autosummary/module.rst
diff --git a/docs/_templates/breadcrumbs.html → docs/source/_templates/breadcrumbs.html b/docs/_templates/breadcrumbs.html → docs/source/_templates/breadcrumbs.html
diff --git a/docs/_templates/layout.html → docs/source/_templates/layout.html b/docs/_templates/layout.html → docs/source/_templates/layout.html
diff --git a/docs/conf.py → docs/source/conf.py b/docs/conf.py → docs/source/conf.py
@@ -15,9 +15,7 @@
 import importlib
 import os
 import re
-import shutil
 import sys
-from distutils.dir_util import copy_tree
 from inspect import getmembers, isclass, isfunction
 from pathlib import Path
 from typing import List, Tuple
@@ -49,15 +47,12 @@
     "sphinx.ext.napoleon",
     "sphinx_autodoc_typehints",
     "sphinx.ext.doctest",
-    "sphinx.ext.todo",
-    "sphinx.ext.coverage",
-    "sphinx.ext.mathjax",
     "sphinx.ext.ifconfig",
     "sphinx.ext.viewcode",
-    "nbsphinx",
     "sphinx_copybutton",
     "sphinxcontrib.mermaid",
     "myst_parser",
+    "notfound.extension",
 ]
 
 # enable autosummary plugin (table of contents for modules/classes/class
@@ -68,6 +63,7 @@
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
+html_static_path = ["_static"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
@@ -82,7 +78,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -305,23 +301,6 @@
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
-# -- Extension configuration -------------------------------------------------
-
-# nbsphinx_prolog = """
-# see here for prolog/epilog details:
-# https://nbsphinx.readthedocs.io/en/0.3.1/prolog-and-epilog.html
-# """
-
-nbsphinx_epilog = """
-.. note::
-
-     Found a bug, or didn't find what you were looking for? 🙏 `Please file a
-     ticket <https://github.com/kedro-org/kedro/issues/new/choose>`_
-"""
-
-# -- NBconvert kedro config -------------------------------------------------
-nbsphinx_kedro_name = "kedro"
-
 # -- Kedro specific configuration -----------------------------------------
 KEDRO_MODULES = [
     "kedro.io",
@@ -493,20 +472,6 @@ def autodoc_process_docstring(app, what, name, obj, options, lines):
     remove_arrows_in_examples(lines)
 
 
-def _prepare_build_dir(app, config):
-    """Get current working directory to the state expected
-    by the ReadTheDocs builder. Shortly, it does the same as
-    ./build-docs.sh script except not running `sphinx-build` step."""
-    build_root = Path(app.srcdir)
-    build_out = Path(app.outdir)
-    copy_tree(str(here / "source"), str(build_root))
-    copy_tree(str(build_root / "api_docs"), str(build_root))
-    shutil.rmtree(str(build_root / "api_docs"))
-    shutil.rmtree(str(build_out), ignore_errors=True)
-    copy_tree(str(build_root / "css"), str(build_out / "_static" / "css"))
-    shutil.rmtree(str(build_root / "css"))
-
-
 def env_override(default_appid):
     build_version = os.getenv("READTHEDOCS_VERSION")
 
@@ -533,7 +498,6 @@ def _add_jinja_filters(app):
 
 
 def setup(app):
-    app.connect("config-inited", _prepare_build_dir)
     app.connect("builder-inited", _add_jinja_filters)
     app.connect("autodoc-process-docstring", autodoc_process_docstring)
     app.add_css_file("css/qb1-sphinx-rtd.css")
@@ -573,4 +537,4 @@ def setup(app):
 # https://github.com/mermaidjs/mermaid.cli#linux-sandbox-issue
 mermaid_params = ["-p", here / "puppeteer-config.json", "-s", "2"]
 # https://github.com/kedro-org/kedro/issues/2451
-mermaid_version = mermaid_init_js = None
+mermaid_version = mermaid_init_js = ""
diff --git a/docs/source/contribution/development_for_databricks.md b/docs/source/contribution/development_for_databricks.md
@@ -7,7 +7,7 @@
 This guide describes how to efficiently develop features and fixes for Kedro on Databricks. Using this guide, you will be able to quickly test your locally modified version of Kedro on Databricks as part of a build-and-test development cycle.
 
 ```{note}
-This page is for people developing changes to Kedro that need to test them on Databricks. If you are working on a Kedro project and need more information about project-deployment, consult the [documentation for deploying Kedro projects on Databricks](../deployment/databricks.md).
+This page is for people developing changes to Kedro that need to test them on Databricks. If you are working on a Kedro project and need more information about workflows, consult the [documentation for developing a Kedro project on Databricks](../integrations/databricks_workspace.md).
 ```
 
 ## Prerequisites

diff --git a/docs/source/deployment/aws_batch.md b/docs/source/deployment/aws_batch.md
@@ -105,7 +105,7 @@ A job queue is the bridge between the submitted jobs and the compute environment
 
 ### Configure the credentials
 
-Ensure you have the necessary AWS credentials in place before moving on, so that your pipeline can access and interact with the AWS services. Check out [the AWS CLI documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config) for instructions on how to set this up.
+Ensure you have the necessary AWS credentials in place before moving on, so that your pipeline can access and interact with the AWS services. Check out [the AWS CLI documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) for instructions on how to set this up.
 
 ```{note}
 You should configure the default region to match the region where you've created the Batch resources.

diff --git a/docs/source/deployment/deployment_guide.md b/docs/source/deployment/deployment_guide.md
@@ -15,7 +15,6 @@ We also provide information to help you deploy to the following:
 * to [Prefect](prefect.md)
 * to [Kubeflow Workflows](kubeflow.md)
 * to [AWS Batch](aws_batch.md)
-* to [Databricks](databricks.md)
 * to [Dask](dask.md)
 
 <!--- There has to be some non-link text in the bullets above, if it's just links, there's a Sphinx bug that fails the build process-->

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -138,6 +138,13 @@ Welcome to Kedro's documentation!
 
    logging/logging
 
+.. toctree::
+   :maxdepth: 2
+   :caption: Integrations
+
+   integrations/databricks.rst
+   integrations/pyspark.rst
+
 .. toctree::
    :maxdepth: 2
    :caption: Development
@@ -160,18 +167,11 @@ Welcome to Kedro's documentation!
    deployment/prefect
    deployment/kubeflow
    deployment/aws_batch
-   deployment/databricks
    deployment/aws_sagemaker
    deployment/aws_step_functions
    deployment/airflow_astronomer
    deployment/dask
 
-.. toctree::
-   :maxdepth: 2
-   :caption: PySpark integration
-
-   tools_integration/pyspark
-
 .. toctree::
    :maxdepth: 2
    :caption: Resources

diff --git a/docs/source/integrations/databricks.md b/docs/source/integrations/databricks.md
@@ -0,0 +1,9 @@
+# Databricks integration
+
+```{toctree}
+:caption: Databricks
+:maxdepth: 2
+
+databricks_workspace.md
+databricks_visualisation.md
+```