From d2b1056ff09afa836f61f73a7a6b63e4eaa36e1d Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Mon, 28 Oct 2024 15:55:04 +0000 Subject: [PATCH 01/15] Check RTD build Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index a883f76bd6..c378c3e758 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -532,3 +532,5 @@ def setup(app): myst_heading_anchors = 5 myst_enable_extensions = ["colon_fence"] + +#check RTD build From a4043540001ab0a887f23f3f98036d70ee485cc4 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Mon, 28 Oct 2024 17:06:11 +0000 Subject: [PATCH 02/15] Change conf.py Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index c378c3e758..3247ede5dc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -223,6 +223,8 @@ # We've decided to rely on RTD's default robots.txt, so we no longer need to specify a custom one, # see discussion at https://github.com/kedro-org/kedro/issues/3741 # html_extra_path = [str(here / "robots.txt")] +html_extra_path = [str(here / "sitemap.xml")] + # Removes, from all docs, the copyright footer. html_show_copyright = False From 5e8f8229cf3b464ab0dd157674ee29591bf61300 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Tue, 12 Nov 2024 15:29:03 +0000 Subject: [PATCH 03/15] Use custom robots.txt and sitemap Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 2 +- docs/source/robots.txt | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 docs/source/robots.txt diff --git a/docs/source/conf.py b/docs/source/conf.py index 3247ede5dc..782fa3f9d3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -223,7 +223,7 @@ # We've decided to rely on RTD's default robots.txt, so we no longer need to specify a custom one, # see discussion at https://github.com/kedro-org/kedro/issues/3741 # html_extra_path = [str(here / "robots.txt")] -html_extra_path = [str(here / "sitemap.xml")] +html_extra_path = [str(here / "robots.txt"), str(here / "sitemap.xml")] # Removes, from all docs, the copyright footer. diff --git a/docs/source/robots.txt b/docs/source/robots.txt new file mode 100644 index 0000000000..bc65593c50 --- /dev/null +++ b/docs/source/robots.txt @@ -0,0 +1,6 @@ +User-agent: * +Disallow: / + +Allow: /en/stable/ +Allow: /projects/kedro-viz/en/stable/ +Allow: /projects/kedro-datasets/en/latest/ From 39f552f7fb72274e44e3be2fd27bb5babfb48a29 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Tue, 12 Nov 2024 16:09:27 +0000 Subject: [PATCH 04/15] Try only robots.txt Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 782fa3f9d3..cb82553a52 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -223,7 +223,7 @@ # We've decided to rely on RTD's default robots.txt, so we no longer need to specify a custom one, # see discussion at https://github.com/kedro-org/kedro/issues/3741 # html_extra_path = [str(here / "robots.txt")] -html_extra_path = [str(here / "robots.txt"), str(here / "sitemap.xml")] +html_extra_path = [str(here / "robots.txt")] # Removes, from all docs, the copyright footer. From fc77b95c01413397ee93317a3d65a63522f397a8 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Tue, 12 Nov 2024 16:29:45 +0000 Subject: [PATCH 05/15] Use custom sitemap.xml Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 2 +- docs/source/robots.txt | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 docs/source/robots.txt diff --git a/docs/source/conf.py b/docs/source/conf.py index cb82553a52..3247ede5dc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -223,7 +223,7 @@ # We've decided to rely on RTD's default robots.txt, so we no longer need to specify a custom one, # see discussion at https://github.com/kedro-org/kedro/issues/3741 # html_extra_path = [str(here / "robots.txt")] -html_extra_path = [str(here / "robots.txt")] +html_extra_path = [str(here / "sitemap.xml")] # Removes, from all docs, the copyright footer. diff --git a/docs/source/robots.txt b/docs/source/robots.txt deleted file mode 100644 index bc65593c50..0000000000 --- a/docs/source/robots.txt +++ /dev/null @@ -1,6 +0,0 @@ -User-agent: * -Disallow: / - -Allow: /en/stable/ -Allow: /projects/kedro-viz/en/stable/ -Allow: /projects/kedro-datasets/en/latest/ From c99e1662735d2d458855bc9804d9830861c47764 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Tue, 12 Nov 2024 16:36:04 +0000 Subject: [PATCH 06/15] Clean comments Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 3247ede5dc..622b7edca6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -534,5 +534,3 @@ def setup(app): myst_heading_anchors = 5 myst_enable_extensions = ["colon_fence"] - -#check RTD build From d26e2d8743db07cf7f4d4f3bb4048b8e41a43cd0 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Tue, 19 Nov 2024 11:23:05 +0000 Subject: [PATCH 07/15] Add robots noindex Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 622b7edca6..2ebbda8bb7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -225,6 +225,7 @@ # html_extra_path = [str(here / "robots.txt")] html_extra_path = [str(here / "sitemap.xml")] +html_meta = {"robots": "noindex, nofollow"} # Removes, from all docs, the copyright footer. html_show_copyright = False From a63bc5162d81213186f4afe88204a2a0d3c1b43e Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Tue, 19 Nov 2024 14:28:36 +0000 Subject: [PATCH 08/15] Add template layout Signed-off-by: Dmitry Sorokin --- docs/source/_templates/layout.html | 5 +++++ docs/source/conf.py | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 docs/source/_templates/layout.html diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html new file mode 100644 index 0000000000..46df78574f --- /dev/null +++ b/docs/source/_templates/layout.html @@ -0,0 +1,5 @@ +{% extends "!layout.html" %} +{% block extrahead %} + {{ super() }} + +{% endblock %} diff --git a/docs/source/conf.py b/docs/source/conf.py index 2ebbda8bb7..622b7edca6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -225,7 +225,6 @@ # html_extra_path = [str(here / "robots.txt")] html_extra_path = [str(here / "sitemap.xml")] -html_meta = {"robots": "noindex, nofollow"} # Removes, from all docs, the copyright footer. html_show_copyright = False From 08926bc10737c99ae9d5d2b5163c4eaef035035a Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Tue, 19 Nov 2024 15:29:28 +0000 Subject: [PATCH 09/15] Add html_meta, remove layout Signed-off-by: Dmitry Sorokin --- docs/source/_templates/layout.html | 5 ----- docs/source/conf.py | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 docs/source/_templates/layout.html diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html deleted file mode 100644 index 46df78574f..0000000000 --- a/docs/source/_templates/layout.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends "!layout.html" %} -{% block extrahead %} - {{ super() }} - -{% endblock %} diff --git a/docs/source/conf.py b/docs/source/conf.py index 622b7edca6..2ebbda8bb7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -225,6 +225,7 @@ # html_extra_path = [str(here / "robots.txt")] html_extra_path = [str(here / "sitemap.xml")] +html_meta = {"robots": "noindex, nofollow"} # Removes, from all docs, the copyright footer. html_show_copyright = False From a35eae10b3ec2a49570bc8eb68477da29ba98c35 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Wed, 20 Nov 2024 11:06:48 +0000 Subject: [PATCH 10/15] Rebuild docs Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2ebbda8bb7..ccb68e6f9f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -226,6 +226,7 @@ html_extra_path = [str(here / "sitemap.xml")] html_meta = {"robots": "noindex, nofollow"} +# rebuild docs # Removes, from all docs, the copyright footer. html_show_copyright = False From ff07526cb0c2a148f7ecad0d8b161190d1b5bb4d Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Thu, 21 Nov 2024 09:25:06 +0000 Subject: [PATCH 11/15] Return layout.html Signed-off-by: Dmitry Sorokin --- docs/source/_templates/layout.html | 5 +++++ docs/source/conf.py | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 docs/source/_templates/layout.html diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html new file mode 100644 index 0000000000..46df78574f --- /dev/null +++ b/docs/source/_templates/layout.html @@ -0,0 +1,5 @@ +{% extends "!layout.html" %} +{% block extrahead %} + {{ super() }} + +{% endblock %} diff --git a/docs/source/conf.py b/docs/source/conf.py index ccb68e6f9f..9472cd632e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -225,9 +225,6 @@ # html_extra_path = [str(here / "robots.txt")] html_extra_path = [str(here / "sitemap.xml")] -html_meta = {"robots": "noindex, nofollow"} -# rebuild docs - # Removes, from all docs, the copyright footer. html_show_copyright = False From 27f67dc42c9eee80f7cea171dd27ff457e3125ce Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Mon, 25 Nov 2024 10:54:38 +0000 Subject: [PATCH 12/15] Remove layout Signed-off-by: Dmitry Sorokin --- docs/source/_templates/layout.html | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/source/_templates/layout.html diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html deleted file mode 100644 index 46df78574f..0000000000 --- a/docs/source/_templates/layout.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends "!layout.html" %} -{% block extrahead %} - {{ super() }} - -{% endblock %} From a838254728bf742cd6742143ac52464973021ff0 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Mon, 25 Nov 2024 12:12:27 +0000 Subject: [PATCH 13/15] Switch to custom robots.txt and sitemap.xml Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 9 +++++---- docs/source/robots.txt | 8 ++++++++ docs/source/sitemap.xml | 13 +++++-------- 3 files changed, 18 insertions(+), 12 deletions(-) create mode 100644 docs/source/robots.txt diff --git a/docs/source/conf.py b/docs/source/conf.py index 9472cd632e..0543a828c0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -220,10 +220,11 @@ # html_extra_path used to define a path to robots.txt which is used by webcrawlers # to ignore or allow certain links. -# We've decided to rely on RTD's default robots.txt, so we no longer need to specify a custom one, -# see discussion at https://github.com/kedro-org/kedro/issues/3741 -# html_extra_path = [str(here / "robots.txt")] -html_extra_path = [str(here / "sitemap.xml")] +# We've tried to rely on RTD's default `robots.txt`, but it doesn't support sub-projects (Viz and Datasets), +# so we decided to use our own `robots.txt` again, with a link to the custom `sitemap.xml`. +# See the discussion at https://github.com/kedro-org/kedro/issues/3741. +# html_extra_path = [str(here / "robots.txt"), str(here / "sitemap.xml")] +html_extra_path = [str(here / "robots.txt")] # Removes, from all docs, the copyright footer. html_show_copyright = False diff --git a/docs/source/robots.txt b/docs/source/robots.txt new file mode 100644 index 0000000000..b1a46f78e2 --- /dev/null +++ b/docs/source/robots.txt @@ -0,0 +1,8 @@ +User-agent: * +Disallow: / + +Allow: /en/stable/ +Allow: /projects/kedro-viz/en/stable/ +Allow: /projects/kedro-datasets/en/latest/ + +Sitemap: https://docs.kedro.org/en/stable/sitemap.xml diff --git a/docs/source/sitemap.xml b/docs/source/sitemap.xml index 059f1ac1c6..4b542d807a 100644 --- a/docs/source/sitemap.xml +++ b/docs/source/sitemap.xml @@ -1,34 +1,31 @@ https://docs.kedro.org/en/stable/ - 2024-09-01T18:53:11.571849+00:00 + 2024-11-26T18:53:11.571849+00:00 monthly 1.0 https://docs.kedro.org/en/latest/ + 2024-11-26T18:53:11.571849+00:00 daily 0.5 https://docs.kedro.org/projects/kedro-viz/en/stable/ - 2024-09-01T18:53:11.571849+00:00 + 2024-11-26T18:53:11.571849+00:00 monthly 1.0 https://docs.kedro.org/projects/kedro-viz/en/latest/ + 2024-11-26T18:53:11.571849+00:00 daily 0.5 - - https://docs.kedro.org/projects/kedro-datasets/en/stable/ - 2024-09-01T18:53:11.571849+00:00 - monthly - 1.0 - https://docs.kedro.org/projects/kedro-datasets/en/latest/ + 2024-11-26T18:53:11.571849+00:00 daily 0.5 From b60209ea13b2f7a76185a30be19ae0b789d829b5 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Mon, 25 Nov 2024 12:34:10 +0000 Subject: [PATCH 14/15] Change html_extra_path Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 3 +-- docs/source/{ => extra_files}/robots.txt | 0 docs/source/{ => extra_files}/sitemap.xml | 0 3 files changed, 1 insertion(+), 2 deletions(-) rename docs/source/{ => extra_files}/robots.txt (100%) rename docs/source/{ => extra_files}/sitemap.xml (100%) diff --git a/docs/source/conf.py b/docs/source/conf.py index 0543a828c0..185c2f4b05 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -223,8 +223,7 @@ # We've tried to rely on RTD's default `robots.txt`, but it doesn't support sub-projects (Viz and Datasets), # so we decided to use our own `robots.txt` again, with a link to the custom `sitemap.xml`. # See the discussion at https://github.com/kedro-org/kedro/issues/3741. -# html_extra_path = [str(here / "robots.txt"), str(here / "sitemap.xml")] -html_extra_path = [str(here / "robots.txt")] +html_extra_path = [str(here / "extra_files")] # Removes, from all docs, the copyright footer. html_show_copyright = False diff --git a/docs/source/robots.txt b/docs/source/extra_files/robots.txt similarity index 100% rename from docs/source/robots.txt rename to docs/source/extra_files/robots.txt diff --git a/docs/source/sitemap.xml b/docs/source/extra_files/sitemap.xml similarity index 100% rename from docs/source/sitemap.xml rename to docs/source/extra_files/sitemap.xml From 2a5f4fdbe984701fd0cb53f2065ff0b520505975 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin Date: Tue, 26 Nov 2024 10:13:01 +0000 Subject: [PATCH 15/15] Add link to RTD github Signed-off-by: Dmitry Sorokin --- docs/source/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 185c2f4b05..cea2a550f8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -222,7 +222,8 @@ # to ignore or allow certain links. # We've tried to rely on RTD's default `robots.txt`, but it doesn't support sub-projects (Viz and Datasets), # so we decided to use our own `robots.txt` again, with a link to the custom `sitemap.xml`. -# See the discussion at https://github.com/kedro-org/kedro/issues/3741. +# See the discussion at https://github.com/kedro-org/kedro/issues/3741 +# and https://github.com/readthedocs/readthedocs.org/issues/6841 html_extra_path = [str(here / "extra_files")] # Removes, from all docs, the copyright footer.