refactor website building process (#892)

Summary:
- group all sphinx build non-html static resources (js, css) into `website/static/_sphinx` to differentiate from other resources used by our website
- gitignore `website/static/_sphinx` but not other `js` `css`
- remove [unnecessary rewriting](https://github.com/pytorch/captum/blob/34387f564b40f63f7ffa0ae5ced3828321d9a92e/scripts/parse_sphinx.py#L60-L67) of `_static/searchtools.js` in script `parse_sphinx.py` since `searchtools.js` from modern sphinx does not have such content, e.g., https://github.com/pytorch/captum/blob/gh-pages/js/searchtools.js
- remove `website/static/_sphinx-sources/` which are not used in our website https://github.com/pytorch/captum/tree/gh-pages/_sphinx-sources
- remove comitted `pygments.css` from website since sphinx will generate the file during building
- remove erroneous manual css imports because docusaurus will auto [marge all css files](https://v1.docusaurus.io/docs/en/api-pages#styles) and use the merged file
![Screen Shot 2022-03-09 at 1 05 32 AM](https://user-images.githubusercontent.com/5113450/157382207-a0abdafc-bfdc-4537-bd97-153781ffc032.png)

Pull Request resolved: #892

Reviewed By: vivekmig

Differential Revision: D34757334

Pulled By: aobo-y

fbshipit-source-id: 8d3dda46088329884db1b4d0e19a528009d9a739

Author: aobo-y

.gitignore

@@ -109,9 +109,8 @@ website/pages/tutorials/*
 
 ## Generated for Sphinx
 website/pages/api/
-website/static/js/*
-!website/static/js/mathjax.js
-!website/static/js/code_block_buttons.js
-website/static/_sphinx-sources/
-node_modules
+website/static/_sphinx/
+
+# Insight
+captum/insights/attr_vis/frontend/node_modules/
 captum/insights/attr_vis/widget/static

scripts/build_docs.sh

@@ -35,43 +35,36 @@ cd sphinx || exit
 make html
 cd .. || exit
 
-echo "-----------------------------------"
-echo "Building Captum Docusaurus site"
-echo "-----------------------------------"
-cd website || exit
-yarn
-
 # run script to parse html generated by sphinx
 echo "--------------------------------------------"
-echo "Parsing Sphinx docs and moving to Docusaurus"
+echo "Parsing Sphinx docs and moving to Website"
 echo "--------------------------------------------"
-cd ..
 mkdir -p "website/pages/api/"
 
 cwd=$(pwd)
 python scripts/parse_sphinx.py -i "${cwd}/sphinx/build/html/" -o "${cwd}/website/pages/api/"
 
 SPHINX_STATIC_DIR='sphinx/build/html/_static/'
-DOCUSAURUS_JS_DIR='website/static/js/'
-DOCUSAURUS_CSS_DIR='website/static/css/'
+WEBSITE_SPHINX_DIR='website/static/_sphinx/'
 
-mkdir -p $DOCUSAURUS_JS_DIR
+mkdir -p $WEBSITE_SPHINX_DIR
 
 # move static files from /sphinx/build/html/_static/*:
-cp "${SPHINX_STATIC_DIR}documentation_options.js" "${DOCUSAURUS_JS_DIR}documentation_options.js"
-cp "${SPHINX_STATIC_DIR}jquery.js" "${DOCUSAURUS_JS_DIR}jquery.js"
-cp "${SPHINX_STATIC_DIR}underscore.js" "${DOCUSAURUS_JS_DIR}underscore.js"
-cp "${SPHINX_STATIC_DIR}doctools.js" "${DOCUSAURUS_JS_DIR}doctools.js"
-cp "${SPHINX_STATIC_DIR}language_data.js" "${DOCUSAURUS_JS_DIR}language_data.js"
-cp "${SPHINX_STATIC_DIR}searchtools.js" "${DOCUSAURUS_JS_DIR}searchtools.js"
-cp "${SPHINX_STATIC_DIR}katex_autorenderer.js" "${DOCUSAURUS_JS_DIR}katex_autorenderer.js"
-cp "${SPHINX_STATIC_DIR}katex-math.css" "${DOCUSAURUS_CSS_DIR}katex-math.css"
+for sphinx_static_file in 'documentation_options.js' \
+               'jquery.js' \
+               'underscore.js' \
+               'doctools.js' \
+               'language_data.js' \
+               'searchtools.js' \
+               'katex_autorenderer.js' \
+               'pygments.css' \
+               'katex-math.css'
+do
+  cp "${SPHINX_STATIC_DIR}${sphinx_static_file}" "${WEBSITE_SPHINX_DIR}${sphinx_static_file}"
+done
 
 # searchindex.js is not static util
-cp "sphinx/build/html/searchindex.js" "${DOCUSAURUS_JS_DIR}searchindex.js"
-
-# copy module sources
-cp -r "sphinx/build/html/_sources/" "website/static/_sphinx-sources/"
+cp "sphinx/build/html/searchindex.js" "${WEBSITE_SPHINX_DIR}searchindex.js"
 
 echo "-----------------------------------"
 echo "Generating tutorials"
@@ -80,7 +73,11 @@ mkdir -p "website/_tutorials"
 mkdir -p "website/static/files"
 python scripts/parse_tutorials.py -w "${cwd}"
 
+echo "-----------------------------------"
+echo "Install Website dependencies"
+echo "-----------------------------------"
 cd website || exit
+yarn
 
 if [[ $BUILD_STATIC == true ]]; then
   echo "-----------------------------------"

scripts/parse_sphinx.py

@@ -5,19 +5,22 @@
 
 from bs4 import BeautifulSoup
 
+# no need to import css from built path
+# coz docusaurus merge all css files within static folder automatically
+# https://v1.docusaurus.io/docs/en/api-pages#styles
 base_scripts = """
 <script type="text/javascript" id="documentation_options" data-url_root="./"
-src="/js/documentation_options.js"></script>
-<script type="text/javascript" src="/js/jquery.js"></script>
-<script type="text/javascript" src="/js/underscore.js"></script>
-<script type="text/javascript" src="/js/doctools.js"></script>
-<script type="text/javascript" src="/js/language_data.js"></script>
-<script type="text/javascript" src="/js/searchtools.js"></script>
+src="/_sphinx/documentation_options.js"></script>
+<script type="text/javascript" src="/_sphinx/jquery.js"></script>
+<script type="text/javascript" src="/_sphinx/underscore.js"></script>
+<script type="text/javascript" src="/_sphinx/doctools.js"></script>
+<script type="text/javascript" src="/_sphinx/language_data.js"></script>
+<script type="text/javascript" src="/_sphinx/searchtools.js"></script>
 """  # noqa: E501
 
 search_js_scripts = """
 <script type="text/javascript">
-    jQuery(function() { Search.loadIndex("/js/searchindex.js"); });
+    jQuery(function() { Search.loadIndex("/_sphinx/searchindex.js"); });
 </script>
 
 <script type="text/javascript" id="searchindexloader"></script>
@@ -26,9 +29,8 @@
 katex_scripts = """
 <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.min.js"></script>
 <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/contrib/auto-render.min.js"></script>
-<script src="/js/katex_autorenderer.js"></script>
+<script src="/_sphinx/katex_autorenderer.js"></script>
 <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.min.css" />
-<link rel="stylesheet" type="text/css" href="/css/katex-math.css" />
 """  # noqa: E501
 
 
@@ -57,15 +59,6 @@ def parse_sphinx(input_dir, output_dir):
                 with open(os.path.join(output_path, fname), "w") as fout:
                     fout.write(out)
 
-    # update reference in JS file
-    with open(os.path.join(input_dir, "_static/searchtools.js"), "r") as js_file:
-        js = js_file.read()
-    js = js.replace(
-        "DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/'", "'_sphinx-sources/'"
-    )
-    with open(os.path.join(input_dir, "_static/searchtools.js"), "w") as js_file:
-        js_file.write(js)
-
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Strip HTML body from Sphinx docs.")
@@ -81,7 +74,7 @@ def parse_sphinx(input_dir, output_dir):
         "--output_dir",
         metavar="path",
         required=True,
-        help="Output directory in Docusaurus.",
+        help="Output directory in website.",
     )
     args = parser.parse_args()
     parse_sphinx(args.input_dir, args.output_dir)

sphinx/source/conf.py

@@ -115,6 +115,11 @@
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
 html_show_copyright = False
 
+# If true, the reST sources are included in the HTML build as _sources/name.
+# The default is True.
+# Uncomment the following line after sphinx 4.5.0 release
+# https://github.com/sphinx-doc/sphinx/issues/9456
+# html_copy_source = False
 
 # -- Options for HTMLHelp output ---------------------------------------------
 

website/siteConfig.js

@@ -73,7 +73,7 @@ const siteConfig = {
   ],
 
   // CSS sources to load
-  stylesheets: [`${baseUrl}css/code_block_buttons.css`],
+  stylesheets: [],
 
   // enable on-page navigation for the current documentation page
   onPageNav: "separate",