From 7508f6df0f44c8d04e5981fb83afd6ab4a7e343f Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 3 Sep 2024 08:42:55 -0700 Subject: [PATCH 001/199] [SPARK-47995][INFRA][PYTHON] Refresh testing image for pyarrow 17 ### What changes were proposed in this pull request? Refresh testing image for pyarrow 17 ### Why are the changes needed? currently the cached `pyarrow==15.0.2` is used in [CI](https://github.com/apache/spark/actions/runs/10674534002/job/29585233434), we need to test Spark with latest pyarrow ### Does this PR introduce _any_ user-facing change? No, infra only ### How was this patch tested? updated ci ### Was this patch authored or co-authored using generative AI tooling? no Closes #46232 Closes #47965 from zhengruifeng/infra_refresh_test_doc. Authored-by: Ruifeng Zheng Signed-off-by: Dongjoon Hyun --- .github/workflows/build_and_test.yml | 2 +- dev/infra/Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0bf7e57c364e4..6fdb8432472f9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -723,7 +723,7 @@ jobs: # See 'ipython_genutils' in SPARK-38517 # See 'docutils<0.18.0' in SPARK-39421 python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \ - ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' 'docutils<0.18.0' \ + ipython ipython_genutils sphinx_plotly_directive 'numpy==1.26.4' pyarrow pandas 'plotly>=4.8' 'docutils<0.18.0' \ 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.9.1' \ 'pandas-stubs==1.2.0.53' 'grpcio==1.62.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \ 'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index b01e3c50e28d3..ce47362999284 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image" # Overwrite this label to avoid exposing the underlying Ubuntu OS version label LABEL org.opencontainers.image.version="" -ENV FULL_REFRESH_DATE 20240318 +ENV FULL_REFRESH_DATE 20240903 ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_NONINTERACTIVE_SEEN true @@ -91,10 +91,10 @@ RUN mkdir -p /usr/local/pypy/pypy3.9 && \ ln -sf /usr/local/pypy/pypy3.9/bin/pypy /usr/local/bin/pypy3.8 && \ ln -sf /usr/local/pypy/pypy3.9/bin/pypy /usr/local/bin/pypy3 RUN curl -sS https://bootstrap.pypa.io/get-pip.py | pypy3 -RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas==2.2.2' scipy coverage matplotlib lxml +RUN pypy3 -m pip install 'numpy==1.26.4' 'six==1.16.0' 'pandas==2.2.2' scipy coverage matplotlib lxml -ARG BASIC_PIP_PKGS="numpy pyarrow>=15.0.0 six==1.16.0 pandas==2.2.2 scipy plotly>=4.8 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2" +ARG BASIC_PIP_PKGS="numpy==1.26.4 pyarrow>=15.0.0 six==1.16.0 pandas==2.2.2 scipy plotly>=4.8 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2" # Python deps for Spark Connect ARG CONNECT_PIP_PKGS="grpcio==1.62.0 grpcio-status==1.62.0 protobuf==4.25.1 googleapis-common-protos==1.56.4 graphviz==0.20.3" From 48f9cc7e716d7c3568049a80a9af7ca1b5c9ec01 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 3 Sep 2024 09:18:28 -0700 Subject: [PATCH 002/199] [SPARK-49494][DOCS] Use spark logos from spark-website in docs ### What changes were proposed in this pull request? Use spark logos from spark-website in docs and remove them from spark main repo ### Why are the changes needed? - reduce spark-website incremental size - fix some cases when `/img/spark-logo-rev.svg` is invalid ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? doc build, https://github.com/yaooqinn/spark/actions/runs/10676269598/artifacts/1884295142 ### Was this patch authored or co-authored using generative AI tooling? no Closes #47966 from yaooqinn/SPARK-49494. Authored-by: Kent Yao Signed-off-by: Dongjoon Hyun --- docs/_layouts/global.html | 2 +- docs/img/spark-logo-hd.png | Bin 11306 -> 0 bytes docs/img/spark-logo-rev.svg | 7 ------- docs/img/spark-logo-reverse.png | Bin 21547 -> 0 bytes python/docs/source/_static/spark-logo-dark.png | Bin 23555 -> 0 bytes python/docs/source/_static/spark-logo-light.png | Bin 18773 -> 0 bytes python/docs/source/conf.py | 6 +++--- 7 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 docs/img/spark-logo-hd.png delete mode 100644 docs/img/spark-logo-rev.svg delete mode 100644 docs/img/spark-logo-reverse.png delete mode 100644 python/docs/source/_static/spark-logo-dark.png delete mode 100644 python/docs/source/_static/spark-logo-light.png diff --git a/docs/_layouts/global.html b/docs/_layouts/global.html index 502113d11b77e..a85fd16451469 100755 --- a/docs/_layouts/global.html +++ b/docs/_layouts/global.html @@ -58,7 +58,7 @@