From ec046d28c8c7a2e7cb342b5d80c24165c2f7e423 Mon Sep 17 00:00:00 2001 From: Radu Carpa Date: Fri, 24 Feb 2023 10:17:28 +0100 Subject: [PATCH 1/2] servers: build alma9 server images To both support python3.6 and python3.9 in docker-entrypoint.sh, the rucio python path is automatically inferred and passed as a variable to j2 scripts. --- .github/workflows/docker-auto-build.yml | 3 +- server/alma9.Dockerfile | 57 +++++++++++++++++++++++++ server/docker-entrypoint.sh | 6 ++- server/gacl | 7 +++ server/rucio.conf.j2 | 22 +++++++++- 5 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 server/alma9.Dockerfile create mode 100644 server/gacl diff --git a/.github/workflows/docker-auto-build.yml b/.github/workflows/docker-auto-build.yml index fb78e60..62f80c4 100644 --- a/.github/workflows/docker-auto-build.yml +++ b/.github/workflows/docker-auto-build.yml @@ -13,6 +13,7 @@ jobs: context: [ 'clients, prepend-rucio, prepend-release, push-tagged, push-latest', 'server, prepend-rucio, prepend-release, push-tagged, push-latest', + 'server/alma9.Dockerfile, prepend-rucio, prepend-release, push-tagged, custom-tag=-alma9', 'daemons, prepend-rucio, prepend-release, push-tagged, push-latest', 'dev, prepend-rucio, prepend-release, push-tagged, push-latest', 'ui, prepend-rucio, prepend-release, push-tagged, push-latest', @@ -154,7 +155,7 @@ jobs: LATEST_TAG="$BASE_TAG:latest" fi - if [[ $CONTEXT =~ custom-tag=([a-zA-Z0-9]*) ]]; then + if [[ $CONTEXT =~ custom-tag=([-a-zA-Z0-9]*) ]]; then CUSTOM_TAG="${BASH_REMATCH[1]}" if [[ -z $IMAGE_TAG ]]; then IMAGE_TAG="${BASE_TAG}:${CUSTOM_TAG}" diff --git a/server/alma9.Dockerfile b/server/alma9.Dockerfile new file mode 100644 index 0000000..f42ea43 --- /dev/null +++ b/server/alma9.Dockerfile @@ -0,0 +1,57 @@ +# Copyright European Organization for Nuclear Research (CERN) 2017 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +FROM almalinux:9 + +ARG TAG + +WORKDIR /tmp + +RUN dnf install -y epel-release.noarch && \ + dnf upgrade -y && \ + dnf install -y \ + gridsite \ + libnsl \ + libaio \ + patch \ + procps-ng \ + python-pip \ + python-mod_wsgi \ + memcached && \ + dnf clean all && \ + rm -rf /var/cache/dnf +RUN rpm -i https://download.oracle.com/otn_software/linux/instantclient/1912000/oracle-instantclient19.12-basiclite-19.12.0.0.0-1.x86_64.rpm; \ + echo "/usr/lib/oracle/19/client64/lib" >/etc/ld.so.conf.d/oracle.conf; \ + ldconfig + +RUN python3 -m pip install --no-cache-dir --upgrade pip && \ + python3 -m pip install --no-cache-dir --upgrade setuptools +RUN python3 -m pip install --no-cache-dir --pre rucio[oracle,mysql,postgresql]==$TAG + +RUN python3 -m pip install --no-cache-dir j2cli +ADD gacl /etc/httpd/ +ADD rucio.config.default.cfg /tmp/ +ADD rucio.conf.j2 /tmp/ +ADD httpd.conf.j2 /tmp/ +ADD 00-mpm.conf.j2 /tmp/ +ADD docker-entrypoint.sh / +ADD robots.txt /var/www/html +RUN rm /etc/httpd/conf.d/zgridsite.conf \ + /etc/httpd/conf.d/welcome.conf \ + /etc/httpd/conf.d/userdir.conf \ + /etc/httpd/conf.d/ssl.conf +RUN mkdir -p /var/log/rucio/trace && chown apache:apache /var/log/rucio/trace +RUN mkdir -p /var/log/rucio/nongrid_trace && chown apache:apache /var/log/rucio/nongrid_trace + +RUN update-crypto-policies --set DEFAULT:SHA1 + +VOLUME /var/log/httpd +VOLUME /opt/rucio/etc + +EXPOSE 80 +EXPOSE 443 + +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh index 725f5fe..68f446e 100755 --- a/server/docker-entrypoint.sh +++ b/server/docker-entrypoint.sh @@ -28,7 +28,9 @@ if [ ! -z "$RUCIO_PRINT_CFG" ]; then echo "" fi -j2 /tmp/rucio.conf.j2 | sed '/^\s*$/d' > /etc/httpd/conf.d/rucio.conf +RUCIO_PYTHON_PATH=$(python3 -c "import os; import rucio; print(os.path.dirname(rucio.__file__))") + +(export RUCIO_PYTHON_PATH; j2 /tmp/rucio.conf.j2 | sed '/^\s*$/d' > /etc/httpd/conf.d/rucio.conf) /usr/bin/memcached -u memcached -p 11211 -m 128 -c 1024 & @@ -47,7 +49,7 @@ then for patchfile in /patch/* do echo "Apply patch ${patchfile}" - patch -p3 -d /usr/local/lib/python3.6/site-packages/rucio < $patchfile + patch -p3 -d "$RUCIO_PYTHON_PATH" < $patchfile done fi diff --git a/server/gacl b/server/gacl new file mode 100644 index 0000000..f3abc23 --- /dev/null +++ b/server/gacl @@ -0,0 +1,7 @@ + + + + + + + diff --git a/server/rucio.conf.j2 b/server/rucio.conf.j2 index 4a0cfd1..536a725 100644 --- a/server/rucio.conf.j2 +++ b/server/rucio.conf.j2 @@ -4,6 +4,9 @@ LoadModule ssl_module /usr/lib64/httpd/modules/mod_ssl.so {% else %} {% set listen_port = 80 %} {% endif %} +{% if RUCIO_HTTPD_GRID_SITE_ENABLED | default('False') == 'True' %} +LoadModule gridsite_module /usr/lib64/httpd/modules/mod_gridsite.so +{% endif %} LoadModule unique_id_module modules/mod_unique_id.so LoadModule wsgi_module /usr/lib64/httpd/modules/mod_wsgi.so Listen {{ listen_port }} @@ -84,7 +87,22 @@ CacheRoot /tmp {% if RUCIO_DEFINE_ALIASES|default('False') == 'True' %} Include /opt/rucio/etc/aliases.conf {% else %} - WSGIScriptAlias / /usr/local/lib/python3.6/site-packages/rucio/web/rest/flaskapi/v1/main.py process-group=rucio application-group=rucio + WSGIScriptAlias / {{ RUCIO_PYTHON_PATH }}/web/rest/flaskapi/v1/main.py process-group=rucio application-group=rucio +{% endif %} + +{% if RUCIO_HTTPD_PROXY_PROTOCOL_ENABLED | default('False') == 'True' %} + RemoteIPProxyProtocol On + RemoteIPProxyProtocolExceptions 127.0.0.1 ::1 {{ RUCIO_HTTPD_PROXY_PROTOCOL_EXCEPTIONS }} +{% endif %} + +{% if RUCIO_HTTPD_GRID_SITE_ENABLED | default('False') == 'True' %} + + GridSiteIndexes {{ RUCIO_HTTPD_GRID_SITE_INDEXES | default('on') }} + GridSiteAuth {{ RUCIO_HTTPD_GRID_SITE_AUTH | default('on') }} + GridSiteGSIProxyLimit {{ RUCIO_HTTPD_GRID_SITE_GSI_PROXY_LIMIT | default('16') }} + GridSiteEnvs {{ RUCIO_HTTPD_GRID_SITE_ENVS | default('on') }} + GridSiteACLPath {{ RUCIO_HTTPD_GRID_SITE_ACL_PATH | default('/etc/httpd/gacl') }} + {% endif %} @@ -93,6 +111,6 @@ Listen {{ RUCIO_METRICS_PORT }} {{ common_virtual_host_config(port=RUCIO_METRICS_PORT, enable_ssl=false) }} - WSGIScriptAlias /metrics /usr/local/lib/python3.6/site-packages/rucio/web/rest/metrics.py process-group=rucio application-group=rucio + WSGIScriptAlias /metrics {{ RUCIO_PYTHON_PATH }}/web/rest/metrics.py process-group=rucio application-group=rucio {% endif %} From 8ab0c877663750ec90ffcfc72b293eb35d21c56d Mon Sep 17 00:00:00 2001 From: Radu Carpa Date: Thu, 9 Mar 2023 11:08:46 +0100 Subject: [PATCH 2/2] daemons: build alma9 images similarly to how it is done in servers in previous commit. --- .github/workflows/docker-auto-build.yml | 1 + daemons/alma9.Dockerfile | 52 +++++++++++++++++++++++++ daemons/start-daemon.sh | 4 +- 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 daemons/alma9.Dockerfile diff --git a/.github/workflows/docker-auto-build.yml b/.github/workflows/docker-auto-build.yml index 62f80c4..166a5c2 100644 --- a/.github/workflows/docker-auto-build.yml +++ b/.github/workflows/docker-auto-build.yml @@ -15,6 +15,7 @@ jobs: 'server, prepend-rucio, prepend-release, push-tagged, push-latest', 'server/alma9.Dockerfile, prepend-rucio, prepend-release, push-tagged, custom-tag=-alma9', 'daemons, prepend-rucio, prepend-release, push-tagged, push-latest', + 'daemons/alma9.Dockerfile, prepend-rucio, prepend-release, push-tagged, custom-tag=-alma9', 'dev, prepend-rucio, prepend-release, push-tagged, push-latest', 'ui, prepend-rucio, prepend-release, push-tagged, push-latest', 'init, prepend-rucio, prepend-release, push-tagged, push-latest', diff --git a/daemons/alma9.Dockerfile b/daemons/alma9.Dockerfile new file mode 100644 index 0000000..a45cd3f --- /dev/null +++ b/daemons/alma9.Dockerfile @@ -0,0 +1,52 @@ +# Copyright European Organization for Nuclear Research (CERN) 2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +FROM almalinux:9 + +ARG TAG + +WORKDIR /tmp + +RUN dnf install -y epel-release.noarch && \ + dnf upgrade -y && \ + dnf install -y \ + fetch-crl \ + gfal2-plugin-file \ + gfal2-plugin-gridftp \ + gfal2-plugin-http \ + gfal2-plugin-srm \ + gfal2-plugin-xrootd \ + libnsl \ + libaio \ + patch \ + python-gfal2 \ + procps-ng \ + python-pip \ + python-mod_wsgi \ + sendmail \ + sendmail-cf \ + memcached \ + xrootd-client && \ + dnf clean all && \ + rm -rf /var/cache/dnf +RUN rpm -i https://download.oracle.com/otn_software/linux/instantclient/1912000/oracle-instantclient19.12-basiclite-19.12.0.0.0-1.x86_64.rpm; \ + echo "/usr/lib/oracle/19/client64/lib" >/etc/ld.so.conf.d/oracle.conf; \ + ldconfig + +RUN python3 -m pip install --no-cache-dir --upgrade pip && \ + python3 -m pip install --no-cache-dir --upgrade setuptools +RUN python3 -m pip install --no-cache-dir --pre rucio[oracle,mysql,postgresql]==$TAG + +RUN python3 -m pip install --no-cache-dir j2cli +ADD rucio.config.default.cfg /tmp/ +ADD start-daemon.sh / + +RUN mkdir /var/log/rucio + +VOLUME /var/log/rucio +VOLUME /opt/rucio/etc + +ENTRYPOINT ["/start-daemon.sh"] diff --git a/daemons/start-daemon.sh b/daemons/start-daemon.sh index 333a978..b0366fb 100755 --- a/daemons/start-daemon.sh +++ b/daemons/start-daemon.sh @@ -16,6 +16,8 @@ if [ ! -z "$RUCIO_PRINT_CFG" ]; then echo "" fi +RUCIO_PYTHON_PATH=$(python3 -c "import os; import rucio; print(os.path.dirname(rucio.__file__))") + /usr/bin/memcached -u memcached -p 11211 -m 64 -c 1024 & if [ "$RUCIO_DAEMON" == "hermes" ] @@ -30,7 +32,7 @@ then for patchfile in /patch/* do echo "Apply patch ${patchfile}" - patch -p3 -d /usr/local/lib/python3.6/site-packages/rucio < $patchfile + patch -p3 -d "$RUCIO_PYTHON_PATH" < $patchfile done fi