diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 7ffeea87ff..637f51d74b 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -98,7 +98,7 @@ jobs: shell: bash run: | docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose-storage-externalmetadata.yml pull + docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata pull i=0; until [ "$i" -ge 3 ]; do IMAGES=$(echo '${{ toJson(matrix.cfg) }}' | $GITHUB_WORKSPACE/dev/rucio/tools/test/build_images.py --output list \ --cache-repo ghcr.io/${{ github.repository }} --branch "${{ needs.setup.outputs.branch }}" \ @@ -117,14 +117,14 @@ jobs: run: | docker image ls sed -i 's;image: docker.io/rucio/rucio-dev.*;image: ${{ fromJSON(steps.images.outputs.images)[0] }};' \ - $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose-storage-externalmetadata.yml + $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml - name: Start containers - run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose-storage-externalmetadata.yml up -d + run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata up -d - name: Initialize tests shell: bash run: | docker exec -t dev_rucio_1 cp etc/rse-accounts.cfg.template etc/rse-accounts.cfg - docker exec -t dev_rucio_1 tools/run_tests_docker.sh -ir + docker exec -t dev_rucio_1 tools/run_tests.sh -ir - name: File Upload/Download Test run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rucio_server.py - name: UploadClient Test @@ -173,4 +173,4 @@ jobs: - name: Test external metadata plugin (postgres) run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaExternalPostgresJSON - name: Stop containers - run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose-storage-externalmetadata.yml down + run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata down diff --git a/etc/docker/dev/README.rst b/etc/docker/dev/README.rst index c70859fce9..dae481c26a 100644 --- a/etc/docker/dev/README.rst +++ b/etc/docker/dev/README.rst @@ -49,11 +49,11 @@ Finally, you can jump into the container with:: To verify that everything is in order, you can now either run the full unit tests or only set up the database. Running the full testing suite takes ~10 minutes:: - tools/run_tests_docker.sh + tools/run_tests.sh Alternatively, you can bootstrap the test environment once with the `-i` option and then selectively or repeatedly run test case modules, test case groups, or even single test cases, for example:: - tools/run_tests_docker.sh -i + tools/run_tests.sh -i tools/pytest.sh tests/test_replica.py tools/pytest.sh -vvv tests/test_replica.py::TestReplicaCore tools/pytest.sh -vvv --full-trace tests/test_replica.py::TestReplicaCore::test_delete_replicas @@ -63,13 +63,13 @@ Using the environment including storage Again run the containers using docker-compose:: - docker-compose --file etc/docker/dev/docker-compose-storage.yml up -d + docker-compose --file etc/docker/dev/docker-compose.yml --profile storage up -d This should show you a few more running containers: the Rucio server, the PostgreSQL database, FTS and its associated MySQL database, the Graphite monitoring, and three XrootD storage servers. With this container you can upload and download data to/from the storage and submit data transfers. To set this up, add the `-r` option to the setup.:: - tools/run_tests_docker.sh -ir + tools/run_tests.sh -ir This creates a few random files and uploads them, creates a few datasets and containers, and requests a replication rule for the container, which starts in state REPLICATING. To demonstrate the transfer capability, the daemons can be run in single-execution mode in order::: @@ -89,14 +89,14 @@ Using the environment including monitoring Again run the containers using docker-compose:: - docker-compose --file etc/docker/dev/docker-compose-storage-monit.yml up -d + docker-compose --file etc/docker/dev/docker-compose.yml --profile storage --profile monitoring up -d Now you will have the same containers as before plus a full monitoring stack with Logstash, Elasticsearch, Kibana and Grafana. To create some events and write them to Elasticsearch first run again the tests as before:: - tools/run_tests_docker.sh -ir + tools/run_tests.sh -ir Then you will have to run the transfer daemons (conveyor-\*) and messaging daemon (hermes) to send the events to ActiveMQ. There a script for that which repeats these daemons in single execution mode from the section in a loop:: @@ -210,7 +210,7 @@ Start the daemons Daemons are not running in the docker environment, but all daemons support single-execution mode with the --run-once argument. Reset the system first with:: - tools/run_tests_docker.sh -ir + tools/run_tests.sh -ir Some files are created. Let's add them to a new dataset:: diff --git a/etc/docker/dev/docker-compose-storage-alldb.yml b/etc/docker/dev/docker-compose-storage-alldb.yml deleted file mode 100644 index d91d72809b..0000000000 --- a/etc/docker/dev/docker-compose-storage-alldb.yml +++ /dev/null @@ -1,186 +0,0 @@ -version: "3" -services: - rucioclient: - image: docker.io/rucio/rucio-dev:latest-alma9 - command: ["sleep", "infinity"] - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 - rucio: - image: docker.io/rucio/rucio-dev:latest-alma9 - ports: - - "127.0.0.1:8443:443" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 - ruciomy8: - image: docker.io/mysql:8 - ports: - - "127.0.0.1:3308:3308" - environment: - - MYSQL_USER=rucio - - MYSQL_PASSWORD=rucio - - MYSQL_ROOT_PASSWORD=rucio - - MYSQL_DATABASE=rucio - - MYSQL_TCP_PORT=3308 - oracle: - image: docker.io/gvenzl/oracle-xe:21-slim - ports: - - "127.0.0.1:1521:1521" - environment: - - ORACLE_PASSWORD=rucio - - ORACLE_ALLOW_REMOTE=true - - ORACLE_DISABLE_ASYNCH_IO=true - - processes=1000 - - sessions=1105 - - transactions=1215 - ruciodb: - image: docker.io/postgres:14 - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_USER=rucio - - POSTGRES_DB=rucio - - POSTGRES_PASSWORD=secret - command: ["-c", "fsync=off","-c", "synchronous_commit=off","-c", "full_page_writes=off"] - graphite: - image: docker.io/graphiteapp/graphite-statsd - ports: - - "127.0.0.1:8080:80" - fts: - image: docker.io/rucio/fts - ports: - - "127.0.0.1:8446:8446" - - "127.0.0.1:8449:8449" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_fts.pem:/etc/grid-security/hostcert.pem:Z - - ../../certs/hostcert_fts.key.pem:/etc/grid-security/hostkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - ftsdb: - image: docker.io/mysql:8 - ports: - - "127.0.0.1:3306:3306" - command: --default-authentication-plugin=mysql_native_password - environment: - - MYSQL_USER=fts - - MYSQL_PASSWORD=fts - - MYSQL_ROOT_PASSWORD=fts - - MYSQL_DATABASE=fts - xrd1: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1094:1094" - environment: - - XRDPORT=1094 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd1.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd1.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd2: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1095:1095" - environment: - - XRDPORT=1095 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd2.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd2.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd3: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1096:1096" - environment: - - XRDPORT=1096 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd3.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd3.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd4: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1097:1097" - environment: - - XRDPORT=1097 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd4.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd4.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - minio: - image: docker.io/minio/minio - ports: - - "127.0.0.1:9000:9000" - environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - volumes: - - ../../certs/hostcert_minio.pem:/root/.minio/certs/public.crt:Z - - ../../certs/hostcert_minio.key.pem:/root/.minio/certs/private.key:Z - command: ["server", "/data"] - activemq: - image: docker.io/webcenter/activemq:latest - ports: - - "127.0.0.1:61613:61613" - - "127.0.0.1:8161:8161" - environment: - - ACTIVEMQ_CONFIG_NAME=activemq - - ACTIVEMQ_CONFIG_DEFAULTACCOUNT=false - - ACTIVEMQ_USERS_fts=supersecret - - ACTIVEMQ_GROUPS_writes=fts - - ACTIVEMQ_USERS_receiver=supersecret - - ACTIVEMQ_GROUPS_reads=receiver - - ACTIVEMQ_CONFIG_SCHEDULERENABLED=true - ssh1: - image: docker.io/rucio/ssh - ports: - - "127.0.0.1:2222:22" - volumes: - - ../../certs/ssh/ruciouser_sshkey.pub:/tmp/sshkey.pub:Z diff --git a/etc/docker/dev/docker-compose-storage-externalmetadata.yml b/etc/docker/dev/docker-compose-storage-externalmetadata.yml deleted file mode 100644 index 84e4dcace2..0000000000 --- a/etc/docker/dev/docker-compose-storage-externalmetadata.yml +++ /dev/null @@ -1,195 +0,0 @@ -version: "3.5" -services: - rucioclient: - image: docker.io/rucio/rucio-dev:latest-alma9 - command: ["sleep", "infinity"] - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 - rucio: - image: docker.io/rucio/rucio-dev:latest-alma9 - ports: - - "127.0.0.1:8443:443" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres11 - ruciodb: - image: docker.io/postgres:11 - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_USER=rucio - - POSTGRES_DB=rucio - - POSTGRES_PASSWORD=secret - command: ["-c", "fsync=off","-c", "synchronous_commit=off","-c", "full_page_writes=off"] - graphite: - image: docker.io/graphiteapp/graphite-statsd - ports: - - "127.0.0.1:8080:80" - fts: - image: docker.io/rucio/fts - ports: - - "127.0.0.1:8446:8446" - - "127.0.0.1:8449:8449" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_fts.pem:/etc/grid-security/hostcert.pem:Z - - ../../certs/hostcert_fts.key.pem:/etc/grid-security/hostkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - ftsdb: - image: docker.io/mysql:5 - ports: - - "127.0.0.1:3306:3306" - environment: - - MYSQL_USER=fts - - MYSQL_PASSWORD=fts - - MYSQL_ROOT_PASSWORD=fts - - MYSQL_DATABASE=fts - xrd1: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1094:1094" - environment: - - XRDPORT=1094 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd1.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd1.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd2: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1095:1095" - environment: - - XRDPORT=1095 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd2.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd2.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd3: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1096:1096" - environment: - - XRDPORT=1096 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd3.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd3.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd4: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1097:1097" - environment: - - XRDPORT=1097 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd4.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd4.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - minio: - image: docker.io/minio/minio - ports: - - "127.0.0.1:9000:9000" - environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - volumes: - - ../../certs/hostcert_minio.pem:/root/.minio/certs/public.crt:Z - - ../../certs/hostcert_minio.key.pem:/root/.minio/certs/private.key:Z - command: ["server", "/data"] - activemq: - image: docker.io/webcenter/activemq:latest - ports: - - "127.0.0.1:61613:61613" - - "127.0.0.1:8161:8161" - environment: - - ACTIVEMQ_CONFIG_NAME=activemq - - ACTIVEMQ_CONFIG_DEFAULTACCOUNT=false - - ACTIVEMQ_USERS_fts=supersecret - - ACTIVEMQ_GROUPS_writes=fts - - ACTIVEMQ_USERS_receiver=supersecret - - ACTIVEMQ_GROUPS_reads=receiver - - ACTIVEMQ_CONFIG_SCHEDULERENABLED=true - ssh1: - image: docker.io/rucio/ssh - ports: - - "127.0.0.1:2222:22" - volumes: - - ../../certs/ssh/ruciouser_sshkey.pub:/tmp/sshkey.pub:Z - mongo: - image: docker.io/mongo:5.0 - ports: - - "127.0.0.1:27017:27017" - influxdb: - image: docker.io/influxdb:latest - ports: - - "127.0.0.1:8086:8086" - environment: - - DOCKER_INFLUXDB_INIT_MODE=setup - - DOCKER_INFLUXDB_INIT_USERNAME=myusername - - DOCKER_INFLUXDB_INIT_PASSWORD=passwordpasswordpassword - - DOCKER_INFLUXDB_INIT_ORG=rucio - - DOCKER_INFLUXDB_INIT_BUCKET=rucio - - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=mytoken - elasticsearch: - image: docker.io/elasticsearch:7.4.0 - ports: - - "127.0.0.1:9200:9200" - - "127.0.0.1:9300:9300" - environment: - - discovery.type=single-node - postgres: - image: docker.io/postgres:14 - ports: - - "127.0.0.1:5433:5433" - environment: - - POSTGRES_USER=rucio - - POSTGRES_DB=metadata - - POSTGRES_PASSWORD=secret - command: ["-p", "5433"] diff --git a/etc/docker/dev/docker-compose-storage-iam.yml b/etc/docker/dev/docker-compose-storage-iam.yml deleted file mode 100644 index fd468229a8..0000000000 --- a/etc/docker/dev/docker-compose-storage-iam.yml +++ /dev/null @@ -1,200 +0,0 @@ -version: "3" -services: - rucioclient: - image: docker.io/rucio/rucio-dev:latest-alma9 - command: ["sleep", "infinity"] - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 - rucio: - image: docker.io/rucio/rucio-dev:latest-alma9 - ports: - - "127.0.0.1:8443:443" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 - ruciodb: - image: docker.io/postgres:14 - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_USER=rucio - - POSTGRES_DB=rucio - - POSTGRES_PASSWORD=secret - command: ["-c", "fsync=off","-c", "synchronous_commit=off","-c", "full_page_writes=off"] - graphite: - image: docker.io/graphiteapp/graphite-statsd - ports: - - "127.0.0.1:8080:80" - fts: - image: docker.io/rucio/fts - ports: - - "127.0.0.1:8446:8446" - - "127.0.0.1:8449:8449" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_fts.pem:/etc/grid-security/hostcert.pem:Z - - ../../certs/hostcert_fts.key.pem:/etc/grid-security/hostkey.pem:Z - ftsdb: - image: docker.io/mysql:8 - ports: - - "127.0.0.1:3306:3306" - command: --default-authentication-plugin=mysql_native_password - environment: - - MYSQL_USER=fts - - MYSQL_PASSWORD=fts - - MYSQL_ROOT_PASSWORD=fts - - MYSQL_DATABASE=fts - xrd1: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1094:1094" - environment: - - XRDPORT=1094 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd1.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd1.key.pem:/tmp/xrdkey.pem:Z - xrd2: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1095:1095" - environment: - - XRDPORT=1095 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd2.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd2.key.pem:/tmp/xrdkey.pem:Z - xrd3: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1096:1096" - environment: - - XRDPORT=1096 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd3.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd3.key.pem:/tmp/xrdkey.pem:Z - xrd4: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1097:1097" - environment: - - XRDPORT=1097 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd4.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd4.key.pem:/tmp/xrdkey.pem:Z - minio: - image: docker.io/minio/minio - ports: - - "127.0.0.1:9000:9000" - environment: - - MINIO_ACCESS_KEY=admin - - MINIO_SECRET_KEY=password - volumes: - - ../../certs/hostcert_minio.pem:/root/.minio/certs/public.crt:Z - - ../../certs/hostcert_minio.key.pem:/root/.minio/certs/private.key:Z - command: ["server", "/data"] - activemq: - image: docker.io/webcenter/activemq:latest - ports: - - "127.0.0.1:61613:61613" - environment: - - ACTIVEMQ_CONFIG_NAME=activemq - - ACTIVEMQ_CONFIG_DEFAULTACCOUNT=false - - ACTIVEMQ_USERS_fts=supersecret - - ACTIVEMQ_GROUPS_writes=fts - - ACTIVEMQ_USERS_receiver=supersecret - - ACTIVEMQ_GROUPS_reads=receiver - - ACTIVEMQ_CONFIG_SCHEDULERENABLED=true - ssh1: - image: docker.io/rucio/ssh - ports: - - "127.0.0.1:2222:22" - volumes: - - ../../certs/ssh/ruciouser_sshkey.pub:/tmp/sshkey.pub:Z - db-iam: - image: mariadb:10.11 - environment: - - TZ=Europe/Paris - - MYSQL_ROOT_PASSWORD=supersecret - - MYSQL_USER=iam - - MYSQL_PASSWORD=secret - - MYSQL_DATABASE=iam_db - ports: - - "127.0.0.1:3307:3306" - nginx-iam: - image: nginx - dns_search: cern.ch - environment: - TZ: Europe/Paris - NGINX_HOST: iam - NGINX_PORT: 443 - ports: - - "127.0.0.1:9443:443" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - # - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - # - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - /etc/grid-security/:/etc/grid-security/ - - /dev/urandom:/dev/random - - ../../iam-assets/iam.conf:/etc/nginx/conf.d/default.conf:ro - iam: - image: indigoiam/iam-login-service:v1.8.2 - volumes: - - ../../iam-assets/keystore.jwks:/keystore.jwks:ro - environment: - - IAM_JAVA_OPTS=-Djava.security.egd=file:/dev/urandom -Dspring.profiles.active=prod,oidc,cern,registration,wlcg-scopes -agentlib:jdwp=transport=dt_socket,server=y,address=1044,suspend=n -Dlogging.file.name=/var/log/iam/iam.log - - IAM_HOST= - - IAM_PORT=8090 - - IAM_BASE_URL=https:// - - IAM_ISSUER=https:// - - IAM_FORWARD_HEADERS_STRATEGY=native - - IAM_KEY_STORE_LOCATION=file:/keystore.jwks - - IAM_JWK_CACHE_LIFETIME=21600 - # - IAM_X509_TRUST_ANCHORS_DIR=/etc/grid-security/certificates - # - IAM_X509_TRUST_ANCHORS_REFRESH=14400 - - IAM_TOMCAT_ACCESS_LOG_ENABLED=false - - IAM_TOMCAT_ACCESS_LOG_DIRECTORY=/tmp - - IAM_ACTUATOR_USER_USERNAME=user - - IAM_ACTUATOR_USER_PASSWORD=secret - - IAM_LOCAL_RESOURCES_ENABLE=true - - IAM_LOCAL_RESOURCES_LOCATION=file:/indigo-iam/local-resources - - IAM_ORGANISATION_NAME=rucio-dc - - IAM_TOPBAR_TITLE="INDIGO IAM for rucio-dc" - - IAM_DB_HOST= - - IAM_DB_PORT=3307 - - IAM_DB_NAME=iam_db - - IAM_DB_USERNAME=iam - - IAM_DB_PASSWORD=secret - ports: - - "127.0.0.1:8090:8090" \ No newline at end of file diff --git a/etc/docker/dev/docker-compose-storage-monit.yml b/etc/docker/dev/docker-compose-storage-monit.yml deleted file mode 100644 index d262544b07..0000000000 --- a/etc/docker/dev/docker-compose-storage-monit.yml +++ /dev/null @@ -1,190 +0,0 @@ -version: "3" -services: - rucioclient: - image: docker.io/rucio/rucio-dev:latest-alma9 - command: ["sleep", "infinity"] - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 - rucio: - image: docker.io/rucio/rucio-dev:latest-alma9 - ports: - - "127.0.0.1:8443:443" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 - command: ["/monit-entrypoint.sh"] - ruciodb: - image: docker.io/postgres:14 - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_USER=rucio - - POSTGRES_DB=rucio - - POSTGRES_PASSWORD=secret - command: ["-c", "fsync=off","-c", "synchronous_commit=off","-c", "full_page_writes=off"] - graphite: - image: docker.io/graphiteapp/graphite-statsd - ports: - - "127.0.0.1:8080:80" - fts: - image: docker.io/rucio/fts - ports: - - "127.0.0.1:8446:8446" - - "127.0.0.1:8449:8449" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_fts.pem:/etc/grid-security/hostcert.pem:Z - - ../../certs/hostcert_fts.key.pem:/etc/grid-security/hostkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - ftsdb: - image: docker.io/mysql:8 - ports: - - "127.0.0.1:3306:3306" - command: --default-authentication-plugin=mysql_native_password - environment: - - MYSQL_USER=fts - - MYSQL_PASSWORD=fts - - MYSQL_ROOT_PASSWORD=fts - - MYSQL_DATABASE=fts - xrd1: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1094:1094" - environment: - - XRDPORT=1094 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd1.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd1.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd2: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1095:1095" - environment: - - XRDPORT=1095 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd2.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd2.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd3: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1096:1096" - environment: - - XRDPORT=1096 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd3.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd3.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd4: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1097:1097" - environment: - - XRDPORT=1097 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd4.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd4.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - minio: - image: docker.io/minio/minio - ports: - - "127.0.0.1:9000:9000" - environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - volumes: - - ../../certs/hostcert_minio.pem:/root/.minio/certs/public.crt:Z - - ../../certs/hostcert_minio.key.pem:/root/.minio/certs/private.key:Z - command: ["server", "/data"] - ssh1: - image: docker.io/rucio/ssh - ports: - - "127.0.0.1:2222:22" - volumes: - - ../../certs/ssh/ruciouser_sshkey.pub:/tmp/sshkey.pub:Z - activemq: - image: docker.io/webcenter/activemq:latest - ports: - - "127.0.0.1:61613:61613" - - "127.0.0.1:8161:8161" - environment: - - ACTIVEMQ_CONFIG_NAME=activemq - - ACTIVEMQ_CONFIG_DEFAULTACCOUNT=false - - ACTIVEMQ_USERS_fts=supersecret - - ACTIVEMQ_USERS_hermes=supersecret - - ACTIVEMQ_USERS_logstash=supersecret - - ACTIVEMQ_USERS_receiver=supersecret - - ACTIVEMQ_GROUPS_writes=hermes,fts - - ACTIVEMQ_GROUPS_reads=logstash,receiver - - ACTIVEMQ_CONFIG_SCHEDULERENABLED=true - elasticsearch: - image: docker.io/elasticsearch:7.4.0 - ports: - - "127.0.0.1:9200:9200" - - "127.0.0.1:9300:9300" - environment: - - discovery.type=single-node - logstash: - image: docker.elastic.co/logstash/logstash-oss:7.3.2 - ports: - - "127.0.0.1:5044:5044" - command: bash -c "logstash-plugin install logstash-input-stomp ; /usr/local/bin/docker-entrypoint" - volumes: - - ./pipeline.conf:/usr/share/logstash/pipeline/pipeline.conf:Z - kibana: - image: docker.io/kibana:7.4.0 - ports: - - "127.0.0.1:5601:5601" - grafana: - image: docker.io/grafana/grafana:latest - ports: - - "127.0.0.1:3000:3000" diff --git a/etc/docker/dev/docker-compose-storage.yml b/etc/docker/dev/docker-compose-storage.yml deleted file mode 100644 index 478cfea58c..0000000000 --- a/etc/docker/dev/docker-compose-storage.yml +++ /dev/null @@ -1,164 +0,0 @@ -version: "3" -services: - rucioclient: - image: docker.io/rucio/rucio-dev:latest-alma9 - command: ["sleep", "infinity"] - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 - rucio: - image: docker.io/rucio/rucio-dev:latest-alma9 - ports: - - "127.0.0.1:8443:443" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z - - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z - - ../../certs/ssh/ruciouser_sshkey.pub:/root/.ssh/ruciouser_sshkey.pub:z - - ../../certs/ssh/ruciouser_sshkey:/root/.ssh/ruciouser_sshkey:z - - ../../../tools:/opt/rucio/tools:Z - - ../../../bin:/opt/rucio/bin:Z - - ../../../lib:/opt/rucio/lib:Z - - ../../../tests:/opt/rucio/tests:Z - environment: - - X509_USER_CERT=/opt/rucio/etc/usercert.pem - - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 - ruciodb: - image: docker.io/postgres:14 - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_USER=rucio - - POSTGRES_DB=rucio - - POSTGRES_PASSWORD=secret - command: ["-c", "fsync=off","-c", "synchronous_commit=off","-c", "full_page_writes=off"] - graphite: - image: docker.io/graphiteapp/graphite-statsd - ports: - - "127.0.0.1:8080:80" - fts: - image: docker.io/rucio/fts - ports: - - "127.0.0.1:8446:8446" - - "127.0.0.1:8449:8449" - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_fts.pem:/etc/grid-security/hostcert.pem:Z - - ../../certs/hostcert_fts.key.pem:/etc/grid-security/hostkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - ftsdb: - image: docker.io/mysql:8 - ports: - - "127.0.0.1:3306:3306" - command: --default-authentication-plugin=mysql_native_password - environment: - - MYSQL_USER=fts - - MYSQL_PASSWORD=fts - - MYSQL_ROOT_PASSWORD=fts - - MYSQL_DATABASE=fts - xrd1: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1094:1094" - environment: - - XRDPORT=1094 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd1.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd1.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd2: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1095:1095" - environment: - - XRDPORT=1095 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd2.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd2.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd3: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1096:1096" - environment: - - XRDPORT=1096 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd3.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd3.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - xrd4: - image: docker.io/rucio/xrootd - ports: - - "127.0.0.1:1097:1097" - environment: - - XRDPORT=1097 - volumes: - - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - - ../../certs/hostcert_xrd4.pem:/tmp/xrdcert.pem:Z - - ../../certs/hostcert_xrd4.key.pem:/tmp/xrdkey.pem:Z - ulimits: - nofile: - soft: 10240 - hard: 10240 - minio: - image: docker.io/minio/minio - ports: - - "127.0.0.1:9000:9000" - environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - volumes: - - ../../certs/hostcert_minio.pem:/root/.minio/certs/public.crt:Z - - ../../certs/hostcert_minio.key.pem:/root/.minio/certs/private.key:Z - command: ["server", "/data"] - activemq: - image: docker.io/webcenter/activemq:latest - ports: - - "127.0.0.1:61613:61613" - environment: - - ACTIVEMQ_CONFIG_NAME=activemq - - ACTIVEMQ_CONFIG_DEFAULTACCOUNT=false - - ACTIVEMQ_USERS_fts=supersecret - - ACTIVEMQ_GROUPS_writes=fts - - ACTIVEMQ_USERS_receiver=supersecret - - ACTIVEMQ_GROUPS_reads=receiver - - ACTIVEMQ_CONFIG_SCHEDULERENABLED=true - ssh1: - image: docker.io/rucio/ssh - ports: - - "127.0.0.1:2222:22" - volumes: - - ../../certs/ssh/ruciouser_sshkey.pub:/tmp/sshkey.pub:Z diff --git a/etc/docker/dev/docker-compose.autotests.yml b/etc/docker/dev/docker-compose.autotests.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/etc/docker/dev/docker-compose.ports.yml b/etc/docker/dev/docker-compose.ports.yml new file mode 100644 index 0000000000..a4b0edd04d --- /dev/null +++ b/etc/docker/dev/docker-compose.ports.yml @@ -0,0 +1,75 @@ +services: + rucio: + ports: + - "127.0.0.1:8443:443" + ruciodb: + ports: + - "127.0.0.1:5432:5432" + graphite: + ports: + - "127.0.0.1:8080:80" + influxdb: + ports: + - "127.0.0.1:8086:8086" + elasticsearch: + ports: + - "127.0.0.1:9200:9200" + - "127.0.0.1:9300:9300" + activemq: + ports: + - "127.0.0.1:61613:61613" + mysql8: + ports: + - "127.0.0.1:3308:3308" + oracle: + ports: + - "127.0.0.1:1521:1521" + fts: + ports: + - "127.0.0.1:8446:8446" + - "127.0.0.1:8449:8449" + ftsdb: + ports: + - "127.0.0.1:3306:3306" + xrd1: + ports: + - "127.0.0.1:1094:1094" + xrd2: + ports: + - "127.0.0.1:1095:1095" + xrd3: + ports: + - "127.0.0.1:1096:1096" + xrd4: + ports: + - "127.0.0.1:1097:1097" + minio: + ports: + - "127.0.0.1:9000:9000" + ssh1: + ports: + - "127.0.0.1:2222:22" + mongo: + ports: + - "127.0.0.1:27017:27017" + postgres: + ports: + - "127.0.0.1:5433:5433" + logstash: + ports: + - "127.0.0.1:5044:5044" + kibana: + ports: + - "127.0.0.1:5601:5601" + grafana: + ports: + - "127.0.0.1:3000:3000" + db-iam: + ports: + - "127.0.0.1:3307:3306" + nginx-iam: + ports: + - "127.0.0.1:9443:443" + iam: + ports: + - "127.0.0.1:8090:8090" diff --git a/etc/docker/dev/docker-compose.yml b/etc/docker/dev/docker-compose.yml index 397e6aa2d5..c6324fa004 100644 --- a/etc/docker/dev/docker-compose.yml +++ b/etc/docker/dev/docker-compose.yml @@ -2,11 +2,15 @@ version: "3" services: rucioclient: image: docker.io/rucio/rucio-dev:latest-alma9 + entrypoint: ["/rucio_source/etc/docker/dev/rucio_entrypoint.sh"] command: ["sleep", "infinity"] + profiles: + - client volumes: - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z + - ../../certs/rucio_ca.pem:/opt/rucio/etc/rucio_ca.pem:z - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z @@ -16,18 +20,21 @@ services: - ../../../bin:/opt/rucio/bin:Z - ../../../lib:/opt/rucio/lib:Z - ../../../tests:/opt/rucio/tests:Z + - ../../../:/rucio_source:ro environment: + - RUCIO_SOURCE_DIR=/rucio_source - X509_USER_CERT=/opt/rucio/etc/usercert.pem - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 + - RDBMS rucio: image: docker.io/rucio/rucio-dev:latest-alma9 - ports: - - "127.0.0.1:8443:443" + entrypoint: ["/rucio_source/etc/docker/dev/rucio_entrypoint.sh"] + command: ["httpd","-D","FOREGROUND"] volumes: - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z + - ../../certs/rucio_ca.pem:/opt/rucio/etc/rucio_ca.pem:z - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z @@ -37,14 +44,14 @@ services: - ../../../bin:/opt/rucio/bin:Z - ../../../lib:/opt/rucio/lib:Z - ../../../tests:/opt/rucio/tests:Z + - ../../../:/rucio_source:ro environment: + - RUCIO_SOURCE_DIR=/rucio_source - X509_USER_CERT=/opt/rucio/etc/usercert.pem - X509_USER_KEY=/opt/rucio/etc/userkey.pem - - RDBMS=postgres14 + - RDBMS ruciodb: image: docker.io/postgres:14 - ports: - - "127.0.0.1:5432:5432" environment: - POSTGRES_USER=rucio - POSTGRES_DB=rucio @@ -52,12 +59,8 @@ services: command: ["-c", "fsync=off","-c", "synchronous_commit=off","-c", "full_page_writes=off"] graphite: image: docker.io/graphiteapp/graphite-statsd - ports: - - "127.0.0.1:8080:80" influxdb: image: docker.io/influxdb:latest - ports: - - "127.0.0.1:8086:8086" environment: - DOCKER_INFLUXDB_INIT_MODE=setup - DOCKER_INFLUXDB_INIT_USERNAME=myusername @@ -67,16 +70,10 @@ services: - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=mytoken elasticsearch: image: docker.io/elasticsearch:7.4.0 - ports: - - "127.0.0.1:9200:9200" - - "127.0.0.1:9300:9300" environment: - discovery.type=single-node activemq: image: docker.io/webcenter/activemq:latest - ports: - - "127.0.0.1:61613:61613" - - "127.0.0.1:8161:8161" environment: - ACTIVEMQ_CONFIG_NAME=activemq - ACTIVEMQ_CONFIG_DEFAULTACCOUNT=false @@ -87,3 +84,217 @@ services: - ACTIVEMQ_CONFIG_SCHEDULERENABLED=true - ACTIVEMQ_USERS_hermes=supersecret - ACTIVEMQ_CONFIG_QUEUES_events=/queue/events' + postgres14: + image: docker.io/postgres:14 + profiles: + - postgres14 + environment: + - POSTGRES_USER=rucio + - POSTGRES_DB=rucio + - POSTGRES_PASSWORD=rucio + command: ["-c", "fsync=off","-c", "synchronous_commit=off","-c", "full_page_writes=off"] + mysql8: + image: docker.io/mysql:8 + profiles: + - mysql8 + environment: + - MYSQL_USER=rucio + - MYSQL_PASSWORD=rucio + - MYSQL_ROOT_PASSWORD=rucio + - MYSQL_DATABASE=rucio + - MYSQL_TCP_PORT=3308 + command: + - "--default-authentication-plugin=mysql_native_password" + - "--character-set-server=latin1" + oracle: + image: docker.io/gvenzl/oracle-xe:18.4.0 + profiles: + - oracle + environment: + - ORACLE_PASSWORD=rucio + - ORACLE_ALLOW_REMOTE=true + - ORACLE_DISABLE_ASYNCH_IO=true + - processes=1000 + - sessions=1105 + - transactions=1215 + volumes: + - ./oracle_setup.sh:/container-entrypoint-initdb.d/oracle_setup.sh:Z + fts: + image: docker.io/rucio/fts + profiles: + - storage + volumes: + - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z + - ../../certs/hostcert_fts.pem:/etc/grid-security/hostcert.pem:Z + - ../../certs/hostcert_fts.key.pem:/etc/grid-security/hostkey.pem:Z + ulimits: + nofile: + soft: 10240 + hard: 10240 + ftsdb: + image: docker.io/mysql:8 + profiles: + - storage + command: --default-authentication-plugin=mysql_native_password + environment: + - MYSQL_USER=fts + - MYSQL_PASSWORD=fts + - MYSQL_ROOT_PASSWORD=fts + - MYSQL_DATABASE=fts + xrd1: + image: docker.io/rucio/xrootd + profiles: + - storage + environment: + - XRDPORT=1094 + volumes: + - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z + - ../../certs/hostcert_xrd1.pem:/tmp/xrdcert.pem:Z + - ../../certs/hostcert_xrd1.key.pem:/tmp/xrdkey.pem:Z + ulimits: + nofile: + soft: 10240 + hard: 10240 + xrd2: + image: docker.io/rucio/xrootd + profiles: + - storage + environment: + - XRDPORT=1095 + volumes: + - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z + - ../../certs/hostcert_xrd2.pem:/tmp/xrdcert.pem:Z + - ../../certs/hostcert_xrd2.key.pem:/tmp/xrdkey.pem:Z + ulimits: + nofile: + soft: 10240 + hard: 10240 + xrd3: + image: docker.io/rucio/xrootd + profiles: + - storage + environment: + - XRDPORT=1096 + volumes: + - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z + - ../../certs/hostcert_xrd3.pem:/tmp/xrdcert.pem:Z + - ../../certs/hostcert_xrd3.key.pem:/tmp/xrdkey.pem:Z + ulimits: + nofile: + soft: 10240 + hard: 10240 + xrd4: + image: docker.io/rucio/xrootd + profiles: + - storage + environment: + - XRDPORT=1097 + volumes: + - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z + - ../../certs/hostcert_xrd4.pem:/tmp/xrdcert.pem:Z + - ../../certs/hostcert_xrd4.key.pem:/tmp/xrdkey.pem:Z + ulimits: + nofile: + soft: 10240 + hard: 10240 + minio: + image: docker.io/minio/minio + profiles: + - storage + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password + volumes: + - ../../certs/hostcert_minio.pem:/root/.minio/certs/public.crt:Z + - ../../certs/hostcert_minio.key.pem:/root/.minio/certs/private.key:Z + command: ["server", "/data"] + ssh1: + image: docker.io/rucio/ssh + profiles: + - storage + volumes: + - ../../certs/ssh/ruciouser_sshkey.pub:/tmp/sshkey.pub:Z + mongo: + image: docker.io/mongo:5.0 + profiles: + - externalmetadata + postgres: + image: docker.io/postgres:14 + profiles: + - externalmetadata + environment: + - POSTGRES_USER=rucio + - POSTGRES_DB=metadata + - POSTGRES_PASSWORD=secret + command: ["-p", "5433"] + logstash: + image: docker.elastic.co/logstash/logstash-oss:7.3.2 + profiles: + - monitoring + command: bash -c "logstash-plugin install logstash-input-stomp ; /usr/local/bin/docker-entrypoint" + volumes: + - ./pipeline.conf:/usr/share/logstash/pipeline/pipeline.conf:Z + kibana: + image: docker.io/kibana:7.4.0 + profiles: + - monitoring + grafana: + image: docker.io/grafana/grafana:latest + profiles: + - monitoring + db-iam: + image: mariadb:10.11 + profiles: + - iam + environment: + - TZ=Europe/Paris + - MYSQL_ROOT_PASSWORD=supersecret + - MYSQL_USER=iam + - MYSQL_PASSWORD=secret + - MYSQL_DATABASE=iam_db + nginx-iam: + image: nginx + profiles: + - iam + dns_search: cern.ch + environment: + TZ: Europe/Paris + NGINX_HOST: iam + NGINX_PORT: 443 + volumes: + - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z + # - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z + # - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z + - /etc/grid-security/:/etc/grid-security/ + - /dev/urandom:/dev/random + - ../../iam-assets/iam.conf:/etc/nginx/conf.d/default.conf:ro + iam: + profiles: + - iam + image: indigoiam/iam-login-service:v1.8.2 + volumes: + - ../../iam-assets/keystore.jwks:/keystore.jwks:ro + environment: + - IAM_JAVA_OPTS=-Djava.security.egd=file:/dev/urandom -Dspring.profiles.active=prod,oidc,cern,registration,wlcg-scopes -agentlib:jdwp=transport=dt_socket,server=y,address=1044,suspend=n -Dlogging.file.name=/var/log/iam/iam.log + - IAM_HOST= + - IAM_PORT=8090 + - IAM_BASE_URL=https:// + - IAM_ISSUER=https:// + - IAM_FORWARD_HEADERS_STRATEGY=native + - IAM_KEY_STORE_LOCATION=file:/keystore.jwks + - IAM_JWK_CACHE_LIFETIME=21600 + # - IAM_X509_TRUST_ANCHORS_DIR=/etc/grid-security/certificates + # - IAM_X509_TRUST_ANCHORS_REFRESH=14400 + - IAM_TOMCAT_ACCESS_LOG_ENABLED=false + - IAM_TOMCAT_ACCESS_LOG_DIRECTORY=/tmp + - IAM_ACTUATOR_USER_USERNAME=user + - IAM_ACTUATOR_USER_PASSWORD=secret + - IAM_LOCAL_RESOURCES_ENABLE=true + - IAM_LOCAL_RESOURCES_LOCATION=file:/indigo-iam/local-resources + - IAM_ORGANISATION_NAME=rucio-dc + - IAM_TOPBAR_TITLE="INDIGO IAM for rucio-dc" + - IAM_DB_HOST= + - IAM_DB_PORT=3307 + - IAM_DB_NAME=iam_db + - IAM_DB_USERNAME=iam + - IAM_DB_PASSWORD=secret diff --git a/tools/test/oracle_setup.sh b/etc/docker/dev/oracle_setup.sh similarity index 100% rename from tools/test/oracle_setup.sh rename to etc/docker/dev/oracle_setup.sh diff --git a/etc/docker/dev/rucio_entrypoint.sh b/etc/docker/dev/rucio_entrypoint.sh new file mode 100755 index 0000000000..49af398ac2 --- /dev/null +++ b/etc/docker/dev/rucio_entrypoint.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# -*- coding: utf-8 -*- +# Copyright European Organization for Nuclear Research (CERN) since 2012 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CFG_PATH="$RUCIO_SOURCE_DIR"/etc/docker/test/extra/ +if [ -z "$RUCIO_HOME" ]; then + RUCIO_HOME=/opt/rucio +fi + +mkdir -p "$RUCIO_HOME/etc" + +generate_rucio_cfg(){ + local override=$1 + local destination=$2 + + python3 $RUCIO_SOURCE_DIR/tools/merge_rucio_configs.py --use-env \ + -s "$CFG_PATH"/rucio_autotests_common.cfg "$override" \ + -d "$destination" +} + +echo "Generating alembic.ini and rucio.cfg" + +if [ -z "$RDBMS" ]; then + cp "$CFG_PATH"/rucio_default.cfg $RUCIO_HOME/etc/rucio.cfg + cp "$CFG_PATH"/alembic_default.ini $RUCIO_HOME/etc/alembic.ini + +elif [ "$RDBMS" == "oracle" ]; then + generate_rucio_cfg "$CFG_PATH"/rucio_oracle.cfg $RUCIO_HOME/etc/rucio.cfg + cp "$CFG_PATH"/alembic_oracle.ini $RUCIO_HOME/etc/alembic.ini + +elif [ "$RDBMS" == "mysql8" ]; then + generate_rucio_cfg "$CFG_PATH"/rucio_mysql8.cfg $RUCIO_HOME/etc/rucio.cfg + cp "$CFG_PATH"/alembic_mysql8.ini $RUCIO_HOME/etc/alembic.ini + +elif [ "$RDBMS" == "sqlite" ]; then + generate_rucio_cfg "$CFG_PATH"/rucio_sqlite.cfg $RUCIO_HOME/etc/rucio.cfg + cp "$CFG_PATH"/alembic_sqlite.ini $RUCIO_HOME/etc/alembic.ini + +elif [ "$RDBMS" == "postgres14" ]; then + generate_rucio_cfg "$CFG_PATH"/rucio_postgres14.cfg $RUCIO_HOME/etc/rucio.cfg + cp "$CFG_PATH"/alembic_postgres14.ini $RUCIO_HOME/etc/alembic.ini + +fi + +exec "$@" diff --git a/etc/docker/test/alma9.Dockerfile b/etc/docker/test/alma9.Dockerfile index 9f147daba6..9581140939 100644 --- a/etc/docker/test/alma9.Dockerfile +++ b/etc/docker/test/alma9.Dockerfile @@ -108,20 +108,12 @@ FROM python as rucio-runtime httpd-devel \ libnsl libaio \ memcached \ + gridsite \ sqlite \ gfal2-devel \ nodejs npm \ glibc-langpack-en - RUN printf "%s\n" \ - "[epel8]" \ - "name=Extra Packages for Enterprise Linux 8 - x86_64" \ - "metalink=https://mirrors.fedoraproject.org/metalink?repo=epel-8&arch=x86_64&infra=\$infra&content=\$contentdir" \ - "enabled=1" \ - "gpgcheck=0" \ - > /etc/yum.repos.d/epel8.repo - RUN dnf install -y gridsite && rm -rf /etc/yum.repos.d/epel8.repo - RUN mkdir -p /var/log/rucio/trace && \ chmod -R 777 /var/log/rucio && \ mkdir -p /etc/grid-security && \ diff --git a/etc/docker/test/extra/alembic_default.ini b/etc/docker/test/extra/alembic_default.ini new file mode 100644 index 0000000000..641fbe7410 --- /dev/null +++ b/etc/docker/test/extra/alembic_default.ini @@ -0,0 +1,64 @@ +# Copyright European Organization for Nuclear Research (CERN) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Authors: +# - Vincent Garonne , 2014 +# - Mario Lassnig , 2019 + +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = lib/rucio/db/sqla/migrate_repo/ + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +sqlalchemy.url = postgresql://rucio:secret@ruciodb/rucio +version_table_schema = dev + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/etc/docker/test/extra/alembic_mysql8.ini b/etc/docker/test/extra/alembic_mysql8.ini index dc5b5d73eb..376e14a587 100644 --- a/etc/docker/test/extra/alembic_mysql8.ini +++ b/etc/docker/test/extra/alembic_mysql8.ini @@ -24,8 +24,8 @@ script_location = lib/rucio/db/sqla/migrate_repo/ # the 'revision' command, regardless of autogenerate # revision_environment = false -sqlalchemy.url = mysql+pymysql://root:secret@mysql8/mysql -version_table_schema = dev +sqlalchemy.url = mysql+pymysql://rucio:rucio@mysql8:3308/rucio +version_table_schema = rucio # Logging configuration [loggers] diff --git a/etc/docker/test/extra/alembic_oracle.ini b/etc/docker/test/extra/alembic_oracle.ini index ef3eb0ced5..7ed7f5d74e 100644 --- a/etc/docker/test/extra/alembic_oracle.ini +++ b/etc/docker/test/extra/alembic_oracle.ini @@ -24,7 +24,7 @@ script_location = lib/rucio/db/sqla/migrate_repo/ # the 'revision' command, regardless of autogenerate # revision_environment = false -sqlalchemy.url = oracle://system:oracle@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=XE))) +sqlalchemy.url = oracle://system:rucio@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=XE))) version_table_schema = SYSTEM # Logging configuration diff --git a/etc/docker/test/extra/alembic_postgres14.ini b/etc/docker/test/extra/alembic_postgres14.ini index 7d9e03824d..56b3b58e7d 100644 --- a/etc/docker/test/extra/alembic_postgres14.ini +++ b/etc/docker/test/extra/alembic_postgres14.ini @@ -24,7 +24,7 @@ script_location = lib/rucio/db/sqla/migrate_repo/ # the 'revision' command, regardless of autogenerate # revision_environment = false -sqlalchemy.url = postgresql://postgres:secret@postgres14/postgres +sqlalchemy.url = postgresql://rucio:rucio@postgres14/rucio version_table_schema = dev # Logging configuration diff --git a/etc/docker/test/extra/rucio_syntax.cfg b/etc/docker/test/extra/rucio_autotests_common.cfg similarity index 92% rename from etc/docker/test/extra/rucio_syntax.cfg rename to etc/docker/test/extra/rucio_autotests_common.cfg index 1d7c761bc3..6da2740461 100644 --- a/etc/docker/test/extra/rucio_syntax.cfg +++ b/etc/docker/test/extra/rucio_autotests_common.cfg @@ -17,7 +17,6 @@ account = root request_retries = 3 [database] -default = sqlite:////tmp/rucio.db pool_recycle=3600 echo=0 pool_reset_on_return=rollback @@ -169,4 +168,8 @@ support_rucio = https://github.com/rucio/rucio/issues/ gcs = /opt/rucio/etc/google-cloud-storage-test.json [oidc] -idpsecrets = etc/idpsecrets.json +idpsecrets = /opt/rucio/etc/idpsecrets.json +admin_issuer = wlcg + +[api] +endpoints = accountlimits, accounts, archives, auth, config, credentials, dids, dirac, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, tmp_dids, traces, vos diff --git a/etc/docker/test/extra/rucio_default.cfg b/etc/docker/test/extra/rucio_default.cfg new file mode 100644 index 0000000000..265695d1a9 --- /dev/null +++ b/etc/docker/test/extra/rucio_default.cfg @@ -0,0 +1,187 @@ +[common] +logdir = /var/log/rucio +loglevel = DEBUG +mailtemplatedir=/opt/rucio/etc/mail_templates + +[client] +rucio_host = https://rucio:443 +auth_host = https://rucio:443 +auth_type = userpass +username = ddmlab +password = secret +ca_cert = /etc/grid-security/certificates/5fca1cb1.0 +client_cert = /opt/rucio/etc/usercert.pem +client_key = /opt/rucio/etc/userkey.pem +client_x509_proxy = $X509_USER_PROXY +account = root +request_retries = 3 + +[database] +default = postgresql://rucio:secret@ruciodb/rucio +schema = dev +echo=0 +pool_recycle=3600 +pool_size=20 +max_overflow=20 +pool_reset_on_return=rollback + +[bootstrap] +userpass_identity = ddmlab +userpass_pwd = secret +userpass_email = rucio-dev@cern.ch + +# Default development client certificate from /opt/rucio/etc/usercert.pem +x509_identity = /CN=Rucio User +x509_email = rucio-dev@cern.ch + +# Default Kerberos account +gss_identity = rucio-dev@CERN.CH +gss_email = rucio-dev@cern.ch + +[monitor] +carbon_server = graphite +carbon_port = 8125 +user_scope = docker + +[conveyor] +scheme = https,davs,gsiftp,root,srm,mock +#scheme = https +#user_transfers = cms +#user_activities = ['dummy_user_activity'] +#hostcert = /etc/grid-security/hostcert.pem +#hostkey = /etc/grid-security/hostkey.pem +transfertool = fts3 +ftshosts = https://fts:8446 +cacert = /etc/grid-security/certificates/5fca1cb1.0 +usercert = /opt/rucio/etc/usercertkey.pem + +[messaging-fts3] +port = 61613 +nonssl_port = 61613 +use_ssl = False +ssl_key_file = /opt/rucio/etc/userkey.pem +ssl_cert_file = /opt/rucio/etc/usercert.pem +destination = /topic/transfer.fts_monitoring_complete +username = receiver +password = supersecret +brokers = activemq +voname = atlas + +[messaging-hermes] +username = hermes +password = supersecret +port = 61613 +nonssl_port = 61613 +use_ssl = False +destination = /queue/events +ssl_key_file = /opt/rucio/etc/userkey.pem +ssl_cert_file = /opt/rucio/etc/usercert.pem +brokers = activemq +voname = atlas +email_from = Rucio +email_test = spamspamspam@cern.ch + +[transmogrifier] +maxdids = 100000 + +[accounts] +# These are accounts that can write into scopes owned by another account +special_accounts = panda, tier0 + +[trace] +tracedir = /var/log/rucio/trace +brokers=activemq +port=61013 +username = username +password = password +topic = /topic/rucio.tracer + +[nongrid-trace] +tracedir = /var/log/rucio/trace +brokers=activemq +port=61013 +username = username +password = password +topic = /topic/rucio.tracer + +[tracer-kronos] +brokers=activemq +port=61013 +ssl_key_file = /opt/rucio/etc/userkey.pem +ssl_cert_file = /opt/rucio/etc/usercert.pem +queue = /queue/Consumer.kronos.rucio.tracer +prefetch_size = 10 +chunksize = 10 +subscription_id = rucio-tracer-listener +use_ssl = False +reconnect_attempts = 100 +excluded_usrdns = /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gangarbt/CN=722147/CN=Robot: Ganga Robot/CN=proxy +username = username +password = password +dataset_wait = 60 + +[injector] +file = /opt/rucio/tools/test.file.1000 +bytes = 1000 +md5 = fd21ce524a9e45060fd3f62c4ef6a386 +adler32 = 52590737 + +[alembic] +cfg = /opt/rucio/etc/alembic.ini + +[messaging-cache] +port = 61023 +ssl_key_file = /opt/rucio/etc/userkey.pem +ssl_cert_file = /opt/rucio/etc/usercert.pem +destination = /topic/rucio.cache +brokers = activemq +voname = atlas +account = cache_mb + +[test] +cacert = /etc/grid-security/certificates/5fca1cb1.0 +usercert = /opt/rucio/etc/usercert.pem +userkey = /opt/rucio/etc/userkey.pem + +[nagios] +proxy = /opt/rucio/etc/usercertkey.pem +rfcproxy = /opt/rucio/etc/usercertkey.pem +fts_servers = https://fts3:8446 + +[auditor] +cache = /opt/rucio/auditor-cache +results = /opt/rucio/auditor-results + +[hermes] +email_from = Rucio +email_test = spamspamspam@cern.ch + +[c3po] +placement_algorithm = t2_free_space +elastic_url = http://elastic:9200 +redis_host = rucio +redis_port = 6379 + +[c3po-popularity] +elastic_url = http://elastic:9200 + +[c3po-site-mapper] +panda_url = http://agis:80/request/pandaqueue/query/list/?json +ddm_url = http://agis:80/request/ddmendpoint/query/list/?json + +[c3po-workload] +panda_url = http://bigpanda:80/jobs/?category=analysis&jobstatus=running +window = 604800 + +[policy] +permission = atlas +schema = atlas +lfn2pfn_algorithm_default = hash +support = rucio-dev@cern.ch +support_rucio = https://github.com/rucio/rucio/issues/ + +[credentials] +gcs = /opt/rucio/etc/google-cloud-storage-test.json + +[api] +endpoints = accountlimits, accounts, archives, auth, config, credentials, dids, dirac, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, traces, vos diff --git a/etc/docker/test/extra/rucio_multi_vo_ts2_postgres14.cfg b/etc/docker/test/extra/rucio_multi_vo_ts2_postgres14.cfg index 070eb53d54..00cf87eb75 100644 --- a/etc/docker/test/extra/rucio_multi_vo_ts2_postgres14.cfg +++ b/etc/docker/test/extra/rucio_multi_vo_ts2_postgres14.cfg @@ -1,182 +1,16 @@ [common] -logdir = /var/log/rucio -loglevel = DEBUG -mailtemplatedir=/opt/rucio/etc/mail_templates multi_vo = True [client] -rucio_host = https://localhost:443 -auth_host = https://localhost:443 -auth_type = userpass -username = ddmlab -password = secret -ca_cert = /opt/rucio/etc/rucio_ca.pem -client_cert = /opt/rucio/etc/ruciouser.pem -client_key = /opt/rucio/etc/ruciouser.key.pem -client_x509_proxy = $X509_USER_PROXY -account = root -request_retries = 3 # Relies on mapping created by bootstrap_tests.py vo = testvo2 [database] -default = postgresql://postgres:secret@postgres14/postgres +default = postgresql://rucio:rucio@postgres14/rucio schema = dev -pool_recycle=3600 -echo=0 -pool_reset_on_return=rollback pool_size = 20 max_overflow = 20 -[bootstrap] -# Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest() -userpass_identity = ddmlab -userpass_pwd = secret -userpass_email = rucio-dev@cern.ch - -# Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt -x509_identity = CN=Rucio User -x509_email = rucio-dev@cern.ch - -# Default DDMLAB cern account -gss_identity = rucio-dev@CERN.CH -gss_email = rucio-dev@cern.ch - -[monitor] -carbon_server = localhost -carbon_port = 8125 -user_scope = travis - -[conveyor] -scheme = srm,root,davs,gsiftp,http,https -transfertool = fts3 -ftshosts = https://fts:8446 -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem - -[messaging-fts3] -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/transfer.fts_monitoring_complete -brokers = activemq -voname = atlas -username = receiver -password = supersecret - -[messaging-hermes] -username = admin -password = secret -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.events -brokers = activemq -voname = atlas -email_from = Rucio -email_test = spamspamspam@cern.ch - -[transmogrifier] -maxdids = 100000 - -[accounts] -# These are accounts that can write into scopes owned by another account -special_accounts = panda, tier0 - -[trace] -tracedir = /var/log/rucio/trace -brokers=activemq -port=61013 -username = username -password = password -topic = /topic/rucio.tracer - -[tracer-kronos] -brokers=activemq -port=61013 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -queue = /queue/Consumer.kronos.rucio.tracer -prefetch_size = 10 -chunksize = 10 -subscription_id = rucio-tracer-listener -use_ssl = False -reconnect_attempts = 100 -excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch -username = username -password = password -dataset_wait = 60 - -[injector] -file = /opt/rucio/tools/test.file.1000 -bytes = 1000 -md5 = fd21ce524a9e45060fd3f62c4ef6a386 -adler32 = 52590737 - -[alembic] -cfg = /opt/rucio/etc/alembic.ini - -[messaging-cache] -port = 61023 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.cache -brokers = activemq -voname = atlas -account = cache_mb - -[test] -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem -userkey = /opt/rucio/etc/ruciouser.key.pem - -[nagios] -proxy = /opt/rucio/etc/usercert.pem -rfcproxy = /opt/rucio/etc/usercert.pem -fts_servers = https://fts:8446 - -[auditor] -cache = /opt/rucio/auditor-cache -results = /opt/rucio/auditor-results - -[hermes] -email_from = Rucio -email_test = spamspamspam@cern.ch - -[c3po] -placement_algorithm = t2_free_space -elastic_url = http://elastic:9200 -redis_host = redis -redis_port = 6379 - -[c3po-popularity] -elastic_url = http://elastic:9200 - -[c3po-site-mapper] -panda_url = http://agis:80/request/pandaqueue/query/list/?json -ddm_url = http://agis:80/request/ddmendpoint/query/list/?json - -[c3po-workload] -panda_url = http://bigpanda:80/jobs/?category=analysis&jobstatus=running -window = 604800 - [policy] permission = generic_multi_vo schema = generic_multi_vo -lfn2pfn_algorithm_default = hash -support = rucio-dev@cern.ch -support_rucio = https://github.com/rucio/rucio/issues/ - -[credentials] -gcs = /opt/rucio/etc/google-cloud-storage-test.json - -[oidc] -idpsecrets = /opt/rucio/etc/idpsecrets.json -admin_issuer = wlcg - -[api] -endpoints = accountlimits, accounts, archives, auth, config, credentials, dids, dirac, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, tmp_dids, traces, vos diff --git a/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg b/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg index ac2d54b176..1af5e01ed6 100644 --- a/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg +++ b/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg @@ -1,182 +1,19 @@ [common] -logdir = /var/log/rucio -loglevel = DEBUG -mailtemplatedir=/opt/rucio/etc/mail_templates multi_vo = True [client] -rucio_host = https://localhost:443 -auth_host = https://localhost:443 -auth_type = userpass -username = ddmlab -password = secret -ca_cert = /opt/rucio/etc/rucio_ca.pem -client_cert = /opt/rucio/etc/ruciouser.pem -client_key = /opt/rucio/etc/ruciouser.key.pem -client_x509_proxy = $X509_USER_PROXY -account = root -request_retries = 3 # Relies on mapping created by bootstrap_tests.py vo = testvo1 [database] -default = postgresql://postgres:secret@postgres14/postgres +default = postgresql://rucio:rucio@postgres14/rucio schema = dev -pool_recycle=3600 -echo=0 -pool_reset_on_return=rollback pool_size = 20 max_overflow = 20 -[bootstrap] -# Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest() -userpass_identity = ddmlab -userpass_pwd = secret -userpass_email = rucio-dev@cern.ch - -# Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt -x509_identity = CN=Rucio User -x509_email = rucio-dev@cern.ch - -# Default DDMLAB cern account -gss_identity = rucio-dev@CERN.CH -gss_email = rucio-dev@cern.ch - -[monitor] -carbon_server = localhost -carbon_port = 8125 -user_scope = travis - -[conveyor] -scheme = srm,root,davs,gsiftp,http,https -transfertool = fts3 -ftshosts = https://fts:8446 -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem - -[messaging-fts3] -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/transfer.fts_monitoring_complete -brokers = activemq -voname = atlas -username = receiver -password = supersecret - -[messaging-hermes] -username = admin -password = secret -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.events -brokers = activemq -voname = atlas -email_from = Rucio -email_test = spamspamspam@cern.ch - -[transmogrifier] -maxdids = 100000 - -[accounts] -# These are accounts that can write into scopes owned by another account -special_accounts = panda, tier0 - -[trace] -tracedir = /var/log/rucio/trace -brokers=activemq -port=61013 -username = username -password = password -topic = /topic/rucio.tracer - -[tracer-kronos] -brokers=activemq -port=61013 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -queue = /queue/Consumer.kronos.rucio.tracer -prefetch_size = 10 -chunksize = 10 -subscription_id = rucio-tracer-listener -use_ssl = False -reconnect_attempts = 100 -excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch -username = username -password = password -dataset_wait = 60 - -[injector] -file = /opt/rucio/tools/test.file.1000 -bytes = 1000 -md5 = fd21ce524a9e45060fd3f62c4ef6a386 -adler32 = 52590737 - -[alembic] -cfg = /opt/rucio/etc/alembic.ini - -[messaging-cache] -port = 61023 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.cache -brokers = activemq -voname = atlas -account = cache_mb - -[test] -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem -userkey = /opt/rucio/etc/ruciouser.key.pem - -[nagios] -proxy = /opt/rucio/etc/usercert.pem -rfcproxy = /opt/rucio/etc/usercert.pem -fts_servers = https://fts:8446 - -[auditor] -cache = /opt/rucio/auditor-cache -results = /opt/rucio/auditor-results - -[hermes] -email_from = Rucio -email_test = spamspamspam@cern.ch - -[c3po] -placement_algorithm = t2_free_space -elastic_url = http://elastic:9200 -redis_host = redis -redis_port = 6379 - -[c3po-popularity] -elastic_url = http://elastic:9200 - -[c3po-site-mapper] -panda_url = http://agis:80/request/pandaqueue/query/list/?json -ddm_url = http://agis:80/request/ddmendpoint/query/list/?json - -[c3po-workload] -panda_url = http://bigpanda:80/jobs/?category=analysis&jobstatus=running -window = 604800 - [policy] permission = generic_multi_vo schema = generic_multi_vo -lfn2pfn_algorithm_default = hash -support = rucio-dev@cern.ch -support_rucio = https://github.com/rucio/rucio/issues/ -[credentials] -gcs = /opt/rucio/etc/google-cloud-storage-test.json - -[oidc] -idpsecrets = /opt/rucio/etc/idpsecrets.json -admin_issuer = wlcg - -[api] -endpoints = accountlimits, accounts, archives, auth, config, credentials, dids, dirac, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, tmp_dids, traces, vos +[alembic] +cfg = /opt/rucio/etc/multi_vo/tst/etc/alembic.ini diff --git a/etc/docker/test/extra/rucio_mysql8.cfg b/etc/docker/test/extra/rucio_mysql8.cfg index eefa7f6cab..f5fe31764c 100644 --- a/etc/docker/test/extra/rucio_mysql8.cfg +++ b/etc/docker/test/extra/rucio_mysql8.cfg @@ -1,179 +1,5 @@ -[common] -logdir = /var/log/rucio -loglevel = DEBUG -mailtemplatedir=/opt/rucio/etc/mail_templates - -[client] -rucio_host = https://localhost:443 -auth_host = https://localhost:443 -auth_type = userpass -username = ddmlab -password = secret -ca_cert = /opt/rucio/etc/rucio_ca.pem -client_cert = /opt/rucio/etc/ruciouser.pem -client_key = /opt/rucio/etc/ruciouser.key.pem -client_x509_proxy = $X509_USER_PROXY -account = root -request_retries = 3 - [database] -default = mysql+pymysql://root:secret@mysql8/mysql -schema=dev -pool_recycle=3600 -echo=0 -pool_reset_on_return=rollback +default = mysql+pymysql://rucio:rucio@mysql8:3308/rucio +schema=rucio pool_size = 20 max_overflow = 20 - -[bootstrap] -# Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest() -userpass_identity = ddmlab -userpass_pwd = secret -userpass_email = rucio-dev@cern.ch - -# Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt -x509_identity = CN=Rucio User -x509_email = rucio-dev@cern.ch - -# Default DDMLAB cern account -gss_identity = rucio-dev@CERN.CH -gss_email = rucio-dev@cern.ch - -[monitor] -carbon_server = localhost -carbon_port = 8125 -user_scope = travis - -[conveyor] -scheme = srm,root,davs,gsiftp,http,https -transfertool = fts3 -ftshosts = https://fts:8446 -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem - -[messaging-fts3] -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/transfer.fts_monitoring_complete -brokers = activemq -voname = atlas -username = receiver -password = supersecret - -[messaging-hermes] -username = admin -password = secret -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.events -brokers = activemq -voname = atlas -email_from = Rucio -email_test = spamspamspam@cern.ch - -[transmogrifier] -maxdids = 100000 - -[accounts] -# These are accounts that can write into scopes owned by another account -special_accounts = panda, tier0 - -[trace] -tracedir = /var/log/rucio/trace -brokers=activemq -port=61013 -username = username -password = password -topic = /topic/rucio.tracer - -[tracer-kronos] -brokers=activemq -port=61013 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -queue = /queue/Consumer.kronos.rucio.tracer -prefetch_size = 10 -chunksize = 10 -subscription_id = rucio-tracer-listener -use_ssl = False -reconnect_attempts = 100 -excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch -username = username -password = password -dataset_wait = 60 - -[injector] -file = /opt/rucio/tools/test.file.1000 -bytes = 1000 -md5 = fd21ce524a9e45060fd3f62c4ef6a386 -adler32 = 52590737 - -[alembic] -cfg = /opt/rucio/etc/alembic.ini - -[messaging-cache] -port = 61023 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.cache -brokers = activemq -voname = atlas -account = cache_mb - -[test] -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem -userkey = /opt/rucio/etc/ruciouser.key.pem - -[nagios] -proxy = /opt/rucio/etc/usercert.pem -rfcproxy = /opt/rucio/etc/usercert.pem -fts_servers = https://fts:8446 - -[auditor] -cache = /opt/rucio/auditor-cache -results = /opt/rucio/auditor-results - -[hermes] -email_from = Rucio -email_test = spamspamspam@cern.ch - -[c3po] -placement_algorithm = t2_free_space -elastic_url = http://elastic:9200 -redis_host = redis -redis_port = 6379 - -[c3po-popularity] -elastic_url = http://elastic:9200 - -[c3po-site-mapper] -panda_url = http://agis:80/request/pandaqueue/query/list/?json -ddm_url = http://agis:80/request/ddmendpoint/query/list/?json - -[c3po-workload] -panda_url = http://bigpanda:80/jobs/?category=analysis&jobstatus=running -window = 604800 - -[policy] -permission = atlas -schema = atlas -lfn2pfn_algorithm_default = hash -support = rucio-dev@cern.ch -support_rucio = https://github.com/rucio/rucio/issues/ - -[credentials] -gcs = /opt/rucio/etc/google-cloud-storage-test.json - -[oidc] -idpsecrets = /opt/rucio/etc/idpsecrets.json -admin_issuer = wlcg - -[api] -endpoints = accountlimits, accounts, archives, auth, config, credentials, dids, dirac, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, tmp_dids, traces, vos diff --git a/etc/docker/test/extra/rucio_oracle.cfg b/etc/docker/test/extra/rucio_oracle.cfg index 1bf4394139..8d22586e22 100644 --- a/etc/docker/test/extra/rucio_oracle.cfg +++ b/etc/docker/test/extra/rucio_oracle.cfg @@ -1,179 +1,5 @@ -[common] -logdir = /var/log/rucio -loglevel = DEBUG -mailtemplatedir=/opt/rucio/etc/mail_templates - -[client] -rucio_host = https://localhost:443 -auth_host = https://localhost:443 -auth_type = userpass -username = ddmlab -password = secret -ca_cert = /opt/rucio/etc/rucio_ca.pem -client_cert = /opt/rucio/etc/ruciouser.pem -client_key = /opt/rucio/etc/ruciouser.key.pem -client_x509_proxy = $X509_USER_PROXY -account = root -request_retries = 3 - [database] -default = oracle://system:oracle@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=XE))) +default = oracle://system:rucio@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=XE))) schema=system -pool_recycle=3600 -echo=0 -pool_reset_on_return=rollback pool_size = 20 max_overflow = 20 - -[bootstrap] -# Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest() -userpass_identity = ddmlab -userpass_pwd = secret -userpass_email = rucio-dev@cern.ch - -# Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt -x509_identity = CN=Rucio User -x509_email = rucio-dev@cern.ch - -# Default DDMLAB cern account -gss_identity = rucio-dev@CERN.CH -gss_email = rucio-dev@cern.ch - -[monitor] -carbon_server = localhost -carbon_port = 8125 -user_scope = travis - -[conveyor] -scheme = srm,root,davs,gsiftp,http,https -transfertool = fts3 -ftshosts = https://fts:8446 -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem - -[messaging-fts3] -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/transfer.fts_monitoring_complete -brokers = activemq -voname = atlas -username = receiver -password = supersecret - -[messaging-hermes] -username = admin -password = secret -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.events -brokers = activemq -voname = atlas -email_from = Rucio -email_test = spamspamspam@cern.ch - -[transmogrifier] -maxdids = 100000 - -[accounts] -# These are accounts that can write into scopes owned by another account -special_accounts = panda, tier0 - -[trace] -tracedir = /var/log/rucio/trace -brokers=activemq -port=61013 -username = username -password = password -topic = /topic/rucio.tracer - -[tracer-kronos] -brokers=activemq -port=61013 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -queue = /queue/Consumer.kronos.rucio.tracer -prefetch_size = 10 -chunksize = 10 -subscription_id = rucio-tracer-listener -use_ssl = False -reconnect_attempts = 100 -excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch -username = username -password = password -dataset_wait = 60 - -[injector] -file = /opt/rucio/tools/test.file.1000 -bytes = 1000 -md5 = fd21ce524a9e45060fd3f62c4ef6a386 -adler32 = 52590737 - -[alembic] -cfg = /opt/rucio/etc/alembic.ini - -[messaging-cache] -port = 61023 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.cache -brokers = activemq -voname = atlas -account = cache_mb - -[test] -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem -userkey = /opt/rucio/etc/ruciouser.key.pem - -[nagios] -proxy = /opt/rucio/etc/usercert.pem -rfcproxy = /opt/rucio/etc/usercert.pem -fts_servers = https://fts:8446 - -[auditor] -cache = /opt/rucio/auditor-cache -results = /opt/rucio/auditor-results - -[hermes] -email_from = Rucio -email_test = spamspamspam@cern.ch - -[c3po] -placement_algorithm = t2_free_space -elastic_url = http://elastic:9200 -redis_host = redis -redis_port = 6379 - -[c3po-popularity] -elastic_url = http://elastic:9200 - -[c3po-site-mapper] -panda_url = http://agis:80/request/pandaqueue/query/list/?json -ddm_url = http://agis:80/request/ddmendpoint/query/list/?json - -[c3po-workload] -panda_url = http://bigpanda:80/jobs/?category=analysis&jobstatus=running -window = 604800 - -[policy] -permission = atlas -schema = atlas -lfn2pfn_algorithm_default = hash -support = rucio-dev@cern.ch -support_rucio = https://github.com/rucio/rucio/issues/ - -[credentials] -gcs = /opt/rucio/etc/google-cloud-storage-test.json - -[oidc] -idpsecrets = /opt/rucio/etc/idpsecrets.json -admin_issuer = wlcg - -[api] -endpoints = accountlimits, accounts, archives, auth, config, credentials, dids, dirac, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, tmp_dids, traces, vos diff --git a/etc/docker/test/extra/rucio_postgres14.cfg b/etc/docker/test/extra/rucio_postgres14.cfg index abaafe9b1f..9f791753ca 100644 --- a/etc/docker/test/extra/rucio_postgres14.cfg +++ b/etc/docker/test/extra/rucio_postgres14.cfg @@ -1,179 +1,5 @@ -[common] -logdir = /var/log/rucio -loglevel = DEBUG -mailtemplatedir=/opt/rucio/etc/mail_templates - -[client] -rucio_host = https://localhost:443 -auth_host = https://localhost:443 -auth_type = userpass -username = ddmlab -password = secret -ca_cert = /opt/rucio/etc/rucio_ca.pem -client_cert = /opt/rucio/etc/ruciouser.pem -client_key = /opt/rucio/etc/ruciouser.key.pem -client_x509_proxy = $X509_USER_PROXY -account = root -request_retries = 3 - [database] -default = postgresql://postgres:secret@postgres14/postgres +default = postgresql://rucio:rucio@postgres14/rucio schema=dev -pool_recycle=3600 -echo=0 -pool_reset_on_return=rollback pool_size = 20 max_overflow = 20 - -[bootstrap] -# Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest() -userpass_identity = ddmlab -userpass_pwd = secret -userpass_email = rucio-dev@cern.ch - -# Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt -x509_identity = CN=Rucio User -x509_email = rucio-dev@cern.ch - -# Default DDMLAB cern account -gss_identity = rucio-dev@CERN.CH -gss_email = rucio-dev@cern.ch - -[monitor] -carbon_server = localhost -carbon_port = 8125 -user_scope = travis - -[conveyor] -scheme = srm,root,davs,gsiftp,http,https -transfertool = fts3 -ftshosts = https://fts:8446 -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem - -[messaging-fts3] -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/transfer.fts_monitoring_complete -brokers = activemq -voname = atlas -username = receiver -password = supersecret - -[messaging-hermes] -username = admin -password = secret -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.events -brokers = activemq -voname = atlas -email_from = Rucio -email_test = spamspamspam@cern.ch - -[transmogrifier] -maxdids = 100000 - -[accounts] -# These are accounts that can write into scopes owned by another account -special_accounts = panda, tier0 - -[trace] -tracedir = /var/log/rucio/trace -brokers=activemq -port=61013 -username = username -password = password -topic = /topic/rucio.tracer - -[tracer-kronos] -brokers=activemq -port=61013 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -queue = /queue/Consumer.kronos.rucio.tracer -prefetch_size = 10 -chunksize = 10 -subscription_id = rucio-tracer-listener -use_ssl = False -reconnect_attempts = 100 -excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch -username = username -password = password -dataset_wait = 60 - -[injector] -file = /opt/rucio/tools/test.file.1000 -bytes = 1000 -md5 = fd21ce524a9e45060fd3f62c4ef6a386 -adler32 = 52590737 - -[alembic] -cfg = /opt/rucio/etc/alembic.ini - -[messaging-cache] -port = 61023 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.cache -brokers = activemq -voname = atlas -account = cache_mb - -[test] -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem -userkey = /opt/rucio/etc/ruciouser.key.pem - -[nagios] -proxy = /opt/rucio/etc/usercert.pem -rfcproxy = /opt/rucio/etc/usercert.pem -fts_servers = https://fts:8446 - -[auditor] -cache = /opt/rucio/auditor-cache -results = /opt/rucio/auditor-results - -[hermes] -email_from = Rucio -email_test = spamspamspam@cern.ch - -[c3po] -placement_algorithm = t2_free_space -elastic_url = http://elastic:9200 -redis_host = redis -redis_port = 6379 - -[c3po-popularity] -elastic_url = http://elastic:9200 - -[c3po-site-mapper] -panda_url = http://agis:80/request/pandaqueue/query/list/?json -ddm_url = http://agis:80/request/ddmendpoint/query/list/?json - -[c3po-workload] -panda_url = http://bigpanda:80/jobs/?category=analysis&jobstatus=running -window = 604800 - -[policy] -permission = atlas -schema = atlas -lfn2pfn_algorithm_default = hash -support = rucio-dev@cern.ch -support_rucio = https://github.com/rucio/rucio/issues/ - -[credentials] -gcs = /opt/rucio/etc/google-cloud-storage-test.json - -[oidc] -idpsecrets = /opt/rucio/etc/idpsecrets.json -admin_issuer = wlcg - -[api] -endpoints = accountlimits, accounts, archives, auth, config, credentials, dids, dirac, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, tmp_dids, traces, vos diff --git a/etc/docker/test/extra/rucio_sqlite.cfg b/etc/docker/test/extra/rucio_sqlite.cfg index 41210741fa..740f86f113 100644 --- a/etc/docker/test/extra/rucio_sqlite.cfg +++ b/etc/docker/test/extra/rucio_sqlite.cfg @@ -1,176 +1,2 @@ -[common] -logdir = /var/log/rucio -loglevel = DEBUG -mailtemplatedir=/opt/rucio/etc/mail_templates - -[client] -rucio_host = https://localhost:443 -auth_host = https://localhost:443 -auth_type = userpass -username = ddmlab -password = secret -ca_cert = /opt/rucio/etc/rucio_ca.pem -client_cert = /opt/rucio/etc/ruciouser.pem -client_key = /opt/rucio/etc/ruciouser.key.pem -client_x509_proxy = $X509_USER_PROXY -account = root -request_retries = 3 - [database] default = sqlite:////tmp/rucio.db -pool_recycle=3600 -echo=0 -pool_reset_on_return=rollback - -[bootstrap] -# Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest() -userpass_identity = ddmlab -userpass_pwd = secret -userpass_email = rucio-dev@cern.ch - -# Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt -x509_identity = CN=Rucio User -x509_email = rucio-dev@cern.ch - -# Default DDMLAB cern account -gss_identity = rucio-dev@CERN.CH -gss_email = rucio-dev@cern.ch - -[monitor] -carbon_server = localhost -carbon_port = 8125 -user_scope = travis - -[conveyor] -scheme = srm,root,davs,gsiftp,http,https -transfertool = fts3 -ftshosts = https://fts:8446 -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem - -[messaging-fts3] -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/transfer.fts_monitoring_complete -brokers = activemq -voname = atlas -username = receiver -password = supersecret - -[messaging-hermes] -username = admin -password = secret -port = 61613 -nonssl_port = 61613 -use_ssl = False -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.events -brokers = activemq -voname = atlas -email_from = Rucio -email_test = spamspamspam@cern.ch - -[transmogrifier] -maxdids = 100000 - -[accounts] -# These are accounts that can write into scopes owned by another account -special_accounts = panda, tier0 - -[trace] -tracedir = /var/log/rucio/trace -brokers=activemq -port=61013 -username = username -password = password -topic = /topic/rucio.tracer - -[tracer-kronos] -brokers=activemq -port=61013 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -queue = /queue/Consumer.kronos.rucio.tracer -prefetch_size = 10 -chunksize = 10 -subscription_id = rucio-tracer-listener -use_ssl = False -reconnect_attempts = 100 -excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch -username = username -password = password -dataset_wait = 60 - -[injector] -file = /opt/rucio/tools/test.file.1000 -bytes = 1000 -md5 = fd21ce524a9e45060fd3f62c4ef6a386 -adler32 = 52590737 - -[alembic] -cfg = /opt/rucio/etc/alembic.ini - -[messaging-cache] -port = 61023 -ssl_key_file = /opt/rucio/etc/usercert.key.pem -ssl_cert_file = /opt/rucio/etc/usercert.pem -destination = /topic/rucio.cache -brokers = activemq -voname = atlas -account = cache_mb - -[test] -cacert = /opt/rucio/etc/rucio_ca.pem -usercert = /opt/rucio/etc/ruciouser.pem -userkey = /opt/rucio/etc/ruciouser.key.pem - -[nagios] -proxy = /opt/rucio/etc/usercert.pem -rfcproxy = /opt/rucio/etc/usercert.pem -fts_servers = https://fts:8446 - -[auditor] -cache = /opt/rucio/auditor-cache -results = /opt/rucio/auditor-results - -[hermes] -email_from = Rucio -email_test = spamspamspam@cern.ch - -[c3po] -placement_algorithm = t2_free_space -elastic_url = http://elastic:9200 -redis_host = redis -redis_port = 6379 - -[c3po-popularity] -elastic_url = http://elastic:9200 - -[c3po-site-mapper] -panda_url = http://agis:80/request/pandaqueue/query/list/?json -ddm_url = http://agis:80/request/ddmendpoint/query/list/?json - -[c3po-workload] -panda_url = http://bigpanda:80/jobs/?category=analysis&jobstatus=running -window = 604800 - -[policy] -permission = atlas -schema = atlas -lfn2pfn_algorithm_default = hash -support = rucio-dev@cern.ch -support_rucio = https://github.com/rucio/rucio/issues/ - -[credentials] -gcs = /opt/rucio/etc/google-cloud-storage-test.json - -[oidc] -idpsecrets = /opt/rucio/etc/idpsecrets.json -admin_issuer = wlcg - -[api] -endpoints = accountlimits, accounts, archives, auth, config, credentials, dids, dirac, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, tmp_dids, traces, vos diff --git a/etc/docker/test/matrix.yml b/etc/docker/test/matrix.yml index 4e18976858..92cd20e3a6 100644 --- a/etc/docker/test/matrix.yml +++ b/etc/docker/test/matrix.yml @@ -21,7 +21,7 @@ python: - remote_dbs - multi_vo - client - - client_synxtax + - client_syntax - syntax - sqlite diff --git a/lib/rucio/db/sqla/session.py b/lib/rucio/db/sqla/session.py index 22c7d9dbf8..3591d18eaf 100644 --- a/lib/rucio/db/sqla/session.py +++ b/lib/rucio/db/sqla/session.py @@ -14,16 +14,19 @@ # limitations under the License. import copy +import logging import os import sys +from datetime import datetime, timedelta from functools import update_wrapper from inspect import isgeneratorfunction, getfullargspec from os.path import basename +from time import sleep from threading import Lock from typing import TYPE_CHECKING -from sqlalchemy import create_engine, event, MetaData -from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError, TimeoutError +from sqlalchemy import create_engine, event, MetaData, text +from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError, TimeoutError, SQLAlchemyError from sqlalchemy.orm import DeclarativeBase, sessionmaker, scoped_session, Session from sqlalchemy.pool import QueuePool, SingletonThreadPool, NullPool @@ -280,6 +283,27 @@ def get_session(): return session +def wait_for_database(timeout: int = 60, interval: int = 2, *, logger=logging.log): + """ Wait for the database for a specific amount of time """ + + end_time = datetime.utcnow() + timedelta(seconds=timeout) + while True: + try: + session = get_session() + if session.bind.dialect.name == 'oracle': + session.execute(text('select 1 from dual')) + else: + session.execute(text('select 1')) + session.close() + break + except SQLAlchemyError as e: + logger(logging.WARNING, 'Still waiting for database: %s', e) + if datetime.utcnow() >= end_time: + raise + + sleep(interval) + + def retry_if_db_connection_error(exception): """Return True if error in connecting to db.""" if isinstance(exception, (OperationalError, DatabaseException)): diff --git a/tests/test_clients.py b/tests/test_clients.py index 3b3478ff50..8aad23db46 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -21,7 +21,7 @@ from rucio.client.baseclient import BaseClient from rucio.client.client import Client -from rucio.common.config import config_get, config_set +from rucio.common.config import config_get, config_set, Config from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException from rucio.common.utils import execute from tests.mocks.mock_http_server import MockServer @@ -142,7 +142,8 @@ def test_import_without_config_file(self, vo): thus have to disable the access to it (move it) and make sure to run the code in a different environment. """ - rename("/opt/rucio/etc/rucio.cfg", "/opt/rucio/etc/rucio.cfg.tmp") + configfile = Config().configfile + rename(configfile, f"{configfile}.tmp") try: exitcode, _, err = execute("python -c 'from rucio.client import Client'") print(exitcode, err) @@ -150,4 +151,4 @@ def test_import_without_config_file(self, vo): assert "RuntimeError: Could not load Rucio configuration file." not in err finally: # This is utterly important to not mess up the environment. - rename("/opt/rucio/etc/rucio.cfg.tmp", "/opt/rucio/etc/rucio.cfg") + rename(f"{configfile}.tmp", configfile) diff --git a/tools/merge_rucio_configs.py b/tools/merge_rucio_configs.py index def2776eab..d6dfbe8aa2 100644 --- a/tools/merge_rucio_configs.py +++ b/tools/merge_rucio_configs.py @@ -130,6 +130,7 @@ def merge_configs(source_file_paths, dest_file_path, use_env=True, logger=loggin logger(logging.INFO, "Merged {} configuration values from ENV".format(config_len(env_config))) if dest_file_path: + logger(logging.INFO, "Writing {}".format(dest_file_path)) with open(dest_file_path, 'w') as dest_file: parser.write(dest_file) else: diff --git a/tools/purge_bin.py b/tools/purge_bin.py deleted file mode 100755 index d97d8dda6e..0000000000 --- a/tools/purge_bin.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright European Organization for Nuclear Research (CERN) since 2012 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import os.path -base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(base_path) -os.chdir(base_path) - -from rucio.db.sqla.session import get_session # noqa: E402 - -# Exit statuses -OK, WARNING, CRITICAL, UNKNOWN = 0, 1, 2, 3 - - -def purge_bin(): - try: - session = get_session() - sql = "select table_name from user_tables" - for table in session.execute(sql): - query = "drop table %s cascade constraints purge" % table[0] - session.execute(query) - except: - pass - finally: - session.remove() - - -if __name__ == "__main__": - purge_bin() diff --git a/tools/run_multi_vo_tests_docker.sh b/tools/run_multi_vo_tests_docker.sh index aeb8748177..9b139d2ee6 100755 --- a/tools/run_multi_vo_tests_docker.sh +++ b/tools/run_multi_vo_tests_docker.sh @@ -52,7 +52,7 @@ echo 'Removing old SQLite databases' rm -f /tmp/rucio.db echo 'Resetting database tables' -tools/reset_database.py +ALEMBIC_CONFIG="$RUCIO_HOME/etc/alembic.ini" tools/reset_database.py if [ $? != 0 ]; then echo 'Failed to reset the database!' exit 1 @@ -64,7 +64,7 @@ if [ -f /tmp/rucio.db ]; then fi echo 'Running full alembic migration' -ALEMBIC_CONFIG="/opt/rucio/etc/alembic.ini" tools/alembic_migration.sh +ALEMBIC_CONFIG="$RUCIO_HOME/etc/alembic.ini" tools/alembic_migration.sh if [ $? != 0 ]; then echo 'Failed to run alembic migration!' exit 1 diff --git a/tools/run_tests.sh b/tools/run_tests.sh index 0403fab314..d578052a60 100755 --- a/tools/run_tests.sh +++ b/tools/run_tests.sh @@ -14,41 +14,55 @@ # See the License for the specific language governing permissions and # limitations under the License. +memcached -u root -d + function usage { echo "Usage: $0 [OPTION]..." echo 'Run Rucio test suite' echo '' - echo '' echo ' -h Show usage' echo ' -2 Run tests twice' - echo ' -c Add pytest options' - echo ' -i Do only the initialization' - echo ' -p Also run pylint tests' + echo ' -p Indicate test selection by TESTS environment variable' echo ' -k Keep database from previous test' - echo ' -a Disable alembic tests' + echo ' -i Do only the initialization' + echo ' -l Also run lint tests' + echo ' -r Activate default RSEs (XRD1, XRD2, XRD3, SSH1)' + echo ' -s Run special tests for Dirac. Includes using BelleII schema' + echo ' -t Verbose output from pytest' + echo ' -a Skip alembic downgrade/upgrade test' echo ' -u Update pip dependencies only' - echo ' -x Stop running tests after the first problem' + echo ' -x exit instantly on first error or failed test' exit } alembic="true" iterations=1 -while getopts h2rcipkaux opt +while getopts h2cpkilrstaux opt do - case "$opt" in - h) usage;; - 2) iterations=2;; - c) pytestextra="$OPTARG";; - i) init_only="true";; - p) pylint="true";; - k) keep_db="true";; - a) alembic="";; - u) pip_only="true";; - x) stop_on_failure="--exitfirst";; - esac + case "$opt" in + h) usage;; + 2) iterations=2;; + p) selection="true";; + k) keep_db="true";; + i) init_only="true";; + l) lint="true";; + r) activate_rse="true";; + s) special="true";selection="true";TESTS="test_dirac.py";; + t) trace="true";; + a) alembic="";; + u) pip_only="true";; + x) stop_on_failure="--exitfirst";; + *) usae; exit 1; + esac done +if [ -z "$RUCIO_HOME" ]; then + RUCIO_HOME=/opt/rucio +fi +echo 'Clearing memcache' +echo flush_all > /dev/tcp/127.0.0.1/11211 + echo 'Update dependencies with pip' pip install --upgrade -r requirements.txt @@ -57,27 +71,43 @@ if test ${pip_only}; then fi echo 'Cleaning *.pyc files' -find lib -iname "*.pyc" | xargs rm +find lib -iname "*.pyc" -print0 | xargs -0 rm echo 'Cleaning old authentication tokens' rm -rf /tmp/.rucio_*/ -echo 'Cleaning storage for local test RSEs' +echo 'Cleaning local RSE directories' rm -rf /tmp/rucio_rse/* -echo 'Running flake8 code style checker' -flake8 --exclude=*.cfg bin/* lib/ tools/*.py -if [ $? != 0 ]; then - echo 'Checker failed, aborting.' - exit +if test ${special}; then + if [ -f "$RUCIO_HOME"/etc/rucio.cfg ]; then + echo 'Remove rucio.cfg' + rm "$RUCIO_HOME"/etc/rucio.cfg + fi + echo 'Using the special config' + ln -s "$RUCIO_HOME"/etc/rucio.cfg.special "$RUCIO_HOME"/etc/rucio.cfg +else + if [ -f "$RUCIO_HOME"/etc/rucio.cfg ]; then + echo 'Using the standard config' + else + echo 'rucio.cfg not found. Will try to do a symlink' + ln -s "$RUCIO_HOME"/etc/rucio.cfg.default "$RUCIO_HOME"/etc/rucio.cfg + fi fi -if test ${pylint}; then +if test ${lint}; then + echo 'Running flake8 code style checker' + flake8 --exclude=*.cfg bin/* lib/ tools/*.py + if [ $? != 0 ]; then + echo 'Checker failed, aborting.' + exit 1 + fi + echo 'Running pylint code style checker' pylint bin/* lib/ tools/*.py if [ $? != 0 ]; then - echo 'Checker failed, aborting.' - exit + echo 'Checker failed, aborting.' + exit 1 fi fi @@ -86,47 +116,91 @@ if test ${keep_db}; then else echo 'Resetting database tables' + echo 'Removing old SQLite databases' rm -f /tmp/rucio.db tools/reset_database.py if [ $? != 0 ]; then echo 'Failed to reset the database!' - exit + exit 1 fi if [ -f /tmp/rucio.db ]; then - echo 'Disable SQLite database access restriction' - chmod 777 /tmp/rucio.db + echo 'Disable SQLite database access restriction' + chmod 666 /tmp/rucio.db fi fi if test ${alembic}; then echo 'Running full alembic migration' - tools/alembic_migration.sh + ALEMBIC_CONFIG="$RUCIO_HOME/etc/alembic.ini" tools/alembic_migration.sh if [ $? != 0 ]; then echo 'Failed to run alembic migration!' exit 1 fi fi +echo 'Graceful restart of Apache' +httpd -k graceful + +echo 'Bootstrapping tests' +tools/bootstrap_tests.py +if [ $? != 0 ]; then + echo 'Failed to bootstrap!' + exit 1 +fi + echo 'Sync rse_repository' -tools/sync_rses.py +if test ${special}; then + tools/sync_rses.py etc/rse_repository.json.special + if [ $? != 0 ]; then + echo 'Failed to sync!' + exit 1 + fi +else + tools/sync_rses.py + if [ $? != 0 ]; then + echo 'Failed to sync!' + exit 1 + fi +fi echo 'Sync metadata keys' tools/sync_meta.py +if [ $? != 0 ]; then + echo 'Failed to sync!' + exit 1 +fi -echo 'Bootstrap tests: Create jdoe account/mock scope' -tools/bootstrap_tests.py +if test ${activate_rse}; then + echo 'Activating default RSEs (XRD1, XRD2, XRD3, SSH1)' + tools/docker_activate_rses.sh +fi if test ${init_only}; then exit fi +PYTEST_SH_ARGS="$stop_on_failure" +if test ${trace}; then + echo 'Running tests in verbose mode' + PYTEST_SH_ARGS="-vvv $PYTEST_SH_ARGS" +else + echo 'Running tests' + PYTEST_SH_ARGS="-v --tb=short $PYTEST_SH_ARGS" +fi + for i in $iterations do - echo 'Running test iteration' $i - echo tools/pytest.sh -v --tb=short $stop_on_failure $pytestextra - tools/pytest.sh -v --tb=short $stop_on_failure $pytestextra + + if test ${selection}; then + echo "Running the following tests:" + echo $TESTS | tr " " "\n" + else + TESTS='' fi + echo 'Running test iteration' $i + echo tools/pytest.sh $PYTEST_SH_ARGS $TESTS + tools/pytest.sh $PYTEST_SH_ARGS $TESTS done diff --git a/tools/run_tests_docker.sh b/tools/run_tests_docker.sh deleted file mode 100755 index 4d05030f1d..0000000000 --- a/tools/run_tests_docker.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- -# Copyright European Organization for Nuclear Research (CERN) since 2012 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -memcached -u root -d - -function usage { - echo "Usage: $0 [OPTION]..." - echo 'Run Rucio test suite' - echo '' - echo ' -h Show usage' - echo ' -i Do only the initialization' - echo ' -r Activate default RSEs (XRD1, XRD2, XRD3, SSH1)' - echo ' -s Run special tests for Dirac. Includes using BelleII schema' - echo ' -t Verbose output from pytest' - echo ' -a Skip alembic downgrade/upgrade test' - echo ' -x exit instantly on first error or failed test' - echo ' -p Indicate test selection by TESTS environment variable' - exit -} - -while getopts hirstaxp opt -do - case "$opt" in - h) usage;; - i) init_only="true";; - r) activate_rse="true";; - s) special="true";selection="true";TESTS="test_dirac.py";; - t) trace="true";; - a) noalembic="true";; - x) exitfirst="-x";; - p) selection="true";; - esac -done -export RUCIO_HOME=/opt/etc/test - -echo 'Clearing memcache' -echo flush_all > /dev/tcp/127.0.0.1/11211 - -echo 'Graceful restart of Apache' -httpd -k graceful - -echo 'Cleaning old authentication tokens' -rm -rf /tmp/.rucio_*/ - -echo 'Cleaning local RSE directories' -rm -rf /tmp/rucio_rse/* - -echo 'Removing old SQLite databases' -rm -f /tmp/rucio.db - -if test ${special}; then - if [ -f /opt/rucio/etc/rucio.cfg ]; then - echo 'Remove rucio.cfg' - rm /opt/rucio/etc/rucio.cfg - fi - echo 'Using the special config' - ln -s /opt/rucio/etc/rucio.cfg.special /opt/rucio/etc/rucio.cfg -else - if [ -f /opt/rucio/etc/rucio.cfg ]; then - echo 'Using the standard config' - else - echo 'rucio.cfg not found. Will try to do a symlink' - ln -s /opt/rucio/etc/rucio.cfg.default /opt/rucio/etc/rucio.cfg - fi -fi - -echo 'Resetting database tables' -tools/reset_database.py -if [ $? != 0 ]; then - echo 'Failed to reset the database!' - exit 1 -fi - -if [ -f /tmp/rucio.db ]; then - echo 'Disable SQLite database access restriction' - chmod 777 /tmp/rucio.db -fi - -if test ${noalembic}; then - echo "Skipping alembic migration" -else - echo "Running full alembic migration" - ALEMBIC_CONFIG="/opt/rucio/etc/alembic.ini" tools/alembic_migration.sh - if [ $? != 0 ]; then - echo 'Failed to run alembic migration!' - exit 1 - fi -fi - -echo 'Bootstrapping tests' -tools/bootstrap_tests.py -if [ $? != 0 ]; then - echo 'Failed to bootstrap!' - exit 1 -fi - -echo 'Sync rse_repository' -if test ${special}; then - tools/sync_rses.py etc/rse_repository.json.special - if [ $? != 0 ]; then - echo 'Failed to sync!' - exit 1 - fi -else - tools/sync_rses.py - if [ $? != 0 ]; then - echo 'Failed to sync!' - exit 1 - fi -fi - -echo 'Sync metadata keys' -tools/sync_meta.py -if [ $? != 0 ]; then - echo 'Failed to sync!' - exit 1 -fi - -if test ${activate_rse}; then - echo 'Activating default RSEs (XRD1, XRD2, XRD3, SSH1)' - tools/docker_activate_rses.sh -fi - -if test ${init_only}; then - exit -fi - -PYTEST_SH_ARGS="${exitfirst:-}" - -if test ${trace}; then - echo 'Running tests in verbose mode' - PYTEST_SH_ARGS="-vvv $PYTEST_SH_ARGS" -else - echo 'Running tests' - PYTEST_SH_ARGS="-v --tb=short $PYTEST_SH_ARGS" -fi - -if test ${selection}; then - echo "Running the following tests:" - echo $TESTS | tr " " "\n" - tools/pytest.sh $PYTEST_SH_ARGS $TESTS -else - tools/pytest.sh $PYTEST_SH_ARGS -fi - -exit $? diff --git a/tools/submit_fts3_transfer.py b/tools/submit_fts3_transfer.py deleted file mode 100755 index 8814ff4965..0000000000 --- a/tools/submit_fts3_transfer.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright European Organization for Nuclear Research (CERN) since 2012 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This submits a transfer to FTS3 via the transfertool. -""" - -import sys -import os.path -base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(base_path) -os.chdir(base_path) - -import time # noqa: E402 -from pprint import pprint # noqa: E402 - -import rucio.transfertool.fts3 # noqa: E402 -from rucio.common.utils import generate_uuid # noqa: E402 - -if __name__ == "__main__": - # token for OAuth 2.0 OIDC authorization scheme is working only with dCache + davs/https protocols (as of September 2019) - token = '' - FTS3_TransferTool = rucio.transfertool.fts3.FTS3Transfertool('https://fts3-xdc.cern.ch:8446', token) - files = [{"sources": ['https://dcache-xdc.desy.de/Users/jaroslav/tests/test.txt'], - "destinations": ['https://dcache-xdc.desy.de/Users/jaroslav/tests/test.txt-%s' % generate_uuid()], - "verify_checksum": False, - "overwrite": True, - "metadata": {'request_id': 'jwttest-%s' % generate_uuid()}, - "activity": "Rucio_Test_JWT_Authorisation"}] - job_params = {'request_id': generate_uuid()} - transfer_id = FTS3_TransferTool.submit(files, job_params, timeout=300) - print("transfer_id = ", transfer_id) - time.sleep(10) - status = FTS3_TransferTool.query([transfer_id]) - pprint(status) diff --git a/tools/test/before_script.sh b/tools/test/before_script.sh deleted file mode 100755 index c7925e6403..0000000000 --- a/tools/test/before_script.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- -# Copyright European Organization for Nuclear Research (CERN) since 2012 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -# Note: this file is not run when RUN_HTTPD is defined as false for the test suite. - -echo "* docker $CONTAINER_RUNTIME_ARGS info" -docker $CONTAINER_RUNTIME_ARGS info -echo -echo "* env" -env -echo - -RESTART_HTTPD=0 - -if [ $RDBMS == "oracle" ]; then - docker $CONTAINER_RUNTIME_ARGS cp tools/test/oracle_setup.sh ${CON_DB}:/ - date - ORACLE_STARTUP_STRING="DATABASE IS READY TO USE" - for i in {1..60}; do - sleep 2 - cont=$(bash -c 'docker '"$CONTAINER_RUNTIME_ARGS"' logs '"$CON_DB"' | grep "'"$ORACLE_STARTUP_STRING"'" | wc -l') - [ "$cont" -eq "1" ] && break - done - date - if [ "$cont" -ne "1" ]; then - echo "Oracle did not start up in time." - docker $CONTAINER_RUNTIME_ARGS logs $CON_DB || true - exit 1 - fi - sleep 3 - docker $CONTAINER_RUNTIME_ARGS exec $CON_DB /oracle_setup.sh - sleep 3 - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/rucio_oracle.cfg /opt/rucio/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/alembic_oracle.ini /opt/rucio/etc/alembic.ini - RESTART_HTTPD=1 - -elif [ $RDBMS == "mysql8" ]; then - date - for i in {1..30}; do - sleep 4 - cont=$(bash -c 'ping=`docker '"$CONTAINER_RUNTIME_ARGS"' exec '"$CON_DB"' mysqladmin --user=root --password=secret ping`; echo $ping 1>&2; echo $ping | grep "mysqld is alive" 1>&2; echo $?') - [ "$cont" -eq "0" ] && break - done - date - if [ "$cont" -ne "0" ]; then - echo Could not connect to MySQL in time. - exit 1 - fi - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/rucio_mysql8.cfg /opt/rucio/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/alembic_mysql8.ini /opt/rucio/etc/alembic.ini - RESTART_HTTPD=1 - -elif [ $RDBMS == "postgres14" ]; then - date - for i in {1..30}; do - sleep 1 - cont=$(bash -c 'docker '"$CONTAINER_RUNTIME_ARGS"' exec '"$CON_DB"' pg_isready 1>&2; echo $?') - [ "$cont" -eq "0" ] && break - done - date - if [ "$cont" -ne "0" ]; then - echo Could not connect to Postgres in time. - exit 1 - fi - if [ $SUITE == "multi_vo" ]; then - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO mkdir -p /opt/rucio/etc/multi_vo/tst/etc - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO mkdir -p /opt/rucio/etc/multi_vo/ts2/etc - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg /opt/rucio/etc/multi_vo/tst/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/rucio_multi_vo_ts2_postgres14.cfg /opt/rucio/etc/multi_vo/ts2/etc/rucio.cfg - fi - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/rucio_postgres14.cfg /opt/rucio/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/alembic_postgres14.ini /opt/rucio/etc/alembic.ini - RESTART_HTTPD=1 - -elif [ $RDBMS == "sqlite" ]; then - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/rucio_sqlite.cfg /opt/rucio/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/alembic_sqlite.ini /opt/rucio/etc/alembic.ini - RESTART_HTTPD=1 -fi - -if [ "$RESTART_HTTPD" == "1" ]; then - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO httpd -k restart -fi - -docker $CONTAINER_RUNTIME_ARGS ps -a diff --git a/tools/test/install_script.sh b/tools/test/install_script.sh deleted file mode 100755 index f68c688254..0000000000 --- a/tools/test/install_script.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- -# Copyright European Organization for Nuclear Research (CERN) since 2012 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -echo "* Using $(command -v python) $(python --version 2>&1) and $(command -v pip) $(pip --version 2>&1)" - -if [ "$SUITE" == "client" -o "$SUITE" == "client_syntax" ]; then - cd /usr/local/src/rucio - cp etc/docker/test/extra/rucio_client.cfg etc/rucio.cfg - -elif [ "$SUITE" == "syntax" -o "$SUITE" == "docs" ]; then - cd /usr/local/src/rucio - cp etc/docker/test/extra/rucio_syntax.cfg etc/rucio.cfg - -elif [ "$SUITE" == "votest" ]; then - RUCIO_HOME=/opt/rucio - VOTEST_HELPER=$RUCIO_HOME/tools/test/votest_helper.py - VOTEST_CONFIG_FILE=$RUCIO_HOME/etc/docker/test/matrix_policy_package_tests.yml - echo "VOTEST: Overriding policy section in rucio.cfg" - python $VOTEST_HELPER --vo $POLICY --vo-config --file $VOTEST_CONFIG_FILE - echo "VOTEST: Restarting httpd to load config" - httpd -k restart -fi diff --git a/tools/test/run_tests.py b/tools/test/run_tests.py index 68ec43f3e7..9c2abf578b 100755 --- a/tools/test/run_tests.py +++ b/tools/test/run_tests.py @@ -15,6 +15,8 @@ # limitations under the License. import json +import io +import itertools import multiprocessing import os import pathlib @@ -24,11 +26,32 @@ import time import traceback import uuid -from collections.abc import Callable from datetime import datetime -from typing import Optional +from tempfile import NamedTemporaryFile +from typing import Optional, Union, NoReturn -from suites import run, Container, rdbms_container, services, CumulativeContextManager, service_hostnames, env_args +import yaml + + +def run(*args, check=True, return_stdout=False, env=None) -> Union[NoReturn, io.TextIOBase]: + kwargs = {'check': check, 'stdout': sys.stderr, 'stderr': subprocess.STDOUT} + if env is not None: + kwargs['env'] = env + if return_stdout: + kwargs['stderr'] = sys.stderr + kwargs['stdout'] = subprocess.PIPE + args = [str(a) for a in args] + print("** Running", " ".join(map(lambda a: repr(a) if ' ' in a else a, args)), kwargs, file=sys.stderr, flush=True) + proc = subprocess.run(args, **kwargs) + if return_stdout: + return proc.stdout + + +def env_args(caseenv): + environment_args = list(itertools.chain(*map(lambda x: ('--env', f'{x[0]}={x[1]}'), caseenv.items()))) + environment_args.append('--env') + environment_args.append('GITHUB_ACTIONS') + return environment_args def matches(small: dict, group: dict): @@ -177,8 +200,6 @@ def run_case(caseenv, image, use_podman, use_namespace, use_httpd, copy_rucio_lo success = run_with_httpd( caseenv=caseenv, image=image, - use_podman=use_podman, - pod=pod, namespace_args=namespace_args, namespace_env=namespace_env, copy_rucio_logs=copy_rucio_logs, @@ -216,7 +237,6 @@ def run_test_directly( pod_net_arg = ['--pod', pod] if use_podman else [] scripts_to_run = ' && '.join( [ - './tools/test/install_script.sh', './tools/test/test.sh' + (' -p' if tests else ''), ] ) @@ -254,74 +274,63 @@ def run_test_directly( def run_with_httpd( caseenv: dict[str, str], image: str, - use_podman: bool, - pod: str, namespace_args: list[str], namespace_env: dict[str, str], copy_rucio_logs: bool, logs_dir: pathlib.Path, tests: list[str], ) -> bool: - pod_net_arg = ['--pod', pod] if use_podman else [] - # Running rucio container from given image - with Container(image, runtime_args=namespace_args, run_args=pod_net_arg, environment=caseenv) as rucio_container: + compose_version = int(run('docker-compose', 'version', '--short', return_stdout=True).decode().split('.')[0]) + + with (NamedTemporaryFile() as compose_override_file): + compose_override_content = yaml.dump({ + 'services': { + 'rucio': { + 'image': image, + 'environment': [f'{k}={v}' for k, v in caseenv.items()], + }, + 'ruciodb': { + 'profiles': ['donotstart'], + } + } + }) + print("Overriding docker-compose configuration with: \n", compose_override_content, flush=True) + with open(compose_override_file.name, 'w') as f: + f.write(compose_override_content) + + rdbms = caseenv.get('RDBMS', '') + project = os.urandom(8).hex() + up_down_args = ( + '--file', 'etc/docker/dev/docker-compose.yml', + '--file', compose_override_file.name, + '--profile', rdbms, + ) + rucio_container = None try: - network_arg = ('--network', 'container:' + rucio_container.cid) - container_run_args = pod_net_arg if use_podman else network_arg - additional_containers = [] - - def create_cnt(cnt_class: Callable) -> Container: - return cnt_class( - runtime_args=namespace_args, - run_args=container_run_args, - ) - - db_container = None - rdbms = caseenv.get('RDBMS', '') - if rdbms: - service_key = caseenv.get('SERVICES', 'default') - db_container_class = rdbms_container.get(rdbms, None) - if db_container_class: - db_container = create_cnt(db_container_class) - additional_containers.append(db_container) - additional_containers += list(map(create_cnt, services[service_key])) - - with CumulativeContextManager(*additional_containers): - db_env = dict() - if db_container: - db_env['CON_DB'] = db_container.cid - - # Running before_script.sh - run( - './tools/test/before_script.sh', - env={ - **os.environ, - **caseenv, - **namespace_env, - **db_env, - "CONTAINER_RUNTIME_ARGS": ' '.join(namespace_args), - "CON_RUCIO": rucio_container.cid, - }, - ) - - # register service hostnames - run('docker', *namespace_args, 'exec', rucio_container.cid, '/bin/sh', '-c', f'echo "127.0.0.1 {" ".join(service_hostnames)}" | tee -a /etc/hosts') - - # Running install_script.sh - run('docker', *namespace_args, 'exec', rucio_container.cid, './tools/test/install_script.sh') - - # Running test.sh - if tests: - tests_env = ('--env', 'TESTS=' + ' '.join(tests)) - tests_arg = ('-p', ) - else: - tests_env = () - tests_arg = () - - run('docker', *namespace_args, 'exec', *tests_env, rucio_container.cid, './tools/test/test.sh', *tests_arg) - - # if everything went through without an exception, mark this case as a success - return True + # Start docker compose + run('docker-compose', '-p', project, *up_down_args, 'up', '-d') + + # Retrieve container names from docker compose + # or use pre-defined names for old, v1, docker-compose + rucio_container = f'{project}_rucio_1' + if compose_version > 1: + rucio_container = next(filter( + lambda c: c['Service'] == 'rucio', + json.loads(run('docker-compose', '-p', project, 'ps', '--format', 'json', return_stdout=True)) + ), {}).get('Name', rucio_container) + + # Running test.sh + if tests: + tests_env = ('--env', 'TESTS=' + ' '.join(tests)) + tests_arg = ('-p', ) + else: + tests_env = () + tests_arg = () + + run('docker', *namespace_args, 'exec', *tests_env, rucio_container, './tools/test/test.sh', *tests_arg) + + # if everything went through without an exception, mark this case as a success + return True except subprocess.CalledProcessError as error: print( f"** Process '{error.cmd}' exited with code {error.returncode}", @@ -330,22 +339,24 @@ def create_cnt(cnt_class: Callable) -> Container: flush=True, ) finally: - run('docker', *namespace_args, 'logs', rucio_container.cid, check=False) - if copy_rucio_logs: - try: - if logs_dir.exists(): - shutil.rmtree(logs_dir) - run('docker', *namespace_args, 'cp', f'{rucio_container.cid}:/var/log', str(logs_dir)) - except Exception: - print( - "** Error on retrieving logs for", - {**caseenv, "IMAGE": image}, - '\n', - traceback.format_exc(), - '\n**', - file=sys.stderr, - flush=True, - ) + if rucio_container: + run('docker', *namespace_args, 'logs', rucio_container, check=False) + if copy_rucio_logs: + try: + if logs_dir.exists(): + shutil.rmtree(logs_dir) + run('docker', *namespace_args, 'cp', f'{rucio_container}:/var/log', str(logs_dir)) + except Exception: + print( + "** Error on retrieving logs for", + {**caseenv, "IMAGE": image}, + '\n', + traceback.format_exc(), + '\n**', + file=sys.stderr, + flush=True, + ) + run('docker-compose', '-p', project, *up_down_args, 'down', '-t', '30', check=False) return False diff --git a/tools/test/suites.py b/tools/test/suites.py deleted file mode 100644 index 5d802239c9..0000000000 --- a/tools/test/suites.py +++ /dev/null @@ -1,263 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright European Organization for Nuclear Research (CERN) since 2012 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import itertools -import subprocess -import sys -from typing import Any, NoReturn, Optional, Union - -DEFAULT_TIMEOUT = 10 -DEFAULT_DB_TIMEOUT = 27 - - -def run(*args, check=True, return_stdout=False, env=None) -> Union[NoReturn, io.TextIOBase]: - kwargs = {'check': check, 'stdout': sys.stderr, 'stderr': subprocess.STDOUT} - if env is not None: - kwargs['env'] = env - if return_stdout: - kwargs['stderr'] = sys.stderr - kwargs['stdout'] = subprocess.PIPE - args = [str(a) for a in args] - print("** Running", " ".join(map(lambda a: repr(a) if ' ' in a else a, args)), kwargs, file=sys.stderr, flush=True) - proc = subprocess.run(args, **kwargs) - if return_stdout: - return proc.stdout - - -def env_args(caseenv): - environment_args = list(itertools.chain(*map(lambda x: ('--env', f'{x[0]}={x[1]}'), caseenv.items()))) - environment_args.append('--env') - environment_args.append('GITHUB_ACTIONS') - return environment_args - - -class Container: - def __init__( - self, - image: "str", - *args, - runtime_args: Optional[list[str]] = None, - run_args: Optional[list[str]] = None, - environment: Optional[dict[str, str]] = None, - stop_timeout: int = DEFAULT_TIMEOUT, - ): - if runtime_args is None: - runtime_args = [] - self.runtime_args = runtime_args - if run_args is None: - run_args = [] - if environment is None: - environment = {} - self.stop_timeout = stop_timeout - self.args = ['docker', *runtime_args, 'run', '--detach', *run_args, *(env_args(environment)), image, *args] - self.cid = None - - def __enter__(self): - stdout = run(*self.args, return_stdout=True) - self.cid = stdout.decode().strip() - if not self.cid: - raise RuntimeError("Could not determine container id after starting the container") - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - run('docker', *self.runtime_args, 'stop', f'--time={self.stop_timeout}', self.cid, check=False) - run('docker', *self.runtime_args, 'rm', '--force', '--volumes', self.cid, check=False) - - def wait(self): - run('docker', *self.runtime_args, 'wait', self.cid, check=False) - - -class CumulativeContextManager: - def __init__(self, *context_managers): - self.context_managers = context_managers - - def __enter__(self): - for mgr in self.context_managers: - mgr.__enter__() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - for mgr in self.context_managers: - mgr.__exit__(exc_type, exc_val, exc_tb) - - -class OracleDB(Container): - def __init__( - self, - runtime_args: Optional[tuple[str]] = None, - run_args: Optional[tuple[str]] = None, - environment: Optional[dict[str, str]] = None, - stop_timeout: int = DEFAULT_DB_TIMEOUT, - ): - if run_args is None: - run_args = tuple() - run_args = ("--no-healthcheck",) + run_args - if environment is None: - environment = dict() - environment['processes'] = "1000" - environment["sessions"] = "1105" - environment["transactions"] = "1215" - environment["ORACLE_ALLOW_REMOTE"] = "true" - environment["ORACLE_PASSWORD"] = "oracle" - environment["ORACLE_DISABLE_ASYNCH_IO"] = "true" - super(OracleDB, self).__init__( - "docker.io/gvenzl/oracle-xe:18.4.0", - runtime_args=runtime_args, - run_args=run_args, - environment=environment, - stop_timeout=stop_timeout, - ) - - -class MySQL5(Container): - def __init__( - self, - runtime_args: Optional[tuple[str]] = None, - run_args: Optional[tuple[str]] = None, - environment: Optional[dict[str, str]] = None, - stop_timeout: int = DEFAULT_DB_TIMEOUT, - ): - if environment is None: - environment = dict() - environment["MYSQL_ROOT_PASSWORD"] = "secret" - environment["MYSQL_ROOT_HOST"] = "%" - super(MySQL5, self).__init__( - "docker.io/mysql/mysql-server:5.7", - runtime_args=runtime_args, - run_args=run_args, - environment=environment, - stop_timeout=stop_timeout, - ) - - -class MySQL8(Container): - def __init__( - self, - runtime_args: Optional[tuple[str]] = None, - run_args: Optional[tuple[str]] = None, - environment: Optional[dict[str, str]] = None, - stop_timeout: int = DEFAULT_DB_TIMEOUT, - ): - if environment is None: - environment = dict() - environment["MYSQL_ROOT_PASSWORD"] = "secret" - environment["MYSQL_ROOT_HOST"] = "%" - super(MySQL8, self).__init__( - "docker.io/mysql/mysql-server:8.0", - "--default-authentication-plugin=mysql_native_password", - "--character-set-server=latin1", - runtime_args=runtime_args, - run_args=run_args, - environment=environment, - stop_timeout=stop_timeout, - ) - - -class Postgres14(Container): - def __init__( - self, - runtime_args: Optional[tuple[str]] = None, - run_args: Optional[tuple[str]] = None, - environment: Optional[dict[str, str]] = None, - stop_timeout: int = DEFAULT_DB_TIMEOUT, - ): - if environment is None: - environment = dict() - environment["POSTGRES_PASSWORD"] = "secret" - super(Postgres14, self).__init__( - "docker.io/postgres:14", - "-c", "max_connections=300", - runtime_args=runtime_args, - run_args=run_args, - environment=environment, - stop_timeout=stop_timeout, - ) - - -class ActiveMQ(Container): - def __init__( - self, - runtime_args: Optional[tuple[str]] = None, - run_args: Optional[tuple[str]] = None, - environment: Optional[dict[str, str]] = None, - stop_timeout: int = DEFAULT_DB_TIMEOUT, - ): - super(ActiveMQ, self).__init__( - "docker.io/webcenter/activemq:latest", - runtime_args=runtime_args, - run_args=run_args, - environment=environment, - stop_timeout=stop_timeout, - ) - - -class InfluxDB(Container): - def __init__( - self, - runtime_args: Optional[tuple[str]] = None, - run_args: Optional[tuple[str]] = None, - environment: Optional[dict[str, str]] = None, - stop_timeout: int = DEFAULT_DB_TIMEOUT, - ): - if environment is None: - environment = dict() - environment["DOCKER_INFLUXDB_INIT_MODE"] = "setup" - environment["DOCKER_INFLUXDB_INIT_USERNAME"] = "myusername" - environment["DOCKER_INFLUXDB_INIT_PASSWORD"] = "passwordpasswordpassword" - environment["DOCKER_INFLUXDB_INIT_ORG"] = "rucio" - environment["DOCKER_INFLUXDB_INIT_BUCKET"] = "rucio" - environment["DOCKER_INFLUXDB_INIT_ADMIN_TOKEN"] = "mytoken" - super(InfluxDB, self).__init__( - "docker.io/influxdb:latest", - runtime_args=runtime_args, - run_args=run_args, - environment=environment, - stop_timeout=stop_timeout, - ) - - -class Elasticsearch(Container): - def __init__( - self, - runtime_args: Optional[tuple[str]] = None, - run_args: Optional[tuple[str]] = None, - environment: Optional[dict[str, str]] = None, - stop_timeout: int = DEFAULT_DB_TIMEOUT, - ): - if environment is None: - environment = dict() - environment["discovery.type"] = "single-node" - super(Elasticsearch, self).__init__( - "docker.elastic.co/elasticsearch/elasticsearch:6.4.2", - runtime_args=runtime_args, - run_args=run_args, - environment=environment, - stop_timeout=stop_timeout, - ) - - -rdbms_container: dict[str, Any] = { - "oracle": OracleDB, - "mysql5": MySQL5, - "mysql8": MySQL8, - "postgres14": Postgres14, - "sqlite": None, -} -services = { - 'default': [ActiveMQ], - 'influxdb_elastic': [ActiveMQ, InfluxDB, Elasticsearch], -} -service_hostnames = ['activemq', 'influxdb', 'elasticsearch'] + list(rdbms_container.keys()) diff --git a/tools/test/test.sh b/tools/test/test.sh index 59bc7d483b..fac52e8ff5 100755 --- a/tools/test/test.sh +++ b/tools/test/test.sh @@ -16,16 +16,53 @@ set -eo pipefail +echo "* Using $(command -v python) $(python --version 2>&1) and $(command -v pip) $(pip --version 2>&1)" + +SOURCE_PATH=/usr/local/src/rucio +CFG_PATH=/usr/local/src/rucio/etc/docker/test/extra/ +if [ -z "$RUCIO_HOME" ]; then + RUCIO_HOME=/opt/rucio +fi + function srchome() { - export RUCIO_HOME=/usr/local/src/rucio - cd $RUCIO_HOME + export RUCIO_HOME="$SOURCE_PATH" + cd $SOURCE_PATH } -if [ "$SUITE" == "syntax" ]; then +function wait_for_database() { + echo 'Waiting for database to be ready' + if ! python3 -c "from rucio.db.sqla.session import wait_for_database; wait_for_database()" + then + echo 'Cannot access database' + exit 1 + fi +} + +if [ "$SUITE" == "client" ]; then + tools/run_tests.sh -i + + cp "$SOURCE_PATH"/etc/docker/test/extra/rucio_client.cfg "$SOURCE_PATH"/etc/rucio.cfg + srchome + tools/pytest.sh -v --tb=short tests/test_clients.py tests/test_bin_rucio.py tests/test_module_import.py + +elif [ "$SUITE" == "client_syntax" ]; then + srchome + CLIENT_BIN_FILES="bin/rucio bin/rucio-admin" + SYNTAX_PYLINT_ARGS="$(tools/test/ignoretool.py --pylint)" + SYNTAX_FLAKE_ARGS="$(tools/test/ignoretool.py --flake8) $CLIENT_BIN_FILES tests/test_clients.py tests/test_bin_rucio.py tests/test_module_import.py" + export SYNTAX_PYLINT_ARGS + export SYNTAX_PYLINT_BIN_ARGS="$CLIENT_BIN_FILES" + export SYNTAX_FLAKE_ARGS + tools/test/check_syntax.sh + +elif [ "$SUITE" == "syntax" ]; then srchome tools/test/check_syntax.sh elif [ "$SUITE" == "docs" ]; then + python3 $SOURCE_PATH/tools/merge_rucio_configs.py --use-env \ + -s "$CFG_PATH"/rucio_autotests_common.cfg "$CFG_PATH"/rucio_sqlite.cfg \ + -d "$SOURCE_PATH"/etc/rucio.cfg srchome export RUCIO_CLIENT_API_OUTPUT="rucio_client_api.md" export RUCIO_REST_API_OUTPUT="rucio_rest_api.md" @@ -37,37 +74,42 @@ elif [ "$SUITE" == "docs" ]; then tools/generate_rest_api_doc.py > $REST_API_DOC_FILENAME tools/test/check_rest_api_documentation.sh $REST_API_DOC_FILENAME +elif [ "$SUITE" == "votest" ]; then + VOTEST_HELPER=$RUCIO_HOME/tools/test/votest_helper.py + VOTEST_CONFIG_FILE=$RUCIO_HOME/etc/docker/test/matrix_policy_package_tests.yml + echo "VOTEST: Overriding policy section in rucio.cfg" + python $VOTEST_HELPER --vo "$POLICY" --vo-config --file $VOTEST_CONFIG_FILE + echo "VOTEST: Restarting httpd to load config" + httpd -k restart -elif [[ "$SUITE" =~ ^client.* ]]; then - if [ "$SUITE" == "client" ]; then - tools/run_tests_docker.sh -i - fi + TESTS=$(python $VOTEST_HELPER --vo "$POLICY" --tests --file $VOTEST_CONFIG_FILE) + export TESTS + tools/run_tests.sh -p - srchome - if [ "$SUITE" == "client" ]; then - tools/pytest.sh -v --tb=short tests/test_clients.py tests/test_bin_rucio.py tests/test_module_import.py - elif [ "$SUITE" == "client_syntax" ]; then - CLIENT_BIN_FILES="bin/rucio bin/rucio-admin" - export SYNTAX_PYLINT_ARGS="$(tools/test/ignoretool.py --pylint)" - export SYNTAX_PYLINT_BIN_ARGS="$CLIENT_BIN_FILES" - export SYNTAX_FLAKE_ARGS="$(tools/test/ignoretool.py --flake8) $CLIENT_BIN_FILES tests/test_clients.py tests/test_bin_rucio.py tests/test_module_import.py" - tools/test/check_syntax.sh - fi +elif [ "$SUITE" == "multi_vo" ]; then + VO1_HOME="$RUCIO_HOME" + mkdir -p "$VO1_HOME/etc" + python3 $SOURCE_PATH/tools/merge_rucio_configs.py --use-env \ + -s "$CFG_PATH"/rucio_autotests_common.cfg "$CFG_PATH"/rucio_multi_vo_tst_postgres14.cfg \ + -d "$VO1_HOME"/etc/rucio.cfg + httpd -k restart -elif [ "$SUITE" == "remote_dbs" ] || [ "$SUITE" == "sqlite" ] || [ "$SUITE" == "py37py38" ]; then - if [ -n "$TESTS" ]; then - tools/run_tests_docker.sh -p - else - tools/run_tests_docker.sh - fi + VO2_HOME="$RUCIO_HOME/../ts2" + mkdir -p "$VO2_HOME/etc" + python3 $SOURCE_PATH/tools/merge_rucio_configs.py --use-env \ + -s "$CFG_PATH"/rucio_autotests_common.cfg "$CFG_PATH"/rucio_multi_vo_ts2_postgres14.cfg \ + -d "$VO2_HOME"/etc/rucio.cfg + + wait_for_database -elif [ "$SUITE" == "multi_vo" ]; then tools/run_multi_vo_tests_docker.sh -elif [ "$SUITE" == "votest" ]; then - RUCIO_HOME=/opt/rucio - VOTEST_HELPER=$RUCIO_HOME/tools/test/votest_helper.py - VOTEST_CONFIG_FILE=$RUCIO_HOME/etc/docker/test/matrix_policy_package_tests.yml - export TESTS=$(python $VOTEST_HELPER --vo $POLICY --tests --file $VOTEST_CONFIG_FILE) - tools/run_tests_docker.sh -p +elif [ "$SUITE" == "remote_dbs" ] || [ "$SUITE" == "sqlite" ]; then + wait_for_database + + if [ -n "$TESTS" ]; then + tools/run_tests.sh -p + else + tools/run_tests.sh + fi fi diff --git a/tools/with_venv.sh b/tools/with_venv.sh deleted file mode 100755 index 3345d6f611..0000000000 --- a/tools/with_venv.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- -# Copyright European Organization for Nuclear Research (CERN) since 2012 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -TOOLS=`dirname $0` -VENV=$TOOLS/../.venv -source $VENV/bin/activate && $@