diff --git a/.github/workflows/full-docker-image.yml b/.github/workflows/full-docker-image.yml new file mode 100644 index 0000000..c40dea1 --- /dev/null +++ b/.github/workflows/full-docker-image.yml @@ -0,0 +1,36 @@ +name: Build test OCI Image + +on: + push: + branches: + - main + - dockerfile +env: + REGISTRY: ghcr.io + IMAGE_NAME: magalucloud/s3-specs +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Login to Github Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: "Build and push OCI image, tag: full" + uses: docker/build-push-action@v5.3.0 + with: + context: . + push: true + file: oci/full.Containerfile + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:full + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:full-${{github.sha}} \ No newline at end of file diff --git a/bin/configure_profiles.py b/bin/configure_profiles.py index 11e5425..0c9e496 100644 --- a/bin/configure_profiles.py +++ b/bin/configure_profiles.py @@ -78,6 +78,10 @@ def configure_profiles(profiles): if not (endpoint and access_key and secret_key and region): print(f"Perfil {profile_name} está incompleto. Ignorando...") continue + + if (access_key == "YOUR-KEY-ID-HERE" or secret_key == "YOUR-SECRET-KEY-HERE" ): + print(f"Perfil {profile_name} está incompleto. Ignorando...") + continue set_aws_profiles(profile_name=profile_name, data=profile_data) set_rclone_profiles(profile_name=profile_name, data=profile_data) @@ -91,7 +95,18 @@ def configure_profiles(profiles): if __name__ == "__main__": if len(sys.argv) > 1: with open(sys.argv[1], 'r') as file: - profiles = yaml.safe_load(file) + profiles_list = yaml.safe_load(file) + profiles = {} + for profile in list(profiles_list.items())[2][1]: + if not isinstance(profile, dict): + profiles = profiles_list + break + profiles[profile.get("profile_name")] = {} + profiles[profile.get("profile_name")]["endpoint"] = profile.get("endpoint_url") + profiles[profile.get("profile_name")]["access_key"] = profile.get("aws_access_key_id") + profiles[profile.get("profile_name")]["secret_key"] = profile.get("aws_secret_access_key") + profiles[profile.get("profile_name")]["region"] = profile.get("region_name") + else: profiles_data = os.getenv("PROFILES") if not profiles_data: @@ -101,4 +116,4 @@ def configure_profiles(profiles): print(f"Number of profiles - {len(profiles)}") configure_profiles(profiles) - print(f"Profile Configurations Done!") + print("Profile Configurations Done!") diff --git a/oci/README.md b/oci/README.md new file mode 100644 index 0000000..595a1a4 --- /dev/null +++ b/oci/README.md @@ -0,0 +1,64 @@ +# Running S3 Specs from OCI image (Docker or Podman) + +## 1. Create a config file with your endpoints and credentials, +following the yaml format in profiles.example.yaml + +```bash +wget https://raw.githubusercontent.com/MagaluCloud/s4-specs/refs/heads/main/params.example.yaml -O params.yaml +vim params.yaml +# Edit to add this values on the number of profiles your tests needs +# (policy and acl tests for example needs 2 different profiles): +# region_name: "br-ne1" +# endpoint_url: "https://br-se1.magaluobjects.com/" +# aws_access_key_id: "YOUR-KEY-ID-HERE" +# aws_secret_access_key: "YOUR-SECRET-KEY-HERE" +``` + +### 1.1. Optional: build the image locally + +If you cloned the repo and want to build the image locally instead of using the published ones, +use the Containerfiles under the `oci` folder: + +```bash +IMAGE_NAME=myimage podman build -t "$IMAGE_NAME" -f ./oci/full.Containerfile . +``` + +## 2. Run all tests (this might take several minutes) + +```bash +# $IMAGE_NAME is the local tag name, if you build the Containerfile yourself, +# or the published one like https://ghcr.io/magalucloud/s3-specs:full +# +# all examples can be run with docker as well +podman run -t \ + -v $(realpath params.yaml):/app/params.yaml \ + "$IMAGE_NAME" \ + -n auto +``` + +> ::note:: Whatever comes after the image name (`-n auto` in the example above) +are pytest arguments, see `oci/entrypoint.sh` for details. The next examples are +variations of test filters using the markers feature of pytest (`-m` argument). + +## 3. Run all tests, excluding specific categories + +In following example we use the `-m` arg with the `not` operator to exclude two categories: + +```bash +podman run -t \ + -v $(realpath params.yaml):/app/params.yaml \ + "$IMAGE_NAME" \ + -n auto -m "not locking not slow" +``` + +## 4. Run only some specific categories + +You can pass pytest arguments to the image entrypoint, for example, you can filter tests of +specific categories, using the `-m` arg for markers, like: + +```bash +podman run -t \ + -v $(realpath params.yaml):/app/params.yaml \ + "$IMAGE_NAME" \ + -n auto -m "basic cold_storage" +``` diff --git a/oci/entrypoint.sh b/oci/entrypoint.sh new file mode 100755 index 0000000..9cd26ae --- /dev/null +++ b/oci/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +#configure profiles +uv run ./bin/configure_profiles.py params.yaml + +# Executar o comando passado para o contêiner +cd docs +exec uv run pytest *_test.py --config ../params.example.yaml "$@" diff --git a/oci/full.Containerfile b/oci/full.Containerfile new file mode 100644 index 0000000..cb5bd98 --- /dev/null +++ b/oci/full.Containerfile @@ -0,0 +1,65 @@ +# s3-specs-full +# +# This Container have: +# - all 3 CLIs (rclone, aws, mgc) +# - Python dependencies to run boto3 based tests. + +ARG AWS_CLI_VERSION="2.15.27" +ARG RCLONE_VERSION="1.66.0" +ARG MGC_VERSION="0.34.1" + +# aws-cli +FROM public.ecr.aws/aws-cli/aws-cli:${AWS_CLI_VERSION} as awscli + +# Main image +FROM ubuntu:latest +RUN apt-get update && \ + apt-get install -y \ + curl \ + unzip \ + python3 \ + git; + +# directory to download binaries +RUN mkdir -p /tools; + +# rclone +ARG RCLONE_VERSION +RUN curl -Lo rclone.zip "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.zip" && \ + unzip -q rclone.zip && rm rclone.zip && \ + mv rclone-v${RCLONE_VERSION}-linux-amd64 /tools/ && \ + ln -s "/tools/rclone-v${RCLONE_VERSION}-linux-amd64/rclone" /usr/local/bin/rclone; + +# aws cli +ARG AWS_CLI_VERSION +COPY --from=awscli /usr/local/aws-cli/ /tools/aws-cli/ +RUN ln -s "/tools/aws-cli/v2/${AWS_CLI_VERSION}/bin/aws" /usr/local/bin/aws && \ + ln -s "/tools/aws-cli/v2/${AWS_CLI_VERSION}/bin/aws_completer" /usr/local/bin/aws_completer; + +# mgc cli +ARG MGC_VERSION +RUN curl -Lo mgc.tar.gz "https://github.com/MagaluCloud/mgccli/releases/download/v${MGC_VERSION}/mgccli_${MGC_VERSION}_linux_amd64.tar.gz" && \ + tar xzvf mgc.tar.gz && rm mgc.tar.gz && \ + ln -s "/tools/mgc" /usr/local/bin/mgc; + +# uv +RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ + ln -s $HOME/.local/bin/uv /usr/local/bin/uv; + +# container workdir +WORKDIR /app + +# Required files to execute the tests +COPY bin /app/bin/ +COPY params.example.yaml /app/params.example.yaml +COPY oci/entrypoint.sh /app/oci/entrypoint.sh +COPY uv.lock /app/uv.lock +COPY pyproject.toml /app/pyproject.toml +COPY README.md /app/README.md +COPY docs /app/docs/ + +# Download python dependencies to be bundled with the image +RUN uv sync + +# Definir o script como ponto de entrada +ENTRYPOINT ["./oci/entrypoint.sh"]