refactor: Add edge test for aws assume role with web identity (#3839)

* refactor: Add edge test for aws assume role with web identity Signed-off-by: Xuanwo <[email protected]> * Make clippy happy Signed-off-by: Xuanwo <[email protected]> * Fix b2 Signed-off-by: Xuanwo <[email protected]> --------- Signed-off-by: Xuanwo <[email protected]>
apache · Dec 28, 2023 · 90d8d51 · 90d8d51
1 parent 2ae0a3d
commit 90d8d51
Show file tree

Hide file tree

Showing 7 changed files with 131 additions and 76 deletions.
diff --git a/.github/workflows/edge_test.yml b/.github/workflows/edge_test.yml
@@ -29,6 +29,7 @@ on:
       - "!core/src/docs/**"
       - "!core/src/services/**"
       - "core/src/services/fs/**"
+      - "core/src/services/s3/**"
       - ".github/workflows/edge_test.yml"
 
 jobs:
@@ -80,7 +81,7 @@ jobs:
           unzip chrome-linux64.zip
           cp -r chrome-linux64/ /tmp/chrome/
           cp -r chromedriver-linux64 /tmp/chrome/chromedriver
-      
+
       - name: Setup wasm-pack
         run: |
           cargo install wasm-pack
@@ -97,9 +98,42 @@ jobs:
           AWS_SECRET_ACCESS_KEY: "minioadmin"
           AWS_EC2_METADATA_DISABLED: "true"
         run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
-      
+
       - name: Test wasm
         working-directory: core/edge/s3_read_on_wasm
         run: |
           export PATH=$PATH:/tmp/chrome/chrome-linux64/:/tmp/chrome/chromedriver-linux64/
-          wasm-pack test --chrome --headless
+          wasm-pack test --chrome --headless
+
+  test_s3_aws_assume_role_with_web_identity:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+      - uses: actions/github-script@v7
+        id: id-token
+        with:
+          script: return await core.getIDToken("sts.amazonaws.com")
+          result-encoding: string
+      - name: Write ID token to file
+        working-directory: core/edge/s3_aws_assume_role_with_web_identity
+        run: echo "${{ steps.id-token.outputs.result }}" > web_identity_token
+
+      - name: Test
+        working-directory: core/edge/s3_aws_assume_role_with_web_identity
+        run: cargo run
+        env:
+          AWS_WEB_IDENTITY_TOKEN_FILE: web_identity_token
+          AWS_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing-assume
+          OPENDAL_TEST: s3
+          OPENDAL_S3_ROOT: CI/
+          OPENDAL_S3_BUCKET: opendal-testing
+          OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
+          OPENDAL_S3_REGION: ap-northeast-1
diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml b/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+edition = "2021"
+name = "edge_test_aws_s3_assume_role_with_web_identity"
+publish = false
+version = "0.0.0"
+
+license.workspace = true
+
+[dependencies]
+opendal = { workspace = true, features = ["tests"] }
+uuid = { version = "1", features = ["serde", "v4"] }
+tokio = { version = "1", features = ["full"] }
diff --git a/core/edge/s3_aws_assume_role_with_web_identity/README.md b/core/edge/s3_aws_assume_role_with_web_identity/README.md
@@ -0,0 +1,18 @@
+# S3 AWS Assume Role With Web Identity
+
+This edge test case is for AWS s3 services that authed by `Assume Role With Web Identity`.
+
+For setup, please configure bucket and OIDC correctly, for example:
+
+```yaml
+- uses: actions/github-script@v7
+  id: id-token
+  with:
+    script: return await core.getIDToken("sts.amazonaws.com")
+    result-encoding: string
+- name: Write ID token to file
+  run: echo "${{ steps.id-token.outputs.result }}" > core/tests/data/web_identity_token
+```
+
+And configure `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN`.
+
diff --git a/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs b/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use opendal::raw::tests::init_test_service;
+use opendal::{Result, Scheme};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let op = init_test_service()?.expect("service must be init");
+    assert_eq!(op.info().scheme(), Scheme::S3);
+
+    let result = op
+        .is_exist(&uuid::Uuid::new_v4().to_string())
+        .await
+        .expect("this operation should never return error");
+    assert!(!result, "the file must be not exist");
+
+    Ok(())
+}
diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs
@@ -301,7 +301,11 @@ impl Accessor for B2Backend {
                 // The max multipart size of b2 is 5 Gb.
                 //
                 // ref: <https://www.backblaze.com/docs/cloud-storage-large-files>
-                write_multi_max_size: Some(5 * 1024 * 1024 * 1024),
+                write_multi_max_size: if cfg!(target_pointer_width = "64") {
+                    Some(5 * 1024 * 1024 * 1024)
+                } else {
+                    Some(usize::MAX)
+                },
 
                 delete: true,
                 copy: true,