feat(services/swift): add support for storage_url configuration in sw…

…ift service (#4302) * update docs.md and add compatible_services.md * update .env.example * add storage_url support for swift * fix cargo fmt check * add swift_with_storage_url test setup * fix cargo clippy error * change endpoint definition and remove account * remove storage_url test setup
apache · Mar 2, 2024 · 5f1d5d1 · 5f1d5d1
1 parent 4d5df20
commit 5f1d5d1
Show file tree

Hide file tree

Showing 6 changed files with 81 additions and 67 deletions.
diff --git a/.env.example b/.env.example
@@ -158,7 +158,6 @@ OPENDAL_AZFILE_ACCOUNT_KEY=<account_key>
 OPENDAL_AZFILE_SHARE_NAME=<shared_name>
 # openstack swift
 OPENDAL_SWIFT_ENDPOINT=<endpoint>
-OPENDAL_SWIFT_ACCOUNT=<account>
 OPENDAL_SWIFT_CONTAINER=<container>
 OPENDAL_SWIFT_ROOT=/path/to/dir
 OPENDAL_SWIFT_TOKEN=<token>

diff --git a/.github/services/swift/swift/action.yml b/.github/services/swift/swift/action.yml
@@ -16,7 +16,7 @@
 # under the License.
 
 name: swift
-description: "Behavior test for OpenStack Swift"
+description: "Behavior test for OpenStack Swift."
 
 runs:
   using: "composite"
@@ -27,19 +27,20 @@ runs:
       run: |
         docker compose -f docker-compose-swift.yml up -d --wait
 
-    - name: Create test token and setup test container
+    - name: Create environment variables and setup test container
       shell: bash
       run: |
-        token=$(curl -i -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0 | grep X-Auth-Token | head -n1 | awk '{print $2}')
-        curl --location --request PUT 'http://127.0.0.1:8080/v1/AUTH_test/testing' --header "X-Auth-Token: $token"
-        echo "OPENDAL_SWIFT_TOKEN=$(echo "$token" | tr -d '[:space:]')" >> $GITHUB_ENV
+        response=$(curl -i -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0)
+        token=$(echo "$response" | grep X-Auth-Token | head -n1 | awk '{print $2}' | tr -d '[:space:]')
+        endpoint=$(echo "$response" | grep X-Storage-Url | head -n1 | awk '{print $2}' | tr -d '[:space:]')
+        curl --location --request PUT "${endpoint}/testing" --header "X-Auth-Token: $token"
+        echo "OPENDAL_SWIFT_TOKEN=${token}" >> $GITHUB_ENV
+        echo "OPENDAL_SWIFT_ENDPOINT=${endpoint}" >> $GITHUB_ENV
 
     - name: Set environment variables
       shell: bash
       run: |
         cat << EOF >> $GITHUB_ENV
-        OPENDAL_SWIFT_ENDPOINT=http://127.0.0.1:8080
-        OPENDAL_SWIFT_ACCOUNT=AUTH_test
         OPENDAL_SWIFT_CONTAINER=testing
         OPENDAL_SWIFT_ROOT=/
         EOF
diff --git a/core/src/services/swift/backend.rs b/core/src/services/swift/backend.rs
@@ -36,7 +36,6 @@ use crate::*;
 #[derive(Default, Clone)]
 pub struct SwiftBuilder {
     endpoint: Option<String>,
-    account: Option<String>,
     container: Option<String>,
     root: Option<String>,
     token: Option<String>,
@@ -48,7 +47,6 @@ impl Debug for SwiftBuilder {
 
         ds.field("root", &self.root);
         ds.field("endpoint", &self.endpoint);
-        ds.field("account", &self.account);
         ds.field("container", &self.container);
 
         if self.token.is_some() {
@@ -64,8 +62,9 @@ impl SwiftBuilder {
     ///
     /// Endpoints should be full uri, e.g.
     ///
-    /// - `https://openstack-controller.example.com:8080`
-    /// - `http://192.168.66.88:8080`
+    /// - `http://127.0.0.1:8080/v1/AUTH_test`
+    /// - `http://192.168.66.88:8080/swift/v1`
+    /// - `https://openstack-controller.example.com:8080/v1/ccount`
     ///
     /// If user inputs endpoint without scheme, we will
     /// prepend `https://` to it.
@@ -78,18 +77,6 @@ impl SwiftBuilder {
         self
     }
 
-    /// Set account of this backend.
-    ///
-    /// It is required. e.g. `TEST_account`
-    pub fn account(&mut self, account: &str) -> &mut Self {
-        self.account = if account.is_empty() {
-            None
-        } else {
-            Some(account.trim_end_matches('/').to_string())
-        };
-        self
-    }
-
     /// Set container of this backend.
     ///
     /// All operations will happen under this container. It is required. e.g. `snapshots`
@@ -132,7 +119,6 @@ impl Builder for SwiftBuilder {
         let mut builder = SwiftBuilder::default();
 
         map.get("endpoint").map(|v| builder.endpoint(v));
-        map.get("account").map(|v| builder.account(v));
         map.get("container").map(|v| builder.container(v));
         map.get("token").map(|v| builder.token(v));
 
@@ -163,17 +149,6 @@ impl Builder for SwiftBuilder {
         };
         debug!("backend use endpoint: {}", &endpoint);
 
-        let account = match self.account.take() {
-            Some(account) => account,
-            None => {
-                return Err(Error::new(
-                    ErrorKind::ConfigInvalid,
-                    "missing account name for Swift",
-                ));
-            }
-        };
-        debug!("backend use account: {}", &account);
-
         let container = match self.container.take() {
             Some(container) => container,
             None => {
@@ -197,7 +172,6 @@ impl Builder for SwiftBuilder {
             core: Arc::new(SwiftCore {
                 root,
                 endpoint,
-                account,
                 container,
                 token,
                 client,

diff --git a/core/src/services/swift/compatible_services.md b/core/src/services/swift/compatible_services.md
@@ -0,0 +1,53 @@
+## Compatible Services
+
+### OpenStack Swift
+
+[OpenStack Swift](https://docs.openstack.org/swift/latest/) is the default implementations of swift servcies.
+
+To connect to OpenStack Swift, we need to set:
+
+- `endpoint`: The endpoint of OpenStack Swift, for example: `http://127.0.0.1:8080/v1/AUTH_test`.
+- `container`: The name of OpenStack Swift container.
+- `token`: OpenStack Swift container personal access token.
+
+```rust,ignore
+builder.endpoint("http://127.0.0.1:8080/v1/AUTH_test");
+builder.container("container");
+builder.token("token");
+```
+
+`endpoint` is the full URL that serves as the access point to all containers under an OpenStack Swift account. It represents the entry point for accessing the resources of the account. Alongside `endpoint`, `token` is used as a credential to verify the user's identity and authorize access to the relevant resources. Both `endpoint` and `token` can be obtained through OpenStack Swift authentication service.
+
+`endpoint` consists of server address and port, API version, authenticated account ID. For instance, it might appear as follows:
+
+- `http://127.0.0.1:8080/v1/AUTH_test`.
+- `http://192.168.66.88:8080/swift/v1`.
+- `https://openstack-controller.example.com:8080/v1/account`.
+
+Please note that the exact format of `endpoint` may vary depending on the deployment configuration and version of swift services. Users can refer to the specific services documentation for the correct `endpoint` format and authentication method.
+
+For more information, refer:
+
+- [OpenStack Swift API](https://docs.openstack.org/api-ref/object-store/).
+- [OpenStack Swift Authentication](https://docs.openstack.org/swift/latest/api/object_api_v1_overview.html).
+
+### Ceph Rados Gateway
+
+[Ceph Rados Gateway](https://docs.ceph.com/en/quincy/radosgw/) supports a RESTful API that is compatible with the basic data access model of OpenStack Swift API.
+
+To connect to Ceph Rados Gateway, we need to set:
+
+- `endpoint`: The endpoint of swift services, for example: `http://127.0.0.1:8080/swift/v1`.
+- `container`: The name of swift container.
+- `token`: swift container personal access token.
+
+```rust,ignore
+builder.endpoint("http://127.0.0.1:8080/swift/v1");
+builder.container("container");
+builder.token("token");
+```
+
+For more information, refer:
+
+- [Ceph Rados Gateway Swift API](https://docs.ceph.com/en/latest/radosgw/swift/#api).
+- [Ceph Rados Gateway Swift Authentication](https://docs.ceph.com/en/latest/radosgw/swift/auth/).
diff --git a/core/src/services/swift/core.rs b/core/src/services/swift/core.rs
@@ -28,7 +28,6 @@ use crate::*;
 pub struct SwiftCore {
     pub root: String,
     pub endpoint: String,
-    pub account: String,
     pub container: String,
     pub token: String,
     pub client: HttpClient,
@@ -39,7 +38,6 @@ impl Debug for SwiftCore {
         f.debug_struct("SwiftCore")
             .field("root", &self.root)
             .field("endpoint", &self.endpoint)
-            .field("account", &self.account)
             .field("container", &self.container)
             .finish_non_exhaustive()
     }
@@ -50,10 +48,9 @@ impl SwiftCore {
         let p = build_abs_path(&self.root, path);
 
         let url = format!(
-            "{}/v1/{}/{}/{}",
-            self.endpoint,
-            self.account,
-            self.container,
+            "{}/{}/{}",
+            &self.endpoint,
+            &self.container,
             percent_encode_path(&p)
         );
 
@@ -80,10 +77,9 @@ impl SwiftCore {
         // The delimiter is used to disable recursive listing.
         // Swift returns a 200 status code when there is no such pseudo directory in prefix.
         let mut url = format!(
-            "{}/v1/{}/{}/?prefix={}&delimiter={}&format=json",
-            self.endpoint,
-            self.account,
-            self.container,
+            "{}/{}/?prefix={}&delimiter={}&format=json",
+            &self.endpoint,
+            &self.container,
             percent_encode_path(&p),
             delimiter
         );
@@ -115,10 +111,9 @@ impl SwiftCore {
         let p = build_abs_path(&self.root, path);
 
         let url = format!(
-            "{}/v1/{}/{}/{}",
-            self.endpoint,
-            self.account,
-            self.container,
+            "{}/{}/{}",
+            &self.endpoint,
+            &self.container,
             percent_encode_path(&p)
         );
 
@@ -140,10 +135,9 @@ impl SwiftCore {
             .to_string();
 
         let url = format!(
-            "{}/v1/{}/{}/{}",
-            self.endpoint,
-            self.account,
-            self.container,
+            "{}/{}/{}",
+            &self.endpoint,
+            &self.container,
             percent_encode_path(&p)
         );
 
@@ -180,10 +174,9 @@ impl SwiftCore {
             .to_string();
 
         let url = format!(
-            "{}/v1/{}/{}/{}",
-            self.endpoint,
-            self.account,
-            self.container,
+            "{}/{}/{}",
+            &self.endpoint,
+            &self.container,
             percent_encode_path(&dst_p)
         );
 
@@ -208,9 +201,8 @@ impl SwiftCore {
         let p = build_abs_path(&self.root, path);
 
         let url = format!(
-            "{}/v1/{}/{}/{}",
+            "{}/{}/{}",
             &self.endpoint,
-            &self.account,
             &self.container,
             percent_encode_path(&p)
         );

diff --git a/core/src/services/swift/docs.md b/core/src/services/swift/docs.md
@@ -1,5 +1,3 @@
-This service will visit the [Swift API](https://docs.openstack.org/api-ref/object-store/) supported by [OpenStack Object Storage](https://docs.openstack.org/swift/latest/).
-
 ## Capabilities
 
 This service can be used to:
@@ -19,7 +17,6 @@ This service can be used to:
 ## Configurations
 
 - `endpoint`: Set the endpoint for backend.
-- `account_name`: Name of Swift account.
 - `container`: Swift container.
 - `token`: Swift personal access token.
 
@@ -44,9 +41,7 @@ async fn main() -> Result<()> {
     // Set the root for swift, all operations will happen under this root
     builder.root("/path/to/dir");
     // set the endpoint of Swift backend
-    builder.endpoint("https://openstack-controller.example.com:8080");
-    // set the account name of Swift workspace
-    builder.account_name("account");
+    builder.endpoint("https://openstack-controller.example.com:8080/v1/account");
     // set the container name of Swift workspace
     builder.container("container");
     // set the auth token for builder