Merge remote-tracking branch 'upstream/main' into prometheus-client-c…

…onsistent-apis

.github/services/hdfs/hdfs_cluster/action.yml

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: hdfs_cluster
+description: 'Behavior test for hdfs cluster'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup HDFS cluster
+      shell: bash
+      working-directory: fixtures/hdfs
+      run: docker compose -f docker-compose-hdfs-cluster.yml up -d --wait
+    - name: Setup java env
+      uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: "11"
+    - name: Setup hadoop env
+      shell: bash
+      run: |
+        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
+        export HADOOP_HOME=/home/runner/hadoop-3.3.5
+        echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV
+        echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV 
+        echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV
+        cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+    - name: Setup opendal env
+      shell: bash
+      run: |
+        cat << EOF >> $GITHUB_ENV
+        OPENDAL_HDFS_ROOT=/tmp/opendal/
+        OPENDAL_HDFS_NAME_NODE=hdfs://localhost:8020
+        OPENDAL_HDFS_ENABLE_APPEND=true
+        EOF

.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: hdfs_cluster_with_atomic_write_dir
+description: 'Behavior test for hdfs cluster with atomic write dir'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup HDFS cluster
+      shell: bash
+      working-directory: fixtures/hdfs
+      run: docker compose -f docker-compose-hdfs-cluster.yml up -d --wait
+    - name: Setup java env
+      uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: "11"
+    - name: Setup hadoop env
+      shell: bash
+      run: |
+        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
+        export HADOOP_HOME=/home/runner/hadoop-3.3.5
+        echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV
+        echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV 
+        echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV
+        cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+    - name: Setup opendal env
+      shell: bash
+      run: |
+        cat << EOF >> $GITHUB_ENV
+        OPENDAL_HDFS_ROOT=/tmp/opendal/
+        OPENDAL_HDFS_ATOMIC_WRITE_DIR=/tmp/atomic_write_dir/opendal/
+        OPENDAL_HDFS_NAME_NODE=hdfs://localhost:8020
+        OPENDAL_HDFS_ENABLE_APPEND=false
+        EOF

.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: hdfs_default_with_atomic_write_dir
+description: 'Behavior test for hdfs default with atomic write dir'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup java env
+      uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: "11"
+    - name: Setup
+      shell: bash
+      run: |
+        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
+
+        export HADOOP_HOME="/home/runner/hadoop-3.3.5"
+        export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
+
+        cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+
+        cat << EOF >> $GITHUB_ENV
+        HADOOP_HOME=${HADOOP_HOME}
+        CLASSPATH=${CLASSPATH}
+        LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
+        OPENDAL_HDFS_ROOT=/tmp/opendal/
+        OPENDAL_HDFS_ATOMIC_WRITE_DIR=/tmp/atomic_write_dir/opendal/
+        OPENDAL_HDFS_NAME_NODE=default
+        OPENDAL_HDFS_ENABLE_APPEND=false
+        EOF

core/src/layers/mod.rs

@@ -62,7 +62,7 @@ pub use self::mime_guess::MimeGuessLayer;
 #[cfg(feature = "layers-prometheus")]
 mod prometheus;
 #[cfg(feature = "layers-prometheus")]
-pub use self::prometheus::PrometheusLayer;
+pub use self::prometheus::{PrometheusLayer, PrometheusLayerBuilder};
 
 #[cfg(feature = "layers-prometheus-client")]
 mod prometheus_client;

core/src/layers/observe/mod.rs

@@ -18,6 +18,17 @@
 //! OpenDAL Observability Layer
 //!
 //! This module offers essential components to facilitate the implementation of observability in OpenDAL.
+//!
+//! # Prometheus Metrics
+//!
+//! These metrics are essential for understanding the behavior and performance of our applications.
+//!
+//! | Metric Name                  | Type      | Description                                                  | Labels                                          |
+//! |------------------------------|-----------|--------------------------------------------------------------|-------------------------------------------------|
+//! | operation_duration_seconds   | Histogram | Histogram of time spent during opendal operations            | scheme, namespace, root, operation, path        |
+//! | operation_bytes.             | Histogram | Histogram of the bytes transferred during opendal operations | scheme, operation, root, operation, path        |
+//! | operation_errors_total       | Counter   | Error counter during opendal operations                      | scheme, operation, root, operation, path, error |
+//!
 
 mod metrics;
 
@@ -40,16 +51,16 @@ pub use metrics::METRIC_OPERATION_ERRORS_TOTAL;
 /// - level = 0: return `None`, which means we ignore the path label.
 /// - level > 0: the path label will be the path split by "/" and get the last n level,
 ///   if n=1 and input path is "abc/def/ghi", and then we'll use "abc/" as the path label.
-pub fn path_label_value(path: &str, path_level: usize) -> Option<&str> {
+pub fn path_label_value(path: &str, level: usize) -> Option<&str> {
     if path.is_empty() {
         return None;
     }
 
-    if path_level > 0 {
+    if level > 0 {
         let label_value = path
             .char_indices()
             .filter(|&(_, c)| c == '/')
-            .nth(path_level - 1)
+            .nth(level - 1)
             .map_or(path, |(i, _)| &path[..i]);
         Some(label_value)
     } else {