!157 fixbug

* Merge branch 'main' of gitee.com:kunpengcompute/boostkit-bigdata into main * fixbug * fixbug * fixbug * FIXBUG * fixbug * fixbug * fixbug * fix * Merge branch 'main' of gitee.com:kunpengcompute/boostkit-bigdata into main * fixbug * fixbug * oom * fixbug * fix bug * fix bug * fix commnet * update omnioperator/omniop-spark-extension-ock/cpp/src/shuffle/ock_splitter.cpp. * update omnioperator/omniop-spark-extension-ock/cpp/src/shuffle/ock_mer… * update omnioperator/omniop-spark-extension-ock/cpp/src/shuffle/ock_mer… * update omnioperator/omniop-spark-extension-ock/cpp/src/shuffle/ock_has… * update omnioperator/omniop-spark-extension-ock/cpp/src/jni/OckShuffleJ… * update omnioperator/omniop-spark-extension-ock/cpp/src/jni/OckShuffleJ… * update omnioperator/omniop-spark-extension-ock/cpp/src/shuffle/ock_mer…
kunpengcompute · Nov 14, 2022 · 7149f1f · 7149f1f
1 parent 839e631
commit 7149f1f
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 3 deletions.
diff --git a/omnioperator/omniop-spark-extension-ock/cpp/src/jni/OckShuffleJniReader.cpp b/omnioperator/omniop-spark-extension-ock/cpp/src/jni/OckShuffleJniReader.cpp
@@ -72,7 +72,7 @@ JNIEXPORT jlong JNICALL Java_com_huawei_ock_spark_jni_OckShuffleJniReader_make(J
     return gBlobReader.Insert(instance);
 }
 
-JNIEXPORT void JNICALL Java_com_huawei_ock_spark_jni_OckShuffleJniReader_close(JNIEnv *, jobject, jlong)
+JNIEXPORT void JNICALL Java_com_huawei_ock_spark_jni_OckShuffleJniReader_close(JNIEnv *env, jobject, jlong jReaderId)
 {
     if (UNLIKELY(env == nullptr)) {
         LOG_ERROR("JNIENV is null.");

diff --git a/omnioperator/omniop-spark-extension-ock/cpp/src/jni/OckShuffleJniReader.h b/omnioperator/omniop-spark-extension-ock/cpp/src/jni/OckShuffleJniReader.h
@@ -23,7 +23,7 @@ JNIEXPORT jlong JNICALL Java_com_huawei_ock_spark_jni_OckShuffleJniReader_make(J
  * Method:    close
  * Signature: (JI)I
  */
-JNIEXPORT void JNICALL Java_com_huawei_ock_spark_jni_OckShuffleJniReader_close(JNIEnv *env, jobject, jlong jReaderId);
+JNIEXPORT void JNICALL Java_com_huawei_ock_spark_jni_OckShuffleJniReader_close(JNIEnv *, jobject, jlong);
 /*
  * Class:     com_huawei_ock_spark_jni_OckShuffleJniReader
  * Method:    nativeGetVectorBatch

diff --git a/...mniop-spark-extension-ock/src/main/java/com/huawei/ock/spark/jni/OckShuffleJniReader.java b/...mniop-spark-extension-ock/src/main/java/com/huawei/ock/spark/jni/OckShuffleJniReader.java
@@ -150,6 +150,14 @@ public void copyVectorDataInVB(Vec dstVec, int colIndex) {
         nativeCopyVecDataInVB(nativeReader, dstVec.getNativeVector(), colIndex);
     }
 
+    /**
+     * close reader.
+     *
+     */
+    public void doClose() {
+        close(nativeReader);
+    }
+
     private native long make(int[] typeIds);
 
     private native long close(long readerId);

diff --git a/...on-ock/src/main/scala/org/apache/spark/shuffle/ock/OckColumnarShuffleBufferIterator.scala b/...on-ock/src/main/scala/org/apache/spark/shuffle/ock/OckColumnarShuffleBufferIterator.scala
@@ -52,7 +52,7 @@ class OckColumnarShuffleBufferIterator[T](
             mapTaskToHostInfo.setNativeObjHandle(0)
         }
         blobMap.values.foreach(reader => {
-            reader.close()
+            reader.doClose()
         })
     }