Add test with null bitmap

tests/ffi.test.ts

@@ -2,7 +2,12 @@ import { readFileSync } from "fs";
 import { describe, it, expect } from "vitest";
 import * as arrow from "apache-arrow";
 import * as wasm from "rust-arrow-ffi";
-import { arrowTableToFFI, arraysEqual, loadIPCTableFromDisk } from "./utils";
+import {
+  arrowTableToFFI,
+  arraysEqual,
+  loadIPCTableFromDisk,
+  validityEqual,
+} from "./utils";
 import { parseField, parseVector } from "../src";
 import { Type } from "../src/types";
 
@@ -622,3 +627,37 @@ describe("date32", (t) => {
 //     expect(originalVector.get(i), wasmVector.get(i));
 //   }
 // });
+
+describe("nullable int", (t) => {
+  function test(copy: boolean) {
+    let columnIndex = TEST_TABLE.schema.fields.findIndex(
+      (field) => field.name == "nullable_int"
+    );
+
+    const originalField = TEST_TABLE.schema.fields[columnIndex];
+    // declare it's not null
+    const originalVector = TEST_TABLE.getChildAt(columnIndex) as arrow.Vector;
+    const fieldPtr = FFI_TABLE.schemaAddr(columnIndex);
+    const field = parseField(WASM_MEMORY.buffer, fieldPtr);
+
+    expect(field.name).toStrictEqual(originalField.name);
+    expect(field.typeId).toStrictEqual(originalField.typeId);
+    expect(field.nullable).toStrictEqual(originalField.nullable);
+
+    const arrayPtr = FFI_TABLE.arrayAddr(0, columnIndex);
+    const wasmVector = parseVector(
+      WASM_MEMORY.buffer,
+      arrayPtr,
+      field.type,
+      copy
+    );
+
+    expect(
+      validityEqual(originalVector, wasmVector),
+      "validity should be equal"
+    ).toBeTruthy();
+  }
+
+  it("copy=false", () => test(false));
+  it("copy=true", () => test(true));
+});

tests/pyarrow_generate_data.py

@@ -115,6 +115,15 @@ def timestamp_array() -> pa.Array:
     return arr
 
 
+def nullable_int() -> pa.Array:
+    # True means null
+    mask = [True, False, True]
+    arr = pa.array([1, 2, 3], type=pa.uint8(), mask=mask)
+    assert isinstance(arr, pa.UInt8Array)
+    assert not arr[0].is_valid
+    return arr
+
+
 class MyExtensionType(pa.ExtensionType):
     """
     Refer to https://arrow.apache.org/docs/python/extending_types.html for
@@ -160,6 +169,7 @@ def table() -> pa.Table:
             "date32": date32_array(),
             "date64": date64_array(),
             "timestamp": timestamp_array(),
+            "nullable_int": nullable_int(),
         }
     )
 

tests/utils.ts

@@ -35,3 +35,32 @@ export function arraysEqual<T>(
 
   return true;
 }
+
+export function validityEqual(v1: arrow.Vector, v2: arrow.Vector): boolean {
+  if (v1.length !== v2.length) {
+    return false;
+  }
+
+  if (v1.data.length !== v2.data.length) {
+    console.log("todo: support different data lengths");
+    return false;
+  }
+  for (let i = 0; i < v1.data.length; i++) {
+    const d1 = v1.data[i];
+    const d2 = v2.data[i];
+    // Check that null bitmaps have same length
+    if (d1 !== null && d2 !== null) {
+      if (d1.nullBitmap.length !== d2.nullBitmap.length) {
+        return false;
+      }
+    }
+  }
+
+  for (let i = 0; i < v1.length; i++) {
+    if (v1.isValid(i) !== v2.isValid(i)) {
+      return false;
+    }
+  }
+
+  return true;
+}