Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: intermittent segfault in PyPy #3089

Merged
merged 15 commits into from
May 2, 2024
Merged

fix: intermittent segfault in PyPy #3089

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
0c10f5e
debug: add debugging flag
ianna Apr 22, 2024
4c40472
test: debug
ianna Apr 22, 2024
e82ee31
fix: allocate memory for 2d arrays in tests
ianna Apr 30, 2024
532ad1a
Merge branch 'main' into ianna/2971-intermittent-segfault-in-pypy
ianna Apr 30, 2024
381d34d
fix: remove legacy parameter from api
ianna May 1, 2024
273f06b
debug: exclude some tests that rely on 2d array parameters in cpu ker…
ianna May 1, 2024
e4baf54
debug: exclude cpu kernel unt tests with 2d arrays as parameters
ianna May 1, 2024
fb09163
fix: typo
ianna May 1, 2024
ddd6b0d
test: remove coverage for pypy
ianna May 2, 2024
8ce374c
fix: restore tests
ianna May 2, 2024
410c40d
fix: restore cpu kernel unit tests
ianna May 2, 2024
338e977
fix: remove debugger
ianna May 2, 2024
23b890f
fix: restore tests
ianna May 2, 2024
1418372
fix: add more tests
ianna May 2, 2024
90808bd
Merge branch 'main' into ianna/2971-intermittent-segfault-in-pypy
jpivarski May 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,20 +128,20 @@ jobs:
run: pipx run nox -s diagnostics -- --check-spec-sorted

- name: Test specification
run: python -m pytest -vv -rs awkward-cpp/tests-spec
run: python -m pytest --pdb -vv -rs awkward-cpp/tests-spec

- name: Test specification with explicitly defined values
run: python -m pytest -vv -rs awkward-cpp/tests-spec-explicit
run: python -m pytest --pdb -vv -rs awkward-cpp/tests-spec-explicit

- name: Test CPU kernels
run: python -m pytest -vv -rs awkward-cpp/tests-cpu-kernels
run: python -m pytest --pdb -vv -rs awkward-cpp/tests-cpu-kernels

- name: Test CPU kernels with explicitly defined values
run: python -m pytest -vv -rs awkward-cpp/tests-cpu-kernels-explicit
run: python -m pytest --pdb -vv -rs awkward-cpp/tests-cpu-kernels-explicit

- name: Test non-kernels
run: >-
python -m pytest -vv -rs tests --cov=awkward --cov-report=term
python -m pytest --pdb -vv -rs tests --cov=awkward --cov-report=term
--cov-report=xml

- name: Upload Codecov results
Expand Down
6 changes: 3 additions & 3 deletions awkward-cpp/src/cpu-kernels/awkward_UnionArray_flatten_length.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ ERROR awkward_UnionArray_flatten_length(
T** offsetsraws) {
*total_length = 0;
for (int64_t i = 0; i < length; i++) {
FROMTAGS tag = fromtags[i];
FROMINDEX idx = fromindex[i];
int64_t tag = (int64_t)fromtags[i];
int64_t idx = (int64_t)fromindex[i];
T start = offsetsraws[tag][idx];
T stop = offsetsraws[tag][idx + 1];
*total_length = *total_length + (stop - start);
*total_length = *total_length + (int64_t)(stop - start);
}
return success();
}
Expand Down
80 changes: 66 additions & 14 deletions dev/generate-tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,10 @@ def gencpukerneltests(specdict):
)

f.write(
"import ctypes\nimport pytest\n\nfrom awkward_cpp.cpu_kernels import lib\n\n"
"import ctypes\n"
"import numpy as np\n"
"import pytest\n\n"
"from awkward_cpp.cpu_kernels import lib\n\n"
)
num = 1
if spec.tests == []:
Expand Down Expand Up @@ -519,13 +522,29 @@ def gencpukerneltests(specdict):
elif count == 2:
f.write(
" " * 4
+ f"{arg}_ptr = (ctypes.POINTER(ctypes.c_{typename}) * len({arg}))()\n"
+ f"{typename}Ptr = ctypes.POINTER(ctypes.c_{typename})\n"
+ " " * 4
+ f"for i in range(len({arg})):\n"
+ f"{typename}PtrPtr = ctypes.POINTER({typename}Ptr)\n"
+ " " * 4
+ f"dim0 = len({arg})\n"
+ " " * 4
+ f"dim1 = len({arg}[0])\n"
+ " " * 4
+ f"{arg}_np_arr_2d = np.empty([dim0, dim1], dtype=np.{typename})\n"
+ " " * 4
+ "for i in range(dim0):\n"
+ " " * 8
+ f"{arg}_ptr[i] = (ctypes.c_{typename} * len({arg}[i]))(*{arg}[i])\n"
+ "for j in range(dim1):\n"
+ " " * 12
+ f"{arg}_np_arr_2d[i][j] = {arg}[i][j]\n"
+ " " * 4
+ f"{arg}_ct_arr = np.ctypeslib.as_ctypes({arg}_np_arr_2d)\n"
+ " " * 4
+ f"{typename}PtrArr = {typename}Ptr * {arg}_ct_arr._length_\n"
+ " " * 4
+ f"{arg} = {arg}_ptr\n"
+ f"{arg}_ct_ptr = ctypes.cast({typename}PtrArr(*(ctypes.cast(row, {typename}Ptr) for row in {arg}_ct_arr)), {typename}PtrPtr)\n"
+ " " * 4
+ f"{arg} = {arg}_ct_ptr\n"
)
f.write(" " * 4 + "funcC = getattr(lib, '" + spec.name + "')\n")
args = ""
Expand Down Expand Up @@ -599,6 +618,7 @@ def gencpuunittests(specdict):

f.write(
"import ctypes\n"
"import numpy as np\n"
"import pytest\n\n"
"from awkward_cpp.cpu_kernels import lib\n\n"
)
Expand Down Expand Up @@ -640,13 +660,29 @@ def gencpuunittests(specdict):
elif count == 2:
f.write(
" " * 4
+ f"{arg}_ptr = (ctypes.POINTER(ctypes.c_{typename}) * len({arg}))()\n"
+ f"{typename}Ptr = ctypes.POINTER(ctypes.c_{typename})\n"
+ " " * 4
+ f"{typename}PtrPtr = ctypes.POINTER({typename}Ptr)\n"
+ " " * 4
+ f"dim0 = len({arg})\n"
+ " " * 4
+ f"dim1 = len({arg}[0])\n"
+ " " * 4
+ f"{arg}_np_arr_2d = np.empty([dim0, dim1], dtype=np.{typename})\n"
+ " " * 4
+ f"for i in range(len({arg})):\n"
+ "for i in range(dim0):\n"
+ " " * 8
+ f"{arg}_ptr[i] = (ctypes.c_{typename} * len({arg}[i]))(*{arg}[i])\n"
+ "for j in range(dim1):\n"
+ " " * 12
+ f"{arg}_np_arr_2d[i][j] = {arg}[i][j]\n"
+ " " * 4
+ f"{arg} = {arg}_ptr\n"
+ f"{arg}_ct_arr = np.ctypeslib.as_ctypes({arg}_np_arr_2d)\n"
+ " " * 4
+ f"{typename}PtrArr = {typename}Ptr * {arg}_ct_arr._length_\n"
+ " " * 4
+ f"{arg}_ct_ptr = ctypes.cast({typename}PtrArr(*(ctypes.cast(row, {typename}Ptr) for row in {arg}_ct_arr)), {typename}PtrPtr)\n"
+ " " * 4
+ f"{arg} = {arg}_ct_ptr\n"
)
for arg, val in test["inputs"].items():
typename = remove_const(
Expand All @@ -669,13 +705,29 @@ def gencpuunittests(specdict):
elif count == 2:
f.write(
" " * 4
+ f"{arg}_ptr = (ctypes.POINTER(ctypes.c_{typename}) * len({arg}))()\n"
+ f"{typename}Ptr = ctypes.POINTER(ctypes.c_{typename})\n"
+ " " * 4
+ f"{typename}PtrPtr = ctypes.POINTER({typename}Ptr)\n"
+ " " * 4
+ f"dim0 = len({arg})\n"
+ " " * 4
+ f"for i in range(len({arg})):\n"
+ f"dim1 = len({arg}[0])\n"
+ " " * 4
+ f"{arg}_np_arr_2d = np.empty([dim0, dim1], dtype=np.{typename})\n"
+ " " * 4
+ "for i in range(dim0):\n"
+ " " * 8
+ f"{arg}_ptr[i] = (ctypes.c_{typename} * len({arg}[i]))(*{arg}[i])\n"
+ "for j in range(dim1):\n"
+ " " * 12
+ f"{arg}_np_arr_2d[i][j] = {arg}[i][j]\n"
+ " " * 4
+ f"{arg}_ct_arr = np.ctypeslib.as_ctypes({arg}_np_arr_2d)\n"
+ " " * 4
+ f"{typename}PtrArr = {typename}Ptr * {arg}_ct_arr._length_\n"
+ " " * 4
+ f"{arg}_ct_ptr = ctypes.cast({typename}PtrArr(*(ctypes.cast(row, {typename}Ptr) for row in {arg}_ct_arr)), {typename}PtrPtr)\n"
+ " " * 4
+ f"{arg} = {arg}_ptr\n"
+ f"{arg} = {arg}_ct_ptr\n"
)

f.write(" " * 4 + "funcC = getattr(lib, '" + spec.name + "')\n")
Expand Down Expand Up @@ -1135,7 +1187,7 @@ def genunittests():
os.path.join(CURRENT_DIR, "..", "awkward-cpp", "tests-spec-explicit", func),
"w",
) as file:
file.write("import pytest\nimport numpy\nimport kernels\n\n")
file.write("import pytest\n" "import numpy\n" "import kernels\n\n")
for test in function["tests"]:
num += 1
funcName = "def test_" + function["name"] + "_" + str(num) + "():\n"
Expand Down
Loading