page now has their pid assigned to resolve racing conditions

root-11 · Jan 30, 2024 · 4ad3486 · 4ad3486
1 parent c05d703
commit 4ad3486
Show file tree

Hide file tree

Showing 7 changed files with 10 additions and 8 deletions.
diff --git a/tablite/_nimlite/funcs/column_selector/collectinfo.nim b/tablite/_nimlite/funcs/column_selector/collectinfo.nim
@@ -112,7 +112,7 @@ proc collectColumnSelectInfo*(table: nimpy.PyObject, cols: nimpy.PyObject, dirPi
 
     var isCorrectType = initTable[string, bool]()
 
-    proc genpage(dirpid: string): ColSliceInfo {.inline.} = (dir_pid, tabliteBase().SimplePage.next_id(dir_pid).to(int))
+    proc genpage(dirpid: string): ColSliceInfo {.inline.} = (dir_pid, tabliteBase().SimplePage.next_id(dir_pid).to(string))
 
     discard pbar.update(5)
     discard pbar.display()

diff --git a/tablite/_nimlite/funcs/column_selector/infos.nim b/tablite/_nimlite/funcs/column_selector/infos.nim
@@ -4,7 +4,7 @@ from std/sugar import collect
 from ../../pymodules import builtins
 from ../../pytypes import KindObjectND, str2ObjKind
 
-type ColSliceInfo* = (string, int)
+type ColSliceInfo* = (string, string)
 type ColInfo* = Table[string, ColSliceInfo]
 type DesiredColumnInfo* = object
     originalName*: string

diff --git a/tablite/_nimlite/funcs/column_selector/sliceconv.nim b/tablite/_nimlite/funcs/column_selector/sliceconv.nim
@@ -35,7 +35,7 @@ proc finalizeSlice(indices: var seq[int], columnNames: seq[string], infos: var T
 
 proc toColSliceInfo(path: Path): ColSliceInfo =
     let workdir = string path.parentDir.parentDir
-    let pid = parseInt(string path.extractFilename.changeFileExt(""))
+    let pid = string path.extractFilename.changeFileExt("")
 
     return (workdir, pid)
 

diff --git a/tablite/_nimlite/funcs/text_reader/text_reader.nim b/tablite/_nimlite/funcs/text_reader/text_reader.nim
@@ -67,7 +67,7 @@ proc textReaderTask*(task: TaskArgs): seq[nimpy.PyObject] =
                 for i in 0..<n_pages:
                     let path = Path(destinations[i])
                     let workdir = string path.parentDir.parentDir
-                    let id = parseInt(string path.extractFilename.changeFileExt(""))
+                    let id = string path.extractFilename.changeFileExt("")
                     let dtypes = pgTypes[i]
                     let len = pgLens[i]
 

diff --git a/tablite/_nimlite/numpy.nim b/tablite/_nimlite/numpy.nim
@@ -1017,7 +1017,7 @@ proc type2PyType(`type`: KindObjectND): nimpy.PyObject =
     of K_TIME: return pymodules.datetime().time
     of K_DATETIME: return pymodules.datetime().datetime
 
-proc newPyPage*(id: int, path: string, len: int, dtypes: Table[KindObjectND, int]): nimpy.PyObject =
+proc newPyPage*(id: string, path: string, len: int, dtypes: Table[KindObjectND, int]): nimpy.PyObject =
     let pyDtypes = pymodules.builtins().dict()
 
     for (dt, n) in dtypes.pairs:

diff --git a/tablite/base.py b/tablite/base.py
@@ -70,7 +70,6 @@ class SimplePage(object):
     autocleanup = True
 
     def __init__(self, id, path, len, py_dtype) -> None:
-        self.id = id
         self.path = Path(path) / "pages" / f"{id}.npy"
         self.len = len
         self.dtype = py_dtype
@@ -97,7 +96,7 @@ def next_id(cls, path):
         path = Path(path)
 
         while True:
-            _id = next(cls.ids)
+            _id = f"{os.getpid()}-{next(cls.ids)}"
             _path = path / "pages" / f"{_id}.npy"
 
             if not _path.exists():
@@ -117,7 +116,7 @@ def __repr__(self) -> str:
             return f"{self.__class__.__name__}({self.path}, <{e}>)"
 
     def __hash__(self) -> int:
-        return hash(self.id)
+        return hash(self.path)
 
     def owns(self):
         parts = self.path.parts

diff --git a/tablite/joins.py b/tablite/joins.py
@@ -598,8 +598,11 @@ def _mp_reindex_page(
         if np.any(mask):
             nones = np.full(ix_arr.shape, fill_value=None)
             array = np.where(mask, nones, array)
+
+
     Constr = type(T)
     remapped_T = Constr({column_name: array}, _path=path)
+
     return remapped_T