Skip to content

Commit

Permalink
Check that file manager values match all tiles
Browse files Browse the repository at this point in the history
  • Loading branch information
SolarDrew authored and Cadair committed Sep 19, 2024
1 parent 49878ef commit 41c9f42
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 7 deletions.
6 changes: 6 additions & 0 deletions dkist/dataset/tests/test_tiled_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,9 @@ def test_file_manager(large_tiled_dataset):
# Have some slicing tests here
assert len(ds.slice_tiles[0].files.filenames) == 9
assert len(ds[:2, :2].files.filenames) == 12

# TODO Also test that the other checks raise errors
# This at least demonstrates that the structure works
ds[1, 1].files.fileuri_array.dtype = np.dtype("<i")
with pytest.raises(AssertionError, match="must be the same across all tiles"):
ds.files
29 changes: 22 additions & 7 deletions dkist/dataset/tiled_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,14 +203,29 @@ def files(self):

@property
def _file_manager(self):
fileuris = [[tile.files.filenames for tile in row] for row in self]
dtype = self[0, 0].files.fileuri_array.dtype
shape = self[0, 0].files.shape
basepath = self[0, 0].files.basepath
chunksize = self[0, 0]._data.chunksize

for tile in self.flat:
try:
assert dtype == tile.files.fileuri_array.dtype
assert shape == tile.files.shape
assert basepath == tile.files.basepath
assert chunksize == tile._data.chunksize
except AssertionError as err:
raise AssertionError("Attributes of TiledDataset.FileManager must be the same across all tiles.") from err

return FileManager(
StripedExternalArray(
fileuris = [[tile.files.filenames for tile in row] for row in self],
target = 1,
dtype = self[0, 0].files.fileuri_array.dtype,
shape = self[0, 0]._data.chunksize,
loader = AstropyFITSLoader,
basepath = self[0, 0].files.basepath,
chunksize = self[0, 0]._data.chunksize
fileuris=fileuris,
target=1,
dtype=dtype,
shape=shape,
loader=AstropyFITSLoader,
basepath=basepath,
chunksize=chunksize
)
)

0 comments on commit 41c9f42

Please sign in to comment.