From 4c6a713dc14fdddf3d1262a444e6907b8aab8781 Mon Sep 17 00:00:00 2001 From: Massimiliano Galli Date: Tue, 17 Sep 2024 13:34:31 +0200 Subject: [PATCH 1/2] Truncate file through EOS using XRootD File interface --- src/fsspec_xrootd/xrootd.py | 48 +++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/src/fsspec_xrootd/xrootd.py b/src/fsspec_xrootd/xrootd.py index c578e68..a16f9b5 100644 --- a/src/fsspec_xrootd/xrootd.py +++ b/src/fsspec_xrootd/xrootd.py @@ -16,6 +16,7 @@ from fsspec.exceptions import FSTimeoutError from fsspec.spec import AbstractBufferedFile from XRootD import client +from XRootD.client import File from XRootD.client.flags import ( DirListFlags, MkDirFlags, @@ -372,21 +373,38 @@ async def _rm_file(self, path: str, **kwargs: Any) -> None: raise OSError(f"File not removed properly: {status.message}") async def _touch(self, path: str, truncate: bool = False, **kwargs: Any) -> None: - if truncate or not await self._exists(path): - status, _ = await _async_wrap(self._myclient.truncate)( - path, size=0, timeout=self.timeout - ) - if not status.ok: - raise OSError(f"File not touched properly: {status.message}") - else: - len = await self._info(path) - status, _ = await _async_wrap(self._myclient.truncate)( - path, - size=len.get("size"), - timeout=self.timeout, - ) + # necessary to write in EOS, see https://github.com/xrootd/xrootd/issues/2304 + if "eos" in self.hostid: + eos_full_path = f"{self.protocol}://{self.hostid}/{path}" + f = File() + status, _ = f.open(eos_full_path, OpenFlags.NEW, timeout=self.timeout) if not status.ok: - raise OSError(f"File not touched properly: {status.message}") + raise OSError(f"Impossible to create file in EOS: {status.message}") + if truncate or not await self._exists(path): + status, _ = await _async_wrap(f.truncate)( + size=0, timeout=self.timeout + ) + else: + len = await self._info(path) + status, _ = await _async_wrap(f.truncate)( + size=len.get("size"), + timeout=self.timeout, + ) + f.close() + else: + if truncate or not await self._exists(path): + status, _ = await _async_wrap(self._myclient.truncate)( + path, size=0, timeout=self.timeout + ) + else: + len = await self._info(path) + status, _ = await _async_wrap(self._myclient.truncate)( + path, + size=len.get("size"), + timeout=self.timeout, + ) + if not status.ok: + raise OSError(f"File not touched properly: {status.message}") touch = sync_wrapper(_touch) @@ -834,7 +852,7 @@ def __init__( self.kwargs = kwargs - if mode not in {"ab", "rb", "wb"}: + if mode not in {"ab", "rb", "wb", "a+b", "r+b", "w+b"}: raise NotImplementedError("File mode not supported") if mode == "rb": if size is not None: From b0dacf13d3d64f158812a5aae78e41a41327d2c3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:10:06 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/fsspec_xrootd/xrootd.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/fsspec_xrootd/xrootd.py b/src/fsspec_xrootd/xrootd.py index a16f9b5..1a5ceb4 100644 --- a/src/fsspec_xrootd/xrootd.py +++ b/src/fsspec_xrootd/xrootd.py @@ -381,9 +381,7 @@ async def _touch(self, path: str, truncate: bool = False, **kwargs: Any) -> None if not status.ok: raise OSError(f"Impossible to create file in EOS: {status.message}") if truncate or not await self._exists(path): - status, _ = await _async_wrap(f.truncate)( - size=0, timeout=self.timeout - ) + status, _ = await _async_wrap(f.truncate)(size=0, timeout=self.timeout) else: len = await self._info(path) status, _ = await _async_wrap(f.truncate)( @@ -391,7 +389,7 @@ async def _touch(self, path: str, truncate: bool = False, **kwargs: Any) -> None timeout=self.timeout, ) f.close() - else: + else: if truncate or not await self._exists(path): status, _ = await _async_wrap(self._myclient.truncate)( path, size=0, timeout=self.timeout