From 3af0af4cc30ca2b9a05526a3a2ff6d2f667b1f53 Mon Sep 17 00:00:00 2001 From: calum-chamberlain Date: Tue, 9 Jan 2024 10:42:48 +1300 Subject: [PATCH] try to add progressbar --- rt_eqcorrscan/database/database_manager.py | 35 ++-------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/rt_eqcorrscan/database/database_manager.py b/rt_eqcorrscan/database/database_manager.py index e358d49..9f9584e 100644 --- a/rt_eqcorrscan/database/database_manager.py +++ b/rt_eqcorrscan/database/database_manager.py @@ -195,35 +195,6 @@ def map( """ return map(fn, *iterables) - def imap( - self, - fn: Callable, - *iterables: Iterable, - timeout=None, - chunksize=1, - ) -> Iterable: - """ - Map iterables to a function - - Parameters - ---------- - fn - callable - iterables - iterable of arguments for `fn` - timeout - Throw-away variable - chunksize - Throw-away variable - - Returns - ------- - The result of mapping `fn` across `*iterables` - """ - return (_ for _ in self.map( - fn=fn, timeout=timeout, chunksize=chunksize, - *iterables)) - def submit(self, fn: Callable, *args, **kwargs) -> _Result: """ Run a single function. @@ -321,7 +292,7 @@ def get_templates(self, use_pickled: bool = True, **kwargs) -> Tribe: paths = list(self._template_paths(**kwargs)) _tread = partial(_lazy_template_read, read_pickle=use_pickled) return Tribe(list(_ for _ in tqdm.tqdm( - (t for t in self.executor.imap(_tread, paths)), + (t for t in self.executor.map(_tread, paths)), total=len(paths)) if _ is not None)) # future = self.executor.map(_tread, paths) # return Tribe([t for t in future if t is not None]) @@ -364,14 +335,14 @@ def put_templates( bank_path=self.bank_path) with self.index_lock: _ = list(tqdm.tqdm( - (_ for _ in self.executor.imap(inner_put_template, templates)), + (_ for _ in self.executor.map(inner_put_template, templates)), total=len(templates))) def pickle_templates(self, **kwargs) -> List: """ Pickle templates in the db for faster reading later. """ paths = [p for p in self._template_paths(**kwargs)] Logger.info(f"Pickling {len(paths)} templates...") - future = self.executor.imap(_pickle_template, paths) + future = self.executor.map(_pickle_template, paths) issues = list(_ for _ in tqdm.tqdm( (f for f in future), total=len(paths)) if _ is not None)