From a8835b0976dc8420832a27f3d068d043a8923632 Mon Sep 17 00:00:00 2001 From: Matthias Klumpp Date: Mon, 3 Oct 2022 23:05:09 +0200 Subject: [PATCH] Adjust to run with newer dependencies --- .github/workflows/pyformat.yml | 4 +- .github/workflows/testandcov.yml | 8 +- docs/ext/normalize_html_id.py | 2 +- environment.yml | 62 +++++----- minian/cnmf.py | 31 +++-- minian/motion_correction.py | 2 +- minian/test/test_cross_reg.py | 12 +- minian/utilities.py | 5 +- minian/visualization.py | 179 +++++++++++++++-------------- pipeline.ipynb | 17 +-- requirements/requirements-base.txt | 52 ++++----- requirements/requirements-doc.txt | 28 ++--- setup.py | 2 +- 13 files changed, 214 insertions(+), 190 deletions(-) diff --git a/.github/workflows/pyformat.yml b/.github/workflows/pyformat.yml index 5d26a2f0..1ebc9fc1 100644 --- a/.github/workflows/pyformat.yml +++ b/.github/workflows/pyformat.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: ['3.10'] steps: # Checks out a copy of your repository on the ubuntu-latest machine @@ -24,7 +24,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Format with black run: | - $CONDA/bin/conda install -c conda-forge black==20.8b1 click==8.0.4 + $CONDA/bin/conda install -c conda-forge black~=22.8 click~=8.0.4 # stop the build if there are Python syntax errors or undefined names $CONDA/bin/black minian - name: Check for modified files diff --git a/.github/workflows/testandcov.yml b/.github/workflows/testandcov.yml index f455e4ca..b02f30e5 100644 --- a/.github/workflows/testandcov.yml +++ b/.github/workflows/testandcov.yml @@ -20,7 +20,7 @@ jobs: max-parallel: 5 matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.8"] + python-version: ['3.10'] defaults: run: shell: bash -l {0} @@ -39,7 +39,7 @@ jobs: environment-file: environment.yml - name: Install test dependencies run: | - conda install -c conda-forge black==20.8b1 click==8.0.4 pytest==7.0.1 pytest-cov pytest-rerunfailures coverage==6.3.3 + conda install -c conda-forge black~=22.8 click~=8.0.4 pytest~=7.1.2 pytest-cov pytest-rerunfailures coverage~=6.3.3 - name: Lint with black run: | black --check minian @@ -48,8 +48,8 @@ jobs: id: n_workers with: cond: ${{ runner.os == 'macOS' }} - if_true: 3 - if_false: 2 + if_true: 2 + if_false: 1 - name: Run doctest run: | pip install -r requirements/requirements-doc.txt diff --git a/docs/ext/normalize_html_id.py b/docs/ext/normalize_html_id.py index 5ef9fe6f..258998d8 100644 --- a/docs/ext/normalize_html_id.py +++ b/docs/ext/normalize_html_id.py @@ -41,4 +41,4 @@ def normalize_id(app, exception): def setup(app): - app.connect("build-finished", normalize_id) \ No newline at end of file + app.connect("build-finished", normalize_id) diff --git a/environment.yml b/environment.yml index 65e1d44d..92a96d79 100644 --- a/environment.yml +++ b/environment.yml @@ -4,41 +4,43 @@ channels: - conda-forge - defaults dependencies: - - bokeh=1.4.0 - - cvxpy>=1.1.11 - - dask=2021.2.0 - - datashader=0.12.1 - - distributed=2021.2.0 - - ecos>=2.0.7 - - ffmpeg - - ffmpeg-python>=0.2.0 + - python=3.10 + - pip + - bokeh~=2.4.3 + - dask=2022.9.2 + - datashader=0.14.2 + - distributed=2022.9.2 + - ecos>=2.0.10 + - ffmpeg>=4.4 - fftw - - holoviews=1.12.7 + - holoviews=1.15.1 - jupyter - - matplotlib-base=3.2 + - matplotlib-base~=3.6.0 - natsort - netcdf4 - - networkx=2.4 - - numba=0.52.0 - - numpy=1.20.2 - - opencv=4.2.0 - - pandas=1.2.3 - - panel=0.8.0 - - param=1.9 - - pyfftw=0.12.0 - - python=3.8 - - scikit-image=0.18.1 - - scikit-learn=0.22.1 - - scipy>=1.4.1 + - networkx~=2.8.7 + - numba~=0.56.2 + - numpy~=1.23.3 + - opencv>=4.6 + - pandas~=1.5.0 + - panel~=0.14.0 + - param~=1.12.2 + - pyfftw~=0.12.0 + - scikit-image~=0.19.3 + - scikit-learn~=1.1.2 + - scipy~=1.9.1 - scs - - simpleitk=2.0.2 + - simpleitk~=2.1.1 - sk-video - - statsmodels>=0.11.1 + - statsmodels>=0.13.2 - tifffile - - xarray=0.16.2 - - zarr - - sparse=0.11.2 - - pymetis=2020.1 - - rechunker=0.3.3 + - xarray~=2022.9.0 + - zarr>=2.12 + - sparse=0.13.0 + - pymetis=2022.1 + - rechunker~=0.5.0 - medpy=0.4.0 - - jinja2=2.11.3 + - jinja2~=3.1.2 + - pip: + - cvxpy>=1.2.1 + - ffmpeg-python>=0.2.0 diff --git a/minian/cnmf.py b/minian/cnmf.py index c6efeb24..1d926fbc 100644 --- a/minian/cnmf.py +++ b/minian/cnmf.py @@ -24,6 +24,8 @@ from scipy.sparse import dia_matrix from skimage import morphology as moph from sklearn.linear_model import LassoLars +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler from statsmodels.tsa.stattools import acovf from .utilities import ( @@ -386,6 +388,8 @@ def update_spatial( C_store=C_store, f=f_in, ) + if not isinstance(cur_blk, sparse.SparseArray): + cur_blk = cur_blk.map_blocks(sparse.COO) else: cur_blk = darr.array(sparse.zeros((cur_sub.shape))) A_new[hblk, wblk, 0] = cur_blk @@ -512,8 +516,11 @@ def update_spatial_perpx( if (f is not None) and sub[-1]: C = np.concatenate([C, f.reshape((-1, 1))], axis=1) idx = np.concatenate([idx, np.array(len(sub) - 1).reshape(-1)]) - clf = LassoLars(alpha=alpha, positive=True) - coef = clf.fit(C, y).coef_ + model = make_pipeline( + StandardScaler(with_mean=False), + LassoLars(alpha=alpha, positive=True, normalize=False), + ) + coef = model.fit(C, y).named_steps["lassolars"].coef_ mask = coef > 0 coef = coef[mask] idx = idx[mask] @@ -611,7 +618,7 @@ def compute_trace( """ fms = Y.coords["frame"] uid = A.coords["unit_id"] - Y = Y.data + Y = Y.data.map_blocks(sparse.COO) A = darr.from_array(A.data.map_blocks(sparse.COO).compute(), chunks=-1) C = C.data.map_blocks(sparse.COO).T b = ( @@ -620,10 +627,10 @@ def compute_trace( .reshape((1, Y.shape[1], Y.shape[2])) .compute() ) - f = f.fillna(0).data.reshape((-1, 1)) + f = f.fillna(0).data.reshape((-1, 1)).map_blocks(sparse.COO) AtA = darr.tensordot(A, A, axes=[(1, 2), (1, 2)]).compute() A_norm = ( - (1 / (A ** 2).sum(axis=(1, 2))) + (1 / (A**2).sum(axis=(1, 2))) .map_blocks( lambda a: sparse.diagonalize(sparse.COO(a)), chunks=(A.shape[0], A.shape[0]) ) @@ -648,7 +655,9 @@ def compute_trace( dims=["frame", "unit_id"], coords={"frame": fms, "unit_id": uid}, ) - return YrA.transpose("unit_id", "frame") + YrA = YrA.transpose("unit_id", "frame") + YrA.data = YrA.data.map_blocks(lambda x: x.todense()) + return YrA def update_temporal( @@ -833,7 +842,7 @@ def update_temporal( \mathbf{b_0} \\right \\rVert ^2 + \\alpha \\left \\lvert \mathbf{G} \mathbf{c} \\right \\rvert \\\\ & \\text{subject to} - & & \mathbf{c} \geq 0, \; \mathbf{G} \mathbf{c} \geq 0 + & & \mathbf{c} \geq 0, \; \mathbf{G} \mathbf{c} \geq 0 \\end{aligned} Where :math:`\mathbf{y}` is the estimated residule trace (`YrA`) for the @@ -855,10 +864,12 @@ def update_temporal( if YrA is None: YrA = compute_trace(Y, A, b, C, f).persist() Ymask = (YrA > 0).any("frame").compute() + if hasattr(Ymask.data, "todense"): + Ymask.data = Ymask.data.todense() A, C, YrA = A.sel(unit_id=Ymask), C.sel(unit_id=Ymask), YrA.sel(unit_id=Ymask) print("grouping overlaping units") A_sps = (A.data.map_blocks(sparse.COO) > 0).compute().astype(np.float32) - A_inter = sparse.tensordot(A_sps, A_sps, axes=[(1, 2), (1, 2)]) + A_inter = sparse.tensordot(A_sps, A_sps, axes=[(1, 2), (1, 2)]).todense() A_usum = np.tile(A_sps.sum(axis=(1, 2)).todense(), (A_sps.shape[0], 1)) A_usum = A_usum + A_usum.T jac = scipy.sparse.csc_matrix(A_inter / (A_usum - A_inter) > jac_thres) @@ -1066,7 +1077,7 @@ def get_ar_coef( else: max_lag = p + add_lag cov = acovf(y, fft=True) - C_mat = toeplitz(cov[:max_lag], cov[:p]) - sn ** 2 * np.eye(max_lag, p) + C_mat = toeplitz(cov[:max_lag], cov[:p]) - sn**2 * np.eye(max_lag, p) g = lstsq(C_mat, cov[1 : max_lag + 1])[0] if pad: res = np.zeros(pad) @@ -1165,6 +1176,8 @@ def update_temporal_block( excluded=["pad", "add_lag"], signature="(f),(),()->(l)", ) + if hasattr(YrA, "todense"): + YrA = YrA.todense() if normalize: amean = YrA.sum(axis=1).mean() norm_factor = YrA.shape[1] / amean diff --git a/minian/motion_correction.py b/minian/motion_correction.py index b3a09a90..2836eb82 100755 --- a/minian/motion_correction.py +++ b/minian/motion_correction.py @@ -617,7 +617,7 @@ def check_temp(fm: np.ndarray, max_sh: int) -> float: if perimeter <= 0: return 0 area = cv2.contourArea(cont) - circularity = 4 * np.pi * (area / (perimeter ** 2)) + circularity = 4 * np.pi * (area / (perimeter**2)) return circularity diff --git a/minian/test/test_cross_reg.py b/minian/test/test_cross_reg.py index 0efa7eb1..c0b02ec3 100644 --- a/minian/test/test_cross_reg.py +++ b/minian/test/test_cross_reg.py @@ -23,11 +23,11 @@ def test_cross_reg_notebook(): cents = pd.read_pickle("./demo_data/cents.pkl") mappings = pd.read_pickle("./demo_data/mappings.pkl") assert len(cents) == 508 - assert int(cents["height"].sum()) == 99091 - assert int(cents["width"].sum()) == 213627 - assert len(mappings) == 431 + assert int(cents["height"].sum()) == 99096 + assert int(cents["width"].sum()) == 213628 + assert len(mappings) == 430 assert mappings[("group", "group")].value_counts().to_dict() == { - ("session2",): 182, - ("session1",): 172, - ("session1", "session2"): 77, + ("session2",): 181, + ("session1",): 171, + ("session1", "session2"): 78, } diff --git a/minian/utilities.py b/minian/utilities.py index a728a0c9..0fb15761 100755 --- a/minian/utilities.py +++ b/minian/utilities.py @@ -837,10 +837,10 @@ def update_graph(self, scheduler, client, tasks, **kwargs): for tk in tasks.keys(): for pattern, annt in self.annt_dict.items(): if re.search(pattern, tk): - ts = parent._tasks.get(tk) + ts = parent.tasks.get(tk) res = annt.get("resources", None) if res: - ts._resource_restrictions = res + ts.resource_restrictions = res pri = annt.get("priority", None) if pri: pri_org = list(ts._priority) @@ -856,6 +856,7 @@ def custom_arr_optimize( rename_dict: Optional[dict] = None, rewrite_dict: Optional[dict] = None, keep_patterns=[], + flush=True, ) -> dict: """ Customized implementation of array optimization function. diff --git a/minian/visualization.py b/minian/visualization.py index 5c0f70f8..b81681c6 100755 --- a/minian/visualization.py +++ b/minian/visualization.py @@ -19,7 +19,7 @@ import sklearn.mixture import skvideo.io import xarray as xr -from bokeh.palettes import Category10_10, Viridis256 +from bokeh.palettes import Category10_10 from dask.diagnostics import ProgressBar from datashader import count_cat from holoviews.operation.datashader import datashade, dynspread @@ -208,14 +208,12 @@ def img(f, ds): except ValueError: curds = self.ds_sub fim = fct.partial(img, ds=curds) - im = hv.DynamicMap(fim, streams=[self.strm_f]).opts( + im = hv.DynamicMap(fim, streams=[self.strm_f]).options( frame_width=500, aspect=self._w / self._h, cmap="Viridis" ) self.xyrange = RangeXY(source=im).rename(x_range="w", y_range="h") if not self._layout: - hv_box = hv.Polygons([]).opts( - style={"fill_alpha": 0.3, "line_color": "white"} - ) + hv_box = hv.Polygons([]).options(fill_alpha=0.3, line_color="white") self.str_box = BoxEdit(source=hv_box) im_ovly = im * hv_box else: @@ -230,15 +228,17 @@ def hist(f, w, h, ds): cur_im = hv.Image( ds.sel(frame=f).compute(), kdims=["width", "height"] ) - return hv.operation.histogram(cur_im, num_bins=50).opts( + return hv.operation.histogram(cur_im, num_bins=50).options( xlabel="fluorescence", ylabel="freq" ) fhist = fct.partial(hist, ds=curds) - his = hv.DynamicMap(fhist, streams=[self.strm_f, self.xyrange]).opts( + his = hv.DynamicMap(fhist, streams=[self.strm_f, self.xyrange]).options( frame_height=int(500 * self._h / self._w), width=150, cmap="Viridis" ) - im_ovly = (im_ovly << his).map(lambda p: p.opts(style=dict(cmap="Viridis"))) + im_ovly = (im_ovly << his).map( + lambda p: hv.opts.apply_groups(p, style=dict(cmap="Viridis")) + ) return im_ovly if self._layout and self.meta_dicts: @@ -261,11 +261,10 @@ def hist(f, w, h, ds): hvsum = hvsum.layout(list(self.meta_dicts.keys())) except: pass - vl = hv.DynamicMap(lambda f: hv.VLine(f), streams=[self.strm_f]).opts( - style=dict(color="red") - ) + vl = hv.DynamicMap(lambda f: hv.VLine(f), streams=[self.strm_f]) + vl = hv.opts.apply_groups(vl, style=dict(color="red")) summ = (hvsum * vl).map( - lambda p: p.opts(frame_width=500, aspect=3), [hv.RGB, hv.Curve] + lambda p: p.options(frame_width=500, aspect=3), [hv.RGB, hv.Curve] ) hvobj = (ims + summ).cols(1) else: @@ -673,7 +672,7 @@ def _temp_comp_sub(self, usub=None): ).to(hv.Curve, "frame") cur_vl = hv.DynamicMap( lambda f, y: hv.VLine(f) if f else hv.VLine(0), streams=[self.strm_f] - ).opts(style=dict(color="red")) + ).options(color="red") cur_cv = hv.Curve([], kdims=["frame"], vdims=["Internsity (A.U.)"]) self.strm_f.source = cur_cv h_cv = len(self._w) // 8 @@ -688,14 +687,17 @@ def _temp_comp_sub(self, usub=None): .add_dimension("time", 0, 0), "trace", ) - .opts(plot=dict(shared_xaxis=True)) + .options(shared_xaxis=True) .map( - lambda p: p.opts(plot=dict(frame_height=h_cv, frame_width=w_cv)), hv.RGB + lambda p: hv.opts.apply_groups( + p, plot=dict(frame_height=h_cv, frame_width=w_cv) + ), + hv.RGB, ) * cur_vl ) - temp_comp[temp_comp.keys()[0]] = temp_comp[temp_comp.keys()[0]].opts( - plot=dict(height=h_cv + 75) + temp_comp[temp_comp.keys()[0]] = hv.opts.apply_groups( + temp_comp[temp_comp.keys()[0]], plot=dict(height=h_cv + 75) ) return pn.panel(temp_comp) @@ -921,7 +923,9 @@ def callback_useAC(val): def _spatial_all(self): metas = self.metas - Asum = hv.Image(self.Asum.sel(**metas), ["width", "height"]).opts( + Asum = hv.Image(self.Asum.sel(**metas), ["width", "height"]) + Asum = hv.opts.apply_groups( + Asum, plot=dict(frame_height=len(self._h), frame_width=len(self._w)), style=dict(cmap="Viridis"), ) @@ -931,33 +935,33 @@ def _spatial_all(self): kdims=["width", "height", "unit_id"], ) .to(hv.Points, ["width", "height"]) - .opts( - style=dict( - alpha=0.1, - line_alpha=0, - size=5, - nonselection_alpha=0.1, - selection_alpha=0.9, - ) + .options( + alpha=0.1, + line_alpha=0, + size=5, + nonselection_alpha=0.1, + selection_alpha=0.9, ) .collate() .overlay("unit_id") - .opts(plot=dict(tools=["hover", "box_select"])) ) + cents = hv.opts.apply_groups(cents, plot=dict(tools=["hover", "box_select"])) self.strm_uid.source = cents fim = fct.partial(hv.Image, kdims=["width", "height"]) - AC = hv.DynamicMap(fim, streams=[self.pipAC]).opts( + AC = hv.DynamicMap(fim, streams=[self.pipAC]) + AC = hv.opts.apply_groups( + AC, plot=dict(frame_height=len(self._h), frame_width=len(self._w)), style=dict(cmap="Viridis"), ) - mov = hv.DynamicMap(fim, streams=[self.pipmov]).opts( + mov = hv.DynamicMap(fim, streams=[self.pipmov]) + mov = hv.opts.apply_groups( + mov, plot=dict(frame_height=len(self._h), frame_width=len(self._w)), style=dict(cmap="Viridis"), ) lab = fct.partial(hv.Labels, kdims=["width", "height"], vdims=["unit_id"]) - ulab = hv.DynamicMap(lab, streams=[self.pipusub]).opts( - style=dict(text_color="red") - ) + ulab = hv.DynamicMap(lab, streams=[self.pipusub]).options(text_color="red") return pn.panel(Asum * cents + AC * ulab + mov) def update_spatial_all(self): @@ -1145,7 +1149,7 @@ def update_plot(self): im_ovly[:, :, 3], ), kdims=["width", "height"], - ).opts(**im_opts) + ).options(**im_opts) ) def update_meta(self): @@ -1418,10 +1422,7 @@ def datashade_ndcurve( var = np.unique(ovly.dimension_values(kdim)).tolist() color_key = [(v, Category10_10[iv]) for iv, v in enumerate(var)] color_pts = hv.NdOverlay( - { - k: hv.Points([0, 0], label=str(k)).opts(style=dict(color=v)) - for k, v in color_key - } + {k: hv.Points([0, 0], label=str(k)).options(color=v) for k, v in color_key} ) ds_ovly = datashade( ovly, @@ -1680,8 +1681,8 @@ def visualize_preprocess( } def _vis(f): - im = hv.Image(f, kdims=["width", "height"]).opts(**opts_im) - cnt = hv.operation.contours(im).opts(**opts_cnt) + im = hv.opts.apply_groups(hv.Image(f, kdims=["width", "height"]), **opts_im) + cnt = hv.opts.apply_groups(hv.operation.contours(im), **opts_cnt) return im, cnt if fn is not None: @@ -1699,18 +1700,20 @@ def _vis(f): ) im_dict[p_str] = cur_im cnt_dict[p_str] = cur_cnt - hv_im = Dynamic(hv.HoloMap(im_dict, kdims=list(pkey)).opts(**opts_im)) - hv_cnt = datashade( - hv.HoloMap(cnt_dict, kdims=list(pkey)), precompute=True, cmap=Viridis256 - ).opts(**opts_cnt) + hv_im = hv.opts.apply_groups( + Dynamic(hv.HoloMap(im_dict, kdims=list(pkey)), **opts_im) + ) + hv_cnt = hv.opts.apply_groups( + datashade( + hv.HoloMap(cnt_dict, kdims=list(pkey)), precompute=True, cmap="Viridis" + ), + **opts_cnt + ) if include_org: im, cnt = _vis(fm) - im = im.relabel("Before").opts(**opts_im) - cnt = ( - datashade(cnt, precompute=True, cmap=Viridis256) - .relabel("Before") - .opts(**opts_cnt) - ) + im = hv.opts.apply_groups(im.relabel("Before"), **opts_im) + cnt = datashade(cnt, precompute=True, cmap="Viridis").relabel("Before") + cnt = hv.opts.apply_groups(cnt, **opts_cnt) return (im + cnt + hv_im + hv_cnt).cols(2) else: im, cnt = _vis(fm) @@ -1749,7 +1752,7 @@ def visualize_seeds( """ h, w = max_proj.sizes["height"], max_proj.sizes["width"] asp = w / h - pt_cmap = {True: "white", False: "red"} + pt_cmap = ["red", "white"] opts_im = dict(plot=dict(frame_width=600, aspect=asp), style=dict(cmap="Viridis")) opts_pts = dict( plot=dict( @@ -1768,7 +1771,7 @@ def visualize_seeds( opts_pts["style"]["color"] = "white" im = hv.Image(max_proj, kdims=["width", "height"]) pts = hv.Points(seeds, kdims=["width", "height"], vdims=vdims) - return im.opts(**opts_im) * pts.opts(**opts_pts) + return hv.opts.apply_groups(im, **opts_im) * hv.opts.apply_groups(pts, **opts_pts) def visualize_gmm_fit( @@ -1805,14 +1808,15 @@ def gaussian(x, mu, sig): hist = np.histogram(values, bins=bins, density=True) gss_dict = dict() for igss, (mu, sig) in enumerate(zip(gmm.means_, gmm.covariances_)): - gss = gaussian(hist[1], np.asscalar(mu), np.asscalar(np.sqrt(sig))) + gss = gaussian(hist[1], mu.item(), np.sqrt(sig).item()) gss_dict[igss] = hv.Curve((hist[1], gss)) - return ( - hv.Histogram(((hist[0] - hist[0].min()) / np.ptp(hist[0]), hist[1])).opts( - style=dict(alpha=0.6, fill_color="gray") + return hv.opts.apply_groups( + hv.Histogram(((hist[0] - hist[0].min()) / np.ptp(hist[0]), hist[1])).options( + alpha=0.6, fill_color="gray" ) - * hv.NdOverlay(gss_dict) - ).opts(plot=dict(height=350, width=500)) + * hv.NdOverlay(gss_dict), + plot={"height": 350, "width": 500}, + ) def visualize_spatial_update( @@ -1885,8 +1889,11 @@ def visualize_spatial_update( cents_df = centroid(A) hv_pts_dict[key] = hv.Points( cents_df, kdims=["width", "height"], vdims=["unit_id"] - ).opts( - plot=dict(tools=["hover"]), style=dict(fill_alpha=0.2, line_alpha=0, size=8) + ) + hv_pts_dict[key] = hv.opts.apply_groups( + hv_pts_dict[key], + plot=dict(tools=["hover"]), + style=dict(fill_alpha=0.2, line_alpha=0, size=8), ) hv_A_dict[key] = hv.Image( A.sum("unit_id").rename("A"), kdims=["width", "height"] @@ -1908,19 +1915,16 @@ def visualize_spatial_update( hv_C = datashade(hv_C) else: hv_C = Dynamic(hv_C) - hv_A = hv_A.opts(frame_width=400, aspect=w / h, colorbar=True, cmap="viridis") - hv_Ab = hv_Ab.opts(frame_width=400, aspect=w / h, colorbar=True, cmap="viridis") + hv_A = hv_A.options(frame_width=400, aspect=w / h, colorbar=True, cmap="viridis") + hv_Ab = hv_Ab.options(frame_width=400, aspect=w / h, colorbar=True, cmap="viridis") hv_C = hv_C.map( - lambda cr: cr.opts(frame_width=500, frame_height=50), + lambda cr: cr.options(frame_width=500, frame_height=50), hv.RGB if datashading else hv.Curve, ) - return ( - hv.NdLayout( - {"pseudo-color": (hv_pts * hv_A), "binary": (hv_pts * hv_Ab)}, - kdims="Spatial Matrix", - ).cols(1) - + hv_C.relabel("Temporal Components") - ) + return hv.NdLayout( + {"pseudo-color": (hv_pts * hv_A), "binary": (hv_pts * hv_Ab)}, + kdims="Spatial Matrix", + ).cols(1) + hv_C.relabel("Temporal Components") def visualize_temporal_update( @@ -2078,18 +2082,21 @@ def visualize_temporal_update( hv_unit = Dynamic(hv_unit) hv_pul = Dynamic(hv_pul) hv_unit = hv_unit.map( - lambda p: p.opts(plot=dict(frame_height=400, frame_width=1000)) + lambda p: hv.opts.apply_groups(p, plot=dict(frame_height=400, frame_width=1000)) ) - hv_pul = hv_pul.opts(plot=dict(frame_width=500, aspect=w / h)).redim( - t=hv.Dimension("t", soft_range=pul_range) - ) - hv_A = hv_A.opts( - plot=dict(frame_width=500, aspect=w / h), style=dict(cmap="Viridis") + hv_pul = hv.opts.apply_groups( + hv_pul, plot=dict(frame_width=500, aspect=w / h) + ).redim(t=hv.Dimension("t", soft_range=pul_range)) + hv_A = hv.opts.apply_groups( + hv_A, plot=dict(frame_width=500, aspect=w / h), style=dict(cmap="Viridis") ) return ( hv_unit.relabel("Current Unit: Temporal Traces") + hv.NdLayout( - {"Simulated Pulse Response": hv_pul, "Spatial Footprint": hv_A}, + { + "Simulated Pulse Response": hv.NdOverlay(hv_pul), + "Spatial Footprint": hv.NdOverlay(hv_A), + }, kdims="Current Unit", ) ).cols(1) @@ -2124,7 +2131,7 @@ def NNsort(cents: pd.DataFrame) -> pd.Series: result.loc[idu_next] = NNord remain_list.remove(idu_next) for k in range(1, int(np.ceil(np.log2(len(result)))) + 1): - qry = kdtree.query(cents_hw.loc[idu_next], 2 ** k) + qry = kdtree.query(cents_hw.loc[idu_next], 2**k) NNs = qry[1][np.isfinite(qry[0])].squeeze() NNs = NNs[np.sort(np.unique(NNs, return_index=True)[1])] NNs = np.array(result.iloc[NNs].index) @@ -2177,21 +2184,21 @@ def visualize_motion(motion: xr.DataArray) -> Union[hv.Layout, hv.NdOverlay]: mwidth = mwidth.assign_coords(grid=np.arange(mwidth.sizes["grid"])) return ( ( - hv.Image(mheight.rename("height_motion"), kdims=["frame", "grid"]).opts( - title="height_motion", **opts_im - ) - + hv.Image(mwidth.rename("width_motion"), kdims=["frame", "grid"]).opts( - title="width_motion", **opts_im - ) + hv.Image( + mheight.rename("height_motion"), kdims=["frame", "grid"] + ).options(title="height_motion", **opts_im) + + hv.Image( + mwidth.rename("width_motion"), kdims=["frame", "grid"] + ).options(title="width_motion", **opts_im) ) .cols(1) - .opts(show_title=True) + .options(show_title=True) ) else: opts_cv = {"frame_width": 500, "tools": ["hover"], "aspect": 2} return hv.NdOverlay( dict( - width=hv.Curve(motion.sel(shift_dim="width")).opts(**opts_cv), - height=hv.Curve(motion.sel(shift_dim="height")).opts(**opts_cv), + width=hv.Curve(motion.sel(shift_dim="width")).options(**opts_cv), + height=hv.Curve(motion.sel(shift_dim="height")).options(**opts_cv), ) ) diff --git a/pipeline.ipynb b/pipeline.ipynb index 359412d2..2e594207 100644 --- a/pipeline.ipynb +++ b/pipeline.ipynb @@ -243,7 +243,7 @@ "subset_mc = None\n", "interactive = True\n", "output_size = 100\n", - "n_workers = int(os.getenv(\"MINIAN_NWORKERS\", 4))\n", + "n_workers = int(os.getenv(\"MINIAN_NWORKERS\", 6))\n", "param_save_minian = {\n", " \"dpath\": minian_ds_path,\n", " \"meta_dict\": dict(session=-1, animal=-2),\n", @@ -306,6 +306,7 @@ " \"jac_thres\": 0.4,\n", "}\n", "\n", + "os.environ[\"NUMBA_THREADING_LAYER\"] = \"omp\"\n", "os.environ[\"OMP_NUM_THREADS\"] = \"1\"\n", "os.environ[\"MKL_NUM_THREADS\"] = \"1\"\n", "os.environ[\"OPENBLAS_NUM_THREADS\"] = \"1\"\n", @@ -436,9 +437,9 @@ "source": [ "cluster = LocalCluster(\n", " n_workers=n_workers,\n", - " memory_limit=\"2GB\",\n", + " memory_limit=\"8GB\",\n", " resources={\"MEM\": 1},\n", - " threads_per_worker=2,\n", + " threads_per_worker=1,\n", " dashboard_address=\":8787\",\n", ")\n", "annt_plugin = TaskAnnotation()\n", @@ -3875,11 +3876,11 @@ "outputs": [], "source": [ "if interactive:\n", - " A = A.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels))\n", - " C = C.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels))\n", - " S = S.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels))\n", - " c0 = c0.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels))\n", - " b0 = b0.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels))" + " A = A.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels.data))\n", + " C = C.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels.data))\n", + " S = S.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels.data))\n", + " c0 = c0.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels.data))\n", + " b0 = b0.assign_coords(unit_labels=(\"unit_id\", cnmfviewer.unit_labels.data))" ] }, { diff --git a/requirements/requirements-base.txt b/requirements/requirements-base.txt index 84c95e2b..e266a8a5 100644 --- a/requirements/requirements-base.txt +++ b/requirements/requirements-base.txt @@ -1,34 +1,34 @@ -bokeh==1.4.0 -cvxpy>=1.1.11 -dask==2021.2.0 -datashader==0.12.1 -distributed==2021.2.0 -ecos>=2.0.7 -ffmpeg-python==0.2.0 -holoviews==1.12.7 +bokeh~=2.4.3 +cvxpy>=1.2.1 +dask~=2022.9.2 +datashader~=0.14.2 +distributed~=2022.9.2 +ecos>=2.0.10 +ffmpeg-python>=0.2.0 +holoviews~=1.15.1 jupyter matplotlib natsort netcdf4 -networkx==2.4 -numba==0.52.0 -numpy==1.20.2 -opencv-python==4.2.0.34 -pandas==1.2.3 -panel==0.8.0 -pyfftw==0.12.0 -scikit-image==0.18.1 -scikit-learn==0.22.1 -scipy>=1.4.1 +networkx~=2.8.7 +numba~=0.56.2 +numpy>=1.22.4 +opencv-python~=4.6.0 +pandas~=1.5.0 +panel~=0.14.0 +pyfftw~=0.12.0 +scikit-image~=0.19.3 +scikit-learn~=1.1.2 +scipy>=1.9.0 scs -simpleitk==2.0.2 +simpleitk~=2.1.1 sk-video -statsmodels>=0.11.1 +statsmodels>=0.13.2 tifffile -xarray==0.17.0 +xarray~=2022.9.0 zarr -sparse==0.11.2 -pymetis==2020.1 -rechunker==0.3.3 -medpy==0.4.0 -jinja2==2.11.3 +sparse~=0.13.0 +pymetis~=2022.1 +rechunker~=0.5.0 +medpy~=0.4.0 +jinja2~=3.1.2 diff --git a/requirements/requirements-doc.txt b/requirements/requirements-doc.txt index 2dc8f237..ef8e49a4 100644 --- a/requirements/requirements-doc.txt +++ b/requirements/requirements-doc.txt @@ -1,15 +1,15 @@ -sphinx==3.5.3 -pydata-sphinx-theme==0.5.2 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==1.0.3 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.4 +sphinx~=4.5.0 +pydata-sphinx-theme~=0.7.2 +sphinxcontrib-applehelp +sphinxcontrib-devhelp +sphinxcontrib-htmlhelp +sphinxcontrib-jsmath +sphinxcontrib-qthelp +sphinxcontrib-serializinghtml beautifulsoup4 -rtds-action==1.0.3 -nbsphinx==0.8.7 -docutils==0.16 -m2r2==0.2.7 -jinja2==2.11.3 -markupsafe==2.0.1 +rtds-action~=1.0.3 +nbsphinx~=0.8.7 +docutils +m2r2 +jinja2~=3.0.3 +markupsafe~=2.1.1 diff --git a/setup.py b/setup.py index 5abcc06f..e4cfab73 100644 --- a/setup.py +++ b/setup.py @@ -38,5 +38,5 @@ def _get_requirements(requirement_file): "minian-install = minian.install:main", ], }, - python_requires=">=3.8", + python_requires=">=3.9", )