Merge branch 'main' of https://github.com/skalalab/cell-analysis-tools

cell_analysis_tools/visualization/umap_tsne_pca.py

@@ -33,7 +33,7 @@ def compute_umap(data_values: np.ndarray, **kwargs) -> pd.DataFrame:
 
     reducer = umap.UMAP(random_state=0, **kwargs)
     fit_umap = reducer.fit_transform(data_values)
-    df_umap = pd.DataFrame(fit_umap, columns=["umap_x", "umap_y"])
+    df_umap = pd.DataFrame(fit_umap, columns=["UMAP Dimension 1", "UMAP Dimension 2"])
     return df_umap, reducer
 
 
@@ -68,7 +68,7 @@ def compute_pca(
     principal_components = pca.fit_transform(data_values)
     df_pca = pd.DataFrame(
         data=principal_components,
-        columns=["principal component 1", "principal component 2"],
+        columns=["Principal Component 1", "Principal Component 2"],
     )
     return df_pca, pca
 

examples_and_templates/SDT_sum_decays/SDTzip.bat

@@ -0,0 +1 @@
+SDTZip.exe -z Panc1_60s_summed_512_BH.sdt

examples_and_templates/SDT_sum_decays/helper.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Jun 16 10:19:56 2022
+
+@author: econtrerasguzman
+"""
+
+from pathlib import Path
+import sys
+from PIL import Image
+
+import numpy as np
+
+def _write_sdt(path_output, im, manufacturer="BH", resolution=256):
+
+
+    # Requires the "sdtheader.dat" built header information
+
+    ### Example1 : random dataset
+    #binary_data=(np.random.randint(100,size=[256*256*256])).astype(np.uint16)
+
+    ### Example2 : any data set with 256x256x256 - uint16
+    # with open('badger.dat','rb') as fid:
+    #     binary_data=np.fromstring(fid.read(),np.uint16)    
+
+    #phantom_data= binary_data.ravel().astype(np.uint16)
+
+    path_header = Path(f"./headers/header_{resolution}_{manufacturer}.dat")
+
+    with open(path_header,'rb') as fid:
+        header_ = fid.read() # prebuilt header_file for all 256x256 files
+
+    # convert image
+    if not isinstance(im, np.ndarray):
+        binary_data = im.to_numpy().astype(np.uint16)
+    else:
+        binary_data = im.astype(np.uint16)
+
+    # combine header and data
+    phantom_data = header_ + binary_data.tobytes()
+
+    # with open('phantom_data.sdt','wb') as fid:
+    #     fid.write(phantom_data)
+
+    with open(path_output,'wb') as fid:
+        fid.write(phantom_data)   

examples_and_templates/SDT_sum_decays/sdt_read/README.md

@@ -0,0 +1,2 @@
+# sdt_read
+sdt reader for bruker and wiscscan sdt files

examples_and_templates/SDT_sum_decays/sdt_read/read_bruker_sdt.py

@@ -0,0 +1,82 @@
+import sdtfile
+import numpy as np
+import zipfile
+
+
+def read_sdt_info_brukerSDT(filename):
+    """ 
+    modified from CGohlke sdtfile.py to read bruker 150 card data
+    gives tarr, x.shape,y.shape,t.shape,c.shape
+    """
+    ## HEADER
+    with open(filename, 'rb') as fh:
+        header = np.rec.fromfile(fh, dtype=sdtfile.sdtfile.FILE_HEADER, shape=1, byteorder='<')
+
+    measure_info = []
+    dtype = np.dtype(sdtfile.sdtfile.MEASURE_INFO)
+    with open(filename, 'rb') as fh:
+        fh.seek(header.meas_desc_block_offset[0])
+        for _ in range(header.no_of_meas_desc_blocks[0]):
+            measure_info.append(
+                np.rec.fromfile(fh, dtype=dtype, shape=1, byteorder='<'))
+            fh.seek(header.meas_desc_block_length[0] - dtype.itemsize, 1)
+
+    times = []
+    block_headers = []
+
+    try:
+        routing_channels_x = measure_info[0]['image_rx'][0]
+    except:
+        routing_channels_x = 1
+
+    offset = header.data_block_offset[0]
+
+    print()
+    with open(filename, 'rb') as fh:
+        for _ in range(header.no_of_data_blocks[0]): ## 
+            fh.seek(offset)
+            # read data block header
+            bh = np.rec.fromfile(fh, dtype=sdtfile.sdtfile.BLOCK_HEADER, shape=1,
+                                 byteorder='<')[0]
+            block_headers.append(bh)
+            # read data block
+            mi = measure_info[bh.meas_desc_block_no]
+
+            dtype = sdtfile.sdtfile.BlockType(bh.block_type).dtype
+            dsize = bh.block_length // dtype.itemsize
+
+            t = np.arange(mi.adc_re[0], dtype=np.float64)
+            t *= mi.tac_r / float(mi.tac_g * mi.adc_re)
+            times.append(t)
+            offset = bh.next_block_offs
+        return (times, [mi.scan_x[0], mi.scan_y[0], mi.adc_re[0], routing_channels_x])
+
+
+def read_sdt150(filename):
+    """ sdt bruker uses data_block001 instead of data_block"""
+    import warnings
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+    t, XYTC = read_sdt_info_brukerSDT(filename)
+
+    with zipfile.ZipFile(filename) as myzip:
+        z1 = myzip.infolist()[0]  # "data_block"
+        with myzip.open(z1.filename) as myfile:
+            dataspl = myfile.read()
+
+    dataSDT = np.fromstring(dataspl, np.uint16)
+
+    if XYTC[3] > 1:
+        dataSDT = dataSDT[:XYTC[0] * XYTC[1] * XYTC[2] * XYTC[3]].reshape([XYTC[3], XYTC[0], XYTC[1], XYTC[2]])
+        if dataSDT[0, :, :, :].sum() == 0:  # bruker uses two channels and keeps one empty!!!
+            dataSDT = np.squeeze(dataSDT[1, :, :, :])
+        else:
+            if dataSDT[1, :, :, :].sum() == 0:  # bruker uses two channels and keeps one empty!!!
+                dataSDT = np.squeeze(dataSDT[0, :, :, :])
+                # print ' ch1 is empty'
+            else:
+                pass
+    else:
+        dataSDT = dataSDT[:XYTC[0] * XYTC[1] * XYTC[2] * XYTC[3]].reshape([XYTC[0], XYTC[1], XYTC[2]])
+    #print("READ DATA IN:",dataSDT.shape)
+    return (dataSDT)
+

examples_and_templates/SDT_sum_decays/sdt_read/read_wiscscan_sdt.py

@@ -0,0 +1,71 @@
+import numpy as np
+import sdtfile
+
+def read_sdt_info_wiscscan(filename):
+    """ 
+    modified from CGohlke sdtfile.py to read wiscscan sdt files
+    gives tarr, x.shape,y.shape,t.shape,c.shape
+    """
+    ## HEADER
+    with open(filename, 'rb') as fh:
+        header = np.rec.fromfile(fh, dtype=sdtfile.sdtfile.FILE_HEADER, shape=1, byteorder='<')
+
+    ## PARSING MEASUREMENT DATA BLOCK HEADER
+    measure_info = []
+    dtype = np.dtype(sdtfile.sdtfile.MEASURE_INFO)
+    with open(filename, 'rb') as fh:
+        fh.seek(header.meas_desc_block_offset[0])
+        for _ in range(header.no_of_meas_desc_blocks[0]):
+            measure_info.append(
+                np.rec.fromfile(fh, dtype=dtype, shape=1, byteorder='<'))
+            fh.seek(header.meas_desc_block_length[0] - dtype.itemsize, 1)
+
+    times = []
+    block_headers = []
+
+    try:
+        routing_channels_x = measure_info[0]['image_rx'][0]
+    except:
+        routing_channels_x = 1
+
+    offset = header.data_block_offset[0]
+
+    with open(filename, 'rb') as fh:
+        for _ in range(header.no_of_data_blocks[0]): ## 
+            fh.seek(offset)
+            # read data block header
+            bh = np.rec.fromfile(fh, dtype=sdtfile.sdtfile.BLOCK_HEADER, shape=1,
+                                 byteorder='<')[0]
+            block_headers.append(bh)
+            # read data block
+            mi = measure_info[bh.meas_desc_block_no]
+
+            dtype = sdtfile.sdtfile.BlockType(bh.block_type).dtype
+            dsize = bh.block_length // dtype.itemsize
+
+            t = np.arange(mi.adc_re[0], dtype=np.float64)
+            t *= mi.tac_r / float(mi.tac_g * mi.adc_re)
+            times.append(t)
+            offset = bh.next_block_offs
+        return (times, [mi.scan_x[0], mi.scan_y[0], mi.adc_re[0], routing_channels_x])
+
+
+def read_sdt_wiscscan(filename):
+    """ sdt wiscscan uses a different header"""
+    import warnings
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+    t, XYTC = read_sdt_info_wiscscan(filename)
+    #print(XYTC)
+    #with zipfile.ZipFile(filename) as myzip:
+    #    z1 = myzip.infolist()[0]  # "data_block"
+
+    with open(filename,'rb') as myfile:
+        dataspl = myfile.read()
+
+    uint16_size=2
+    dataSDT = np.fromstring(dataspl[-np.prod(np.array(XYTC))*uint16_size:], np.uint16)
+
+    dataSDT = dataSDT[:XYTC[0] * XYTC[1] * XYTC[2]].reshape([ XYTC[0], XYTC[1], XYTC[2]])
+
+    #print("READ DATA IN:",dataSDT.shape)
+    return (dataSDT)

examples_and_templates/SDT_sum_decays/sum_roi_decays.py

@@ -0,0 +1,132 @@
+from pathlib import Path
+
+from tempfile import TemporaryDirectory
+import shutil
+import matplotlib.pylab as plt
+import matplotlib as mpl
+mpl.rcParams['figure.dpi'] = 300
+
+import tifffile
+
+from natsort import natsorted
+from cell_analysis_tools.visualization import compare_images
+
+from sdt_read.read_bruker_sdt import read_sdt150
+from sdt_read.read_wiscscan_sdt import read_sdt_wiscscan
+import os
+
+from helper import _write_sdt
+import numpy as np
+import re
+
+from tqdm import tqdm
+
+import subprocess
+#%%
+
+debug = False
+
+path_sdts = Path("./sdts")
+path_masks = Path("./masks")
+path_output = Path("./sdts_summed")
+# generate a list of folders to process
+
+list_path_sdts = [sdt for sdt in path_sdts.rglob("*.sdt") if sdt.is_file()]
+
+#SDT files
+list_masks_files = [str(p) for p in natsorted(path_masks.glob("*.tiff"))]
+
+# iterate throught the SDT files
+for idx, path_sdt in tqdm(enumerate(list_path_sdts[:])):
+    pass
+
+    # Find mask
+    base_name = path_sdt.stem
+    path_mask = list(filter(re.compile(f".*{base_name}.*").search, list_masks_files))[0]
+
+    if not Path(path_mask).exists():
+        print(f"Mask not found for : {path_sdt.name}")
+        continue
+
+    # load mask
+    labels = tifffile.imread(path_mask)
+
+    ############## Jenu's code to load SDT's
+    if os.path.getsize(path_sdt) > (2**25): # (file size is equal to 33555190, but ~32 MB is a good marker)
+    	im = read_sdt_wiscscan(path_sdt)
+    else:
+    	im = read_sdt150(path_sdt)
+    ##############
+
+    if debug:
+        compare_images('sdt', im.sum(axis=2), "mask", labels)
+
+    # placeholder array
+    sdt_decay_summed = np.zeros_like(im) 
+
+    # iterate through labels
+    list_labels = [l for l in np.unique(labels) if l != 0]
+    for label in list_labels:
+        pass
+        mask_label = labels == label 
+
+        decay_roi = im * mask_label[...,np.newaxis] # mask decays
+
+        decay_summed = decay_roi.sum(axis=(0,1)) 
+
+        if debug:
+
+            fig, ax = plt.subplots(1,2, figsize=(10,5))
+            fig.suptitle(f"{path_sdt.name} | label: {label}")
+            ax[0].imshow(decay_roi.sum(axis=2))
+            ax[0].set_aspect('equal')
+            ax[1].plot(decay_summed)
+            plt.show()
+            # ax[1].set_aspect('equal')
+
+            # plt.title(f"{path_sdt.name} | label: {label}")
+            # plt.imshow(decay_roi.sum(axis=2))
+            # plt.show()
+
+            # plt.title(f"label: {label}")
+            # plt.plot(decay_summed)
+            # plt.show()
+
+        sdt_decay_summed[mask_label] = decay_summed[np.newaxis, np.newaxis,:]
+
+        # test 512x512x256
+        # temp_array = np.zeros((512,512,256))
+        # temp_array[:256,256:,...] = im
+        # temp_array[:256,:256,...] = im
+        # temp_array[256:,256:,...] = sdt_decay_summed
+        # temp_array[256:,:256,...] = sdt_decay_summed
+        # _write_sdt(path_output / f"{path_sdt.stem}_summed_512_BH.sdt", 
+        #             temp_array, 
+        #             resolution=512, 
+        #             manufacturer="BH")
+        #####
+
+    # create temporary directory to process and compress files
+    with TemporaryDirectory() as tempdir:
+
+        tempdir = Path(tempdir)
+        print(tempdir / f"{path_sdt.stem}_summed.sdt")
+        width, _, _ = im.shape
+        path_file = tempdir / f"{path_sdt.stem}_summed.sdt" 
+        _write_sdt(path_file, sdt_decay_summed, resolution=width)
+
+        #### COMPRESS FILES
+        args = ["SDTZip.exe", "-z", str(path_file)]
+        subprocess.run(args)
+
+        path_compressed_file = path_file.parent / path_file.name.replace(".", ".compressed.")
+        while not path_compressed_file.exists():
+            pass # wait while file is being created
+
+
+        # rename compressed file to uncompressed file
+        shutil.copy2(path_compressed_file, path_output / path_file.name)
+
+
+
+