Skip to content

Commit

Permalink
semantic training
Browse files Browse the repository at this point in the history
  • Loading branch information
xiuliren committed Aug 29, 2022
1 parent 93fc8c2 commit 8279b46
Show file tree
Hide file tree
Showing 12 changed files with 663 additions and 20 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ Neuron segmentation and synapse detection using PyTorch
# Features
- [x] Training using whole terabyte or even petabyte of image volume.
- [x] Training using multiple version of image datasets as data augmentation.
- [x] Data augmentation without zero-filling.

# Install
python setup.py install
Expand Down
286 changes: 286 additions & 0 deletions environment.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,286 @@
name: pytorch
channels:
- pytorch
- nvidia
- conda-forge
- defaults
dependencies:
- _libgcc_mutex=0.1=conda_forge
- _openmp_mutex=4.5=1_gnu
- backcall=0.2.0=pyhd3eb1b0_0
- blas=1.0=mkl
- blas-devel=3.9.0=11_linux64_mkl
- bzip2=1.0.8=h7b6447c_0
- c-ares=1.18.1=h7f8727e_0
- ca-certificates=2021.10.26=h06a4308_2
- certifi=2021.10.8=py38h06a4308_2
- charset-normalizer=2.0.4=pyhd3eb1b0_0
- conda=4.11.0=py38h578d9bd_0
- conda-package-handling=1.7.3=py38h27cfd23_1
- cudatoolkit=11.1.74=h6bb024c_0
- ffmpeg=4.2.2=h20bf706_0
- freetype=2.11.0=h70c0345_0
- giflib=5.2.1=h7b6447c_0
- gmp=6.2.1=h2531618_2
- gnutls=3.6.15=he1e5248_0
- icu=58.2=he6710b0_3
- intel-openmp=2021.4.0=h06a4308_3561
- ipython=7.26.0=py38hb070fc8_0
- jedi=0.18.0=py38h06a4308_1
- jpeg=9b=h024ee3a_2
- krb5=1.19.2=hac12032_0
- lame=3.100=h7b6447c_0
- lcms2=2.12=h3be6417_0
- ld_impl_linux-64=2.35.1=h7274673_9
- libarchive=3.4.2=h62408e4_0
- libblas=3.9.0=11_linux64_mkl
- libcblas=3.9.0=11_linux64_mkl
- libcurl=7.80.0=h0b77cf5_0
- libedit=3.1.20210910=h7f8727e_0
- libev=4.33=h7f8727e_1
- libffi=3.3=he6710b0_2
- libgcc-ng=11.2.0=h1d223b6_11
- libgomp=11.2.0=h1d223b6_11
- libidn2=2.3.2=h7f8727e_0
- liblapack=3.9.0=11_linux64_mkl
- liblapacke=3.9.0=11_linux64_mkl
- libnghttp2=1.46.0=hce63b2e_0
- libopus=1.3.1=h7b6447c_0
- libpng=1.6.37=hbc83047_0
- libsolv=0.7.16=h8b12597_0
- libssh2=1.9.0=h1ba5d50_1
- libstdcxx-ng=11.2.0=he4da1e4_11
- libtasn1=4.16.0=h27cfd23_0
- libtiff=4.2.0=h85742a9_0
- libunistring=0.9.10=h27cfd23_0
- libuv=1.40.0=h7b6447c_0
- libvpx=1.7.0=h439df22_0
- libwebp=1.2.0=h89dd481_0
- libwebp-base=1.2.0=h27cfd23_0
- libxml2=2.9.12=h03d6c58_0
- lz4-c=1.9.3=h295c915_1
- mamba=0.7.3=py38h9709c9f_0
- matplotlib-inline=0.1.2=pyhd3eb1b0_2
- mkl=2021.3.0=h06a4308_520
- mkl-devel=2021.3.0=h66538d2_520
- mkl-include=2021.3.0=h06a4308_520
- mkl-service=2.4.0=py38h7f8727e_0
- mkl_fft=1.3.1=py38hd3c417c_0
- mkl_random=1.2.2=py38h51133e4_0
- ncurses=6.3=h7f8727e_2
- nettle=3.7.3=hbbd107a_1
- ninja=1.10.2=py38hd09550d_3
- olefile=0.46=pyhd3eb1b0_0
- openh264=2.1.1=h4ff587b_0
- openssl=1.1.1m=h7f8727e_0
- parso=0.8.3=pyhd3eb1b0_0
- pexpect=4.8.0=pyhd3eb1b0_3
- pickleshare=0.7.5=pyhd3eb1b0_1003
- pip=21.2.4=py38h06a4308_0
- prompt-toolkit=3.0.20=pyhd3eb1b0_0
- ptyprocess=0.7.0=pyhd3eb1b0_2
- pybind11=2.8.1=py38hd09550d_0
- pycosat=0.6.3=py38h7b6447c_1
- pygments=2.10.0=pyhd3eb1b0_0
- pysocks=1.7.1=py38h06a4308_0
- python=3.8.12=h12debd9_0
- python_abi=3.8=2_cp38
- pytorch=1.9.1=py3.8_cuda11.1_cudnn8.0.5_0
- readline=8.1.2=h7f8727e_1
- reproc=14.2.1=h36c2ea0_0
- reproc-cpp=14.2.1=h58526e2_0
- ruamel_yaml=0.15.100=py38h27cfd23_0
- setuptools=58.0.4=py38h06a4308_0
- sqlite=3.37.0=hc218d9a_0
- tk=8.6.11=h1ccaba5_0
- torchaudio=0.9.1=py38
- traitlets=5.1.1=pyhd3eb1b0_0
- typing_extensions=3.10.0.2=pyh06a4308_0
- wcwidth=0.2.5=pyhd3eb1b0_0
- x264=1!157.20191217=h7b6447c_0
- xtensor=0.24.0=h4bd325d_0
- xtensor-blas=0.20.0=h4bd325d_0
- xtensor-python=0.26.0=py38h2b96118_1
- xtl=0.7.4=h4bd325d_0
- xz=5.2.5=h7b6447c_0
- yaml=0.2.5=h7b6447c_0
- zlib=1.2.11=h7f8727e_4
- zstd=1.4.9=haebb681_0
- pip:
- absl-py==0.12.0
- argon2-cffi==21.3.0
- astunparse==1.6.3
- attrs==21.1.0
- awscli==1.19.72
- bleach==4.1.0
- boto3==1.17.68
- botocore==1.20.112
- brotli==1.0.9
- brotlipy==0.7.0
- cachetools==4.2.2
- cffi==1.14.5
- chardet==4.0.0
- click==8.1.3
- cloud-files==4.9.1
- cloud-volume==8.8.1
- colorama==0.4.3
- compressed-segmentation==2.2.0
- compresso==3.0.0
- connected-components-3d==3.2.0
- crc32c==2.2
- cryptography==3.4.7
- cycler==0.10.0
- cython==0.29.21
- decorator==4.4.2
- deflate==0.3.0
- defusedxml==0.7.1
- deprecated==1.2.12
- detect-secrets==1.1.0
- dijkstra3d==1.9.1
- dill==0.3.3
- docutils==0.15.2
- dracopy==1.1.1
- edt==2.1.0
- einops==0.4.0
- entrypoints==0.3
- fasteners==0.16
- fastremap==1.13.2
- fill-voids==2.0.1
- flatbuffers==2.0
- fpzip==1.1.4
- fvcore==0.1.5.post20220512
- gast==0.5.3
- gevent==21.1.2
- google-api-core==1.26.3
- google-apitools==0.5.32
- google-auth==1.30.0
- google-auth-oauthlib==0.4.4
- google-cloud-core==1.6.0
- google-cloud-storage==1.38.0
- google-crc32c==1.1.2
- google-pasta==0.2.0
- google-resumable-media==1.2.0
- googleapis-common-protos==1.53.0
- greenlet==1.1.0
- grpcio==1.37.1
- h5py==3.2.1
- httplib2==0.19.1
- humanize==3.5.0
- idna==2.10
- imageio==2.9.0
- inflection==0.5.1
- iniconfig==1.1.1
- iopath==0.1.10
- ipykernel==6.7.0
- ipywidgets==8.0.0b1
- jinja2==3.0.3
- jmespath==0.10.0
- joblib==1.0.1
- json5==0.9.5
- jsonschema==3.2.0
- jupyter==1.0.0
- jupyter-client==7.1.2
- jupyter-console==6.4.0
- jupyter-core==4.9.1
- jupyterlab-pygments==0.1.2
- jupyterlab-widgets==2.0.0b1
- keras==2.8.0rc1
- keras-preprocessing==1.1.2
- kimimaro==2.1.1
- kiwisolver==1.3.1
- libclang==12.0.0
- markdown==3.3.4
- matplotlib==3.4.1
- mistune==0.8.4
- multiprocess==0.70.11.1
- nbclient==0.5.10
- nbconvert==6.4.0
- nbformat==5.1.3
- nest-asyncio==1.5.4
- networkx==2.5.1
- neuroglancer==2.29
- nibabel==3.2.1
- nose==1.3.7
- notebook==6.4.7
- numpy==1.20.2
- oauth2client==4.1.3
- oauthlib==3.1.0
- opencv-python==4.5.1.48
- opt-einsum==3.3.0
- orjson==3.5.2
- packaging==20.9
- pandas==1.4.3
- pandocfilters==1.5.0
- pathos==0.2.7
- pillow==8.2.0
- pluggy==0.13.1
- portalocker==2.5.1
- posix-ipc==1.0.5
- pox==0.2.9
- ppft==1.6.6.3
- prometheus-client==0.12.0
- protobuf==3.15.8
- psutil==5.4.3
- py==1.10.0
- pyasn1==0.4.8
- pyasn1-modules==0.2.8
- pycparser==2.20
- pynrrd==0.4.2
- pyopenssl==20.0.1
- pyparsing==2.4.7
- pyrsistent==0.17.3
- pysimdjson==3.2.0
- pyspng-seunglab==1.0.0
- pytest==6.2.4
- python-dateutil==2.8.1
- python-jsonschema-objects==0.3.14
- pytz==2021.1
- pywavelets==1.1.1
- pyyaml==5.4.1
- pyzmq==22.3.0
- qtconsole==5.2.2
- qtpy==2.0.0
- requests==2.25.1
- requests-oauthlib==1.3.0
- rsa==4.7.2
- s3transfer==0.4.0
- scikit-image==0.18.1
- scikit-learn==0.24.2
- scipy==1.6.3
- send2trash==1.8.1b0
- simpleitk==1.2.4
- simplejpeg==1.6.0
- six==1.15.0
- sockjs-tornado==1.0.7
- tabulate==0.8.10
- tenacity==7.0.0
- tensorboard==2.8.0
- tensorboard-data-server==0.6.0
- tensorboard-plugin-wit==1.8.0
- tensorflow==2.8.0rc1
- tensorflow-io-gcs-filesystem==0.23.1
- terminado==0.12.1
- testpath==0.5.0
- tf-estimator-nightly==2.8.0.dev2021122109
- threadpoolctl==2.1.0
- tifffile==2021.4.8
- tinybrain==1.2.1
- toml==0.10.2
- torch==1.8.1
- torch-tb-profiler==0.2.1
- torchio==0.18.36
- torchvision==0.9.1
- tornado==6.1
- tqdm==4.64.0
- typing-extensions==3.7.4.3
- urllib3==1.26.4
- werkzeug==1.0.1
- wheel==0.37.1
- widgetsnbextension==4.0.0b1
- wrapt==1.12.1
- yacs==0.1.8
- zmesh==0.5.0
- zope-event==4.5.0
- zope-interface==5.4.0
- zstandard==0.15.2
prefix: /mnt/home/jwu/anaconda3/envs/pytorch
13 changes: 11 additions & 2 deletions neutorch/dataset/base.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from abc import abstractproperty
from typing import Union
from functools import cached_property
import math
Expand All @@ -24,6 +25,12 @@ def worker_init_fn(worker_id: int):
dataset.start = overall_start + worker_id * per_worker
dataset.end = min(dataset.start + per_worker, overall_end)

def path_to_dataset_name(path: str, dataset_names: list):
for dataset_name in dataset_names:
if dataset_name in path:
return dataset_name



class DatasetBase(torch.utils.data.IterableDataset):
def __init__(self,
Expand All @@ -48,8 +55,10 @@ def __init__(self,
self.transform.shrink_size[:3] + \
self.transform.shrink_size[-3:]

# inherite this class and build the samples
self.samples = None
@cached_property
@abstractproperty
def samples(self):
pass

@cached_property
def sample_num(self):
Expand Down
Loading

0 comments on commit 8279b46

Please sign in to comment.