Skip to content

Commit

Permalink
specs+regime: misalignment detector update
Browse files Browse the repository at this point in the history
  • Loading branch information
@nkemnitz
nkemnitz committed Jan 3, 2024
1 parent 739dd02 commit cbf8a5a
Show file tree
Hide file tree
Showing 6 changed files with 605 additions and 160 deletions.
147 changes: 76 additions & 71 deletions specs/nico/training/aced_misd/preprocess/01_gen_warp_fields.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import "list"
#TGT_IMG_PATH: #BASE_PATH + "datasets/"
#WARPED_SRC_IMG_PATH: #BASE_PATH + "pairwise_aligned/" // + k + "warped_enc/"
#PERLIN_FIELD_PATH: #BASE_PATH + "misd/misalignment_fields/"
#TEMP_PATH1: "gs://tmp_2w/nico/zetta_cvols/"

#DATASETS: {
"microns_pinky": {
Expand All @@ -17,21 +18,21 @@ import "list"
"bounds": [[0, 819200], [0, 983040], [0, 400]]
"resolution": [32, 32, 40]
},
"microns_minnie": {
"contiguous": false
"bounds": [[0, 1703936], [0, 1441792], [0, 320]]
"resolution": [32, 32, 40]
},
"microns_interneuron": {
"contiguous": false
"bounds": [[0, 720896], [0, 720896], [0, 1280]]
"resolution": [32, 32, 40]
},
"aibs_v1dd": {
"contiguous": false
"bounds": [[0.0, 1231667.2], [0.0, 834355.2], [0.0, 1080.0]]
"resolution": [38.8, 38.8, 45.0]
},
// "microns_minnie": {
// "contiguous": false
// "bounds": [[0, 1703936], [0, 1441792], [0, 320]]
// "resolution": [32, 32, 40]
// },
// "microns_interneuron": {
// "contiguous": false
// "bounds": [[0, 720896], [0, 720896], [0, 1280]]
// "resolution": [32, 32, 40]
// },
// "aibs_v1dd": {
// "contiguous": false
// "bounds": [[0.0, 1231667.2], [0.0, 834355.2], [0.0, 1080.0]]
// "resolution": [38.8, 38.8, 45.0]
// },
"kim_n2da": {
"contiguous": true
"bounds": [[0, 32768], [0, 32768], [0, 31050]]
Expand All @@ -42,66 +43,66 @@ import "list"
"bounds": [[0, 229376], [0, 196608], [0, 7320]]
"resolution": [32, 32, 40]
},
"kronauer_cra9": {
"contiguous": true
"bounds": [[0, 393216], [0, 327680], [0, 588]]
"resolution": [32, 32, 42]
},
// "kronauer_cra9": {
// "contiguous": true
// "bounds": [[0, 393216], [0, 327680], [0, 588]]
// "resolution": [32, 32, 42]
// },
"kubota_001": {
"contiguous": true
"bounds": [[0, 204800], [0, 204800], [0, 12000]]
"resolution": [40, 40, 40]
},
"lee_fanc": {
"contiguous": false
"bounds": [[0.0, 352256.0], [0.0, 951091.2], [0.0, 2700.0]]
"resolution": [34.4, 34.4, 45.0]
},
"lee_banc": {
"contiguous": false
"bounds": [[0, 819200], [0, 1015808], [0, 900]]
"resolution": [32, 32, 45]
},
// "lee_fanc": {
// "contiguous": false
// "bounds": [[0.0, 352256.0], [0.0, 951091.2], [0.0, 2700.0]]
// "resolution": [34.4, 34.4, 45.0]
// },
// "lee_banc": {
// "contiguous": false
// "bounds": [[0, 819200], [0, 1015808], [0, 900]]
// "resolution": [32, 32, 45]
// },
"lee_ppc": {
"contiguous": true
"bounds": [[0, 98304], [0, 98304], [0, 36400]]
"resolution": [32, 32, 40]
},
"lee_mosquito": {
"contiguous": false
"bounds": [[0, 704512], [0, 450560], [0, 2240]]
"resolution": [32, 32, 40]
},
"lichtman_zebrafish": {
"contiguous": false
"bounds": [[0, 294912], [0, 393216], [0, 4560]]
"resolution": [32, 32, 30]
},
// "lee_mosquito": {
// "contiguous": false
// "bounds": [[0, 704512], [0, 450560], [0, 2240]]
// "resolution": [32, 32, 40]
// },
// "lichtman_zebrafish": {
// "contiguous": false
// "bounds": [[0, 294912], [0, 393216], [0, 4560]]
// "resolution": [32, 32, 30]
// },
"prieto_godino_larva": {
"contiguous": true
"bounds": [[0, 134976], [0, 144992], [0, 14400]]
"resolution": [32, 32, 32]
},
"fafb_v15": {
"contiguous": false
"bounds": [[0, 884736], [0, 393216], [0, 2000]]
"resolution": [32, 32, 40]
},
"lichtman_h01": {
"contiguous": false
"bounds": [[0, 3440640], [0, 1933312], [0, 198]]
"resolution": [32, 32, 33]
},
// "fafb_v15": {
// "contiguous": false
// "bounds": [[0, 884736], [0, 393216], [0, 2000]]
// "resolution": [32, 32, 40]
// },
// "lichtman_h01": {
// "contiguous": false
// "bounds": [[0, 3440640], [0, 1933312], [0, 198]]
// "resolution": [32, 32, 33]
// },
"janelia_hemibrain": {
"contiguous": true
"bounds": [[0, 317824], [0, 331168], [0, 3296]]
"resolution": [32, 32, 32]
},
"janelia_manc": {
"contiguous": false
"bounds": [[0, 262144], [0, 360448], [0, 5952]]
"resolution": [32, 32, 32]
},
// "janelia_manc": {
// "contiguous": false
// "bounds": [[0, 262144], [0, 360448], [0, 5952]]
// "resolution": [32, 32, 32]
// },
"nguyen_thomas_2022": {
"contiguous": true
"bounds": [[0, 998400], [0, 921600], [0, 400]]
Expand Down Expand Up @@ -156,7 +157,7 @@ import "list"


#DST_INFO_CHUNK_SIZE: [2048, 2048, 1]
#PERLIN_FIELD_DS_FACTOR: math.Pow(2, 3)
#PERLIN_FIELD_DS_FACTOR: math.Pow(2, 0)
#FIELD_INFO_OVERRIDE: {
_dataset_bounds: _
_dst_resolution: _
Expand Down Expand Up @@ -192,28 +193,30 @@ import "list"
#MEDIAN_DISP: 7.5
#PERLIN_NOISE_TEMPLATE: {
_bounds: _
let vx_res = dst_resolution
let x_mult = math.Ceil(((_bounds[0][1] - _bounds[0][0]) / vx_res[0]) / 2048)
let y_mult = math.Ceil(((_bounds[1][1] - _bounds[1][0]) / vx_res[1]) / 2048)
_layer_name: _
"@type": "build_subchunkable_apply_flow"
op: {
"@type": "VolumetricCallableOperation"
fn: {
"@type": "gen_biased_perlin_noise_field"
"@mode": "partial"
shape: [2, x_mult * 2048, y_mult * 2048, 1]
res: [ x_mult * 2, y_mult * 2 ]
shape: [2, 6144 + 2 * processing_blend_pads[0][0], 6144 + 2 * processing_blend_pads[0][1], 1]
res: [ 1, 1 ]
max_displacement_px: #MAX_DISP / #PERLIN_FIELD_DS_FACTOR
field_magn_thr_px: #MEDIAN_DISP / #PERLIN_FIELD_DS_FACTOR
octaves: 8
octaves: 9
persistence: 0.8
device: "cpu"
}
crop_pad: [0, 0, 0]
}
dst_resolution: _
skip_intermediaries: true
processing_chunk_sizes: [[x_mult * 2048, y_mult * 2048, 1]]
processing_crop_pads: [[0, 0, 0]]
processing_chunk_sizes: [[6144, 6144, 1]]
processing_blend_pads: [[1024, 1024, 0]]
processing_blend_modes: "quadratic"
max_reduction_chunk_sizes: [8192, 8192, 1]
level_intermediaries_dirs: [#TEMP_PATH1 + _layer_name + "/raw_perlin_32nm"]
allow_cache_up_to_level: 1
expand_bbox_resolution: true
bbox: {
"@type": "BBox3D.from_coords",
Expand All @@ -222,34 +225,36 @@ import "list"
}
dst: {
"@type": "build_cv_layer"
path: _
path: #PERLIN_FIELD_PATH + _layer_name + "/raw_perlin_32nm"
info_field_overrides: #FIELD_INFO_OVERRIDE & {
_dataset_bounds: _bounds
_dst_resolution: dst_resolution
}
on_info_exists: "overwrite"
}
}


"@type": "mazepa.execute_on_gcp_with_sqs"
worker_image: "us.gcr.io/zetta-research/zetta_utils:nico_py3.9_20231118"
worker_resources: {
memory: "10560Mi"
worker_image: "us-east1-docker.pkg.dev/zetta-research/zutils/zetta_utils:nico_py3.9_20231213"
worker_resource_requests: {
memory: "40560Mi"
}
worker_replicas: 100
worker_replicas: 50
batch_gap_sleep_sec: 0.1
do_dryrun_estimation: true
local_test: false
worker_cluster_project: "zetta-research"
worker_cluster_region: "us-east1"
worker_cluster_name: "zutils-x3"
show_progress: true
target: {
"@type": "mazepa.concurrent_flow"
stages: [
for key, dataset in #DATASETS {
#PERLIN_NOISE_TEMPLATE & {
_bounds: dataset.bounds,
dst: path: #PERLIN_FIELD_PATH + key + "/raw_perlin"
_layer_name: key,

let ds_factor = [#PERLIN_FIELD_DS_FACTOR, #PERLIN_FIELD_DS_FACTOR, 1]
let res = [ for j in [0, 1, 2] {dataset.resolution[j] * ds_factor[j]} ]
Expand Down
Loading

0 comments on commit cbf8a5a

Please sign in to comment.