Skip to content

Commit

Permalink
sharded meshes, manifest calls with verify=false (seung-lab#497)
Browse files Browse the repository at this point in the history
  • Loading branch information
@akhileshh
akhileshh authored Oct 23, 2020
1 parent 0167c5f commit 958d7a5
Show file tree
Hide file tree
Showing 14 changed files with 316 additions and 140 deletions.
5 changes: 4 additions & 1 deletion WHATS_NEW.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,7 @@ NOTE: You may need to do this more than once because Neuroglancer cannot know if
## Additional Keybind to Complete Annotation
`Backquote` key (under `ESC`) can be used to complete an annotation.<sup>*</sup>

<sub>****This serves as an alternative control for when double clicking doesn't work.***</sub>
<sub>****This serves as an alternative control for when double clicking doesn't work.***</sub>

## Sharded meshes
You can now view datasets with sharded meshes ( Minnie65 / `minnie3_v1` ).
1 change: 0 additions & 1 deletion examples/dependent-project/third_party/neuroglancer
View file

This file was deleted.

1 change: 1 addition & 0 deletions examples/dependent-project/third_party/neuroglancer
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../node_modules/neuroglancer/src/neuroglancer
197 changes: 140 additions & 57 deletions src/neuroglancer/datasource/graphene/backend.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {requestAsyncComputation} from 'neuroglancer/async_computation/request';
import {authFetch as cancellableFetchOk, responseIdentity} from 'neuroglancer/authentication/backend';
import {Chunk, ChunkManager, WithParameters} from 'neuroglancer/chunk_manager/backend';
import {GenericSharedDataSource} from 'neuroglancer/chunk_manager/generic_file_source';
import {ChunkedGraphSourceParameters, DataEncoding, MeshSourceParameters, ShardingHashFunction, ShardingParameters, SkeletonSourceParameters, VolumeChunkEncoding, VolumeChunkSourceParameters} from 'neuroglancer/datasource/graphene/base';
import {ChunkedGraphSourceParameters, DataEncoding, MeshSourceParameters, ShardingParameters, SkeletonSourceParameters, VolumeChunkEncoding, VolumeChunkSourceParameters} from 'neuroglancer/datasource/graphene/base';
import {assignMeshFragmentData, decodeJsonManifestChunk, decodeTriangleVertexPositionsAndIndices, decodeTriangleVertexPositionsAndIndicesDraco, FragmentChunk, ManifestChunk, MeshSource} from 'neuroglancer/mesh/backend';
import {SkeletonChunk, SkeletonSource} from 'neuroglancer/skeleton/backend';
import {decodeSkeletonChunk} from 'neuroglancer/skeleton/decode_precomputed_skeleton';
Expand All @@ -33,25 +33,14 @@ import {fetchHttpByteRange} from 'neuroglancer/util/byte_range_http_requests';
import {CancellationToken} from 'neuroglancer/util/cancellation';
import {Borrowed} from 'neuroglancer/util/disposable';
import {convertEndian32, Endianness} from 'neuroglancer/util/endian';
import {murmurHash3_x86_128Hash64Bits} from 'neuroglancer/util/hash';
import {responseArrayBuffer, responseJson} from 'neuroglancer/util/http_request';
import {HttpError, responseArrayBuffer, responseJson} from 'neuroglancer/util/http_request';
import {stableStringify} from 'neuroglancer/util/json';
import {Uint64} from 'neuroglancer/util/uint64';
import {registerPromiseRPC, registerSharedObject, RPCPromise} from 'neuroglancer/worker_rpc';
import {GRAPHENE_MANIFEST_REFRESH_PROMISE} from 'neuroglancer/datasource/graphene/base';
import {GRAPHENE_MANIFEST_REFRESH_PROMISE, GRAPHENE_MANIFEST_SHARDED} from 'neuroglancer/datasource/graphene/base';

const DracoLoader = require('dracoloader');

const shardingHashFunctions: Map<ShardingHashFunction, (out: Uint64) => void> = new Map([
[
ShardingHashFunction.MURMURHASH3_X86_128,
(out) => {
murmurHash3_x86_128Hash64Bits(out, 0, out.low, out.high);
}
],
[ShardingHashFunction.IDENTITY, (_out) => {}],
]);

interface ShardInfo {
shardUrl: string;
offset: Uint64;
Expand All @@ -62,41 +51,37 @@ interface DecodedMinishardIndex {
shardUrl: string;
}

interface MinishardIndexSource extends GenericSharedDataSource<Uint64, DecodedMinishardIndex> {
interface MinishardIndexSource extends GenericSharedDataSource<string, DecodedMinishardIndex> {
sharding: ShardingParameters;
}

function getMinishardIndexDataSource(
chunkManager: Borrowed<ChunkManager>,
parameters: {url: string, sharding: ShardingParameters|undefined}): MinishardIndexSource|
parameters: {url: string, sharding: ShardingParameters|undefined, layer:number}): MinishardIndexSource|
undefined {
const {url, sharding} = parameters;
const {url, sharding, layer} = parameters;
if (sharding === undefined) return undefined;
const source =
GenericSharedDataSource.get<Uint64, DecodedMinishardIndex>(
GenericSharedDataSource.get<string, DecodedMinishardIndex>(
chunkManager, stableStringify({type: 'graphene:shardedDataSource', url, sharding}), {
download: async function(
shardAndMinishard: Uint64, cancellationToken: CancellationToken) {
const minishard = Uint64.lowMask(new Uint64(), sharding.minishardBits);
Uint64.and(minishard, minishard, shardAndMinishard);
const shard = Uint64.lowMask(new Uint64(), sharding.shardBits);
const temp = new Uint64();
Uint64.rshift(temp, shardAndMinishard, sharding.minishardBits);
Uint64.and(shard, shard, temp);
const shardUrl = `${url}/${
shard.toString(16).padStart(Math.ceil(sharding.shardBits / 4), '0')}.shard`;
// Retrive minishard index start/end offsets.
shardFileAndMiniShard: string, cancellationToken: CancellationToken) {
const parts = shardFileAndMiniShard.split(':');
const shardFile = parts[0];
const miniShard: Uint64 = Uint64.parseString(parts[1]);
const shardUrl = `${url}/initial/${layer}/${shardFile}`;
// Retrive miniShard index start/end offsets.

const shardIndexSize = new Uint64(16);
Uint64.lshift(shardIndexSize, shardIndexSize, sharding.minishardBits);

// Multiply minishard by 16.
const shardIndexStart = Uint64.lshift(new Uint64(), minishard, 4);
// Multiply miniShard by 16.
const shardIndexStart = Uint64.lshift(new Uint64(), miniShard, 4);
const shardIndexEnd = Uint64.addUint32(new Uint64(), shardIndexStart, 16);
const shardIndexResponse = await fetchHttpByteRange(
shardUrl, shardIndexStart, shardIndexEnd, cancellationToken);
if (shardIndexResponse.byteLength !== 16) {
throw new Error(`Failed to retrieve minishard offset`);
throw new Error(`Failed to retrieve miniShard offset`);
}
const shardIndexDv = new DataView(shardIndexResponse);
const minishardStartOffset = new Uint64(
Expand Down Expand Up @@ -124,14 +109,14 @@ function getMinishardIndexDataSource(
}
if ((minishardIndexResponse.byteLength % 24) !== 0) {
throw new Error(
`Invalid minishard index length: ${minishardIndexResponse.byteLength}`);
`Invalid miniShard index length: ${minishardIndexResponse.byteLength}`);
}
const minishardIndex = new Uint32Array(minishardIndexResponse);
convertEndian32(minishardIndex, Endianness.LITTLE);

const minishardIndexSize = minishardIndex.byteLength / 24;
let prevEntryKeyLow = 0, prevEntryKeyHigh = 0;
// Offsets in the minishard index are relative to the end of the shard index.
// Offsets in the miniShard index are relative to the end of the shard index.
let prevStartLow = shardIndexSize.low, prevStartHigh = shardIndexSize.high;
for (let i = 0; i < minishardIndexSize; ++i) {
let entryKeyLow = prevEntryKeyLow + minishardIndex[i * 2];
Expand Down Expand Up @@ -165,13 +150,29 @@ function getMinishardIndexDataSource(
}
return {data: {data: minishardIndex, shardUrl}, size: minishardIndex.byteLength};
},
encodeKey: (key: Uint64) => key.toString(),
encodeKey: (key: string) => key,
sourceQueueLevel: 1,
}) as MinishardIndexSource;
source.sharding = sharding;
return source;
}

function getGrapheneMinishardIndexDataSources(
chunkManager: Borrowed<ChunkManager>,
parameters: {url: string, sharding: Array<ShardingParameters>|undefined}): Array<MinishardIndexSource>|
undefined {
const {url, sharding} = parameters;
if (sharding === undefined) return undefined;
const sources = new Array<MinishardIndexSource>();
for (const index in sharding)
{
const layer = Number(index);
sources[layer] = getMinishardIndexDataSource(
chunkManager, {url: url, sharding: sharding[layer], layer:layer})!;
}
return sources;
}

function findMinishardEntry(minishardIndex: DecodedMinishardIndex, key: Uint64) {
const minishardIndexData = minishardIndex.data;
const minishardIndexSize = minishardIndexData.length / 6;
Expand All @@ -194,16 +195,10 @@ function findMinishardEntry(minishardIndex: DecodedMinishardIndex, key: Uint64)
async function getShardedData(
minishardIndexSource: MinishardIndexSource, chunk: Chunk, key: Uint64,
cancellationToken: CancellationToken): Promise<{shardInfo: ShardInfo, data: ArrayBuffer}> {
const {sharding} = minishardIndexSource;
const hashFunction = shardingHashFunctions.get(sharding.hash)!;
const hashCode = Uint64.rshift(new Uint64(), key, sharding.preshiftBits);
hashFunction(hashCode);
const shardAndMinishard =
Uint64.lowMask(new Uint64(), sharding.minishardBits + sharding.shardBits);
Uint64.and(shardAndMinishard, shardAndMinishard, hashCode);
const parts = (chunk as FragmentChunk).fragmentId!.split(':');
const getPriority = () => ({priorityTier: chunk.priorityTier, priority: chunk.priority});
const minishardIndex =
await minishardIndexSource.getData(shardAndMinishard, getPriority, cancellationToken);
await minishardIndexSource.getData(`${parts[3]}:${parts[4]}`, getPriority, cancellationToken);
const {startOffset, endOffset} = findMinishardEntry(minishardIndex, key);
let data =
await fetchHttpByteRange(minishardIndex.shardUrl, startOffset, endOffset, cancellationToken);
Expand All @@ -224,7 +219,6 @@ chunkDecoders.set(VolumeChunkEncoding.COMPRESSED_SEGMENTATION, decodeCompressedS
@registerSharedObject() export class GrapheneVolumeChunkSource extends
(WithParameters(VolumeChunkSource, VolumeChunkSourceParameters)) {
chunkDecoder = chunkDecoders.get(this.parameters.encoding)!;

async download(chunk: VolumeChunk, cancellationToken: CancellationToken) {
const {parameters} = this;
let url: string;
Expand Down Expand Up @@ -309,21 +303,107 @@ export function decodeDracoFragmentChunk(
chunk, decodeTriangleVertexPositionsAndIndicesDraco(response, decoderModule));
}

@registerSharedObject() export class GrapheneMeshSource extends
(WithParameters(MeshSource, MeshSourceParameters)) {
download(chunk: ManifestChunk, cancellationToken: CancellationToken) {
const {parameters} = this;
return cancellableFetchOk(
`${parameters.manifestUrl}/manifest/${chunk.objectId}:${parameters.lod}?verify=True`,
{}, responseJson, cancellationToken)
.then(response => decodeManifestChunk(chunk, response));
async function getUnverifiedFragmentPromise(
chunk: FragmentChunk,
parameters: MeshSourceParameters,
minishardIndexSources: MinishardIndexSource[],
cancellationToken: CancellationToken) {
if (chunk.fragmentId && chunk.fragmentId.charAt(0) === '~'){
let objectId = Uint64.parseString(chunk.key!);
let layer = Number(chunk.fragmentId.substr(1).split(':')[1]);
let data: ArrayBuffer;
({data} =
await getShardedData(minishardIndexSources[layer]!, chunk, objectId, cancellationToken));
return Promise.resolve(data);
}
return cancellableFetchOk(
`${parameters.fragmentUrl}/dynamic/${chunk.fragmentId}`, {}, responseArrayBuffer,
cancellationToken);
}


function getVerifiedFragmentPromise(
chunk: FragmentChunk,
parameters: MeshSourceParameters,
cancellationToken: CancellationToken) {
if (chunk.fragmentId && chunk.fragmentId.charAt(0) === '~') {
let parts = chunk.fragmentId.substr(1).split(':');
let startOffset: Uint64|number, endOffset: Uint64|number;
startOffset = Number(parts[1]);
endOffset = startOffset+Number(parts[2]);
return fetchHttpByteRange(
`${parameters.fragmentUrl}/initial/${parts[0]}`,
startOffset,
endOffset,
cancellationToken
);
}
return cancellableFetchOk(
`${parameters.fragmentUrl}/dynamic/${chunk.fragmentId}`, {}, responseArrayBuffer,
cancellationToken);
}


function getFragmentDownloadPromise(
chunk: FragmentChunk,
parameters: MeshSourceParameters,
minishardIndexSources: MinishardIndexSource[],
cancellationToken: CancellationToken
) {
let fragmentDownloadPromise;
if (parameters.sharding){
if (chunk.verifyFragment !== undefined && !chunk.verifyFragment) {
// Download shard fragments without verification
fragmentDownloadPromise =
getUnverifiedFragmentPromise(chunk, parameters, minishardIndexSources, cancellationToken);
}
else {
// Download shard fragments with verification (response contains size and offset)
fragmentDownloadPromise = getVerifiedFragmentPromise(chunk, parameters, cancellationToken);
}
} else {
fragmentDownloadPromise = cancellableFetchOk(
`${parameters.fragmentUrl}/${chunk.fragmentId}`, {}, responseArrayBuffer,
cancellationToken);
}
return fragmentDownloadPromise;
}

@registerSharedObject() //
export class GrapheneMeshSource extends
(WithParameters(MeshSource, MeshSourceParameters)) {
protected minishardIndexSources: MinishardIndexSource[];
async download(chunk: ManifestChunk, cancellationToken: CancellationToken) {
const {parameters} = this;
let url = `${parameters.manifestUrl}/manifest`;
let manifestUrl = `${url}/${chunk.objectId}:${parameters.lod}?verify=1&prepend_seg_ids=1`;

// speculative manifest isn't working all the time
// race condition is the prime suspect so use verify=true
chunk.verifyFragments = true;

// parameters.sharding is a proxy for mesh format
// if undefined, mesh format is old else new
if (parameters.sharding !== undefined) {
chunk.manifestType = GRAPHENE_MANIFEST_SHARDED;
if (this.minishardIndexSources === undefined) {
this.minishardIndexSources = getGrapheneMinishardIndexDataSources(
this.chunkManager, {url: parameters.fragmentUrl, sharding: parameters.sharding})!;
}
if (!chunk.verifyFragments) {
manifestUrl = `${url}/${chunk.objectId}:${parameters.lod}?verify=0&prepend_seg_ids=1`;
}
}
await cancellableFetchOk(manifestUrl, {}, responseJson, cancellationToken)
.then(response => decodeManifestChunk(chunk, response));
}

async downloadFragment(chunk: FragmentChunk, cancellationToken: CancellationToken) {
const {parameters, minishardIndexSources} = this;
const fragmentDownloadPromise = getFragmentDownloadPromise(
chunk, parameters, minishardIndexSources, cancellationToken
);

downloadFragment(chunk: FragmentChunk, cancellationToken: CancellationToken) {
const {parameters} = this;
const fragmentDownloadPromise = cancellableFetchOk(
`${parameters.fragmentUrl}/${chunk.fragmentId}`, {}, responseArrayBuffer,
cancellationToken);
const dracoModulePromise = DracoLoader.default;
const readyToDecode = Promise.all([fragmentDownloadPromise, dracoModulePromise]);
return readyToDecode.then(
Expand All @@ -338,6 +418,9 @@ export function decodeDracoFragmentChunk(
}
},
error => {
if (error instanceof HttpError && error.status == 404) {
chunk.source!.removeChunk(chunk);
}
Promise.reject(error);
});
}
Expand All @@ -347,7 +430,7 @@ export function decodeDracoFragmentChunk(
export class GrapheneSkeletonSource extends
(WithParameters(SkeletonSource, SkeletonSourceParameters)) {
private minishardIndexSource = getMinishardIndexDataSource(
this.chunkManager, {url: this.parameters.url, sharding: this.parameters.metadata.sharding});
this.chunkManager, {url: this.parameters.url, sharding: this.parameters.metadata.sharding, layer:0});
async download(chunk: SkeletonChunk, cancellationToken: CancellationToken) {
const {minishardIndexSource, parameters} = this;
let response: ArrayBuffer;
Expand Down
6 changes: 4 additions & 2 deletions src/neuroglancer/datasource/graphene/base.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {VertexAttributeInfo} from 'neuroglancer/skeleton/base';
import {mat4} from 'neuroglancer/util/geom';

export const PYCG_APP_VERSION = 1;
export const GRAPHENE_MANIFEST_SHARDED = 'GrapheneSharded';
export const GRAPHENE_MANIFEST_REFRESH_PROMISE = 'GrapheneMeshSource.RefreshManifestPromise';

export enum VolumeChunkEncoding {
Expand All @@ -44,6 +45,8 @@ export class MeshSourceParameters {
manifestUrl: string;
fragmentUrl: string;
lod: number;
sharding: Array<ShardingParameters>|undefined;
verifyMesh: boolean;

static RPC_ID = 'graphene/MeshSource';
}
Expand Down Expand Up @@ -72,11 +75,10 @@ export class MultiscaleMeshMetadata {
transform: mat4;
lodScaleMultiplier: number;
vertexQuantizationBits: number;
sharding: ShardingParameters|undefined;
sharding: Array<ShardingParameters>|undefined;
}

export class MultiscaleMeshSourceParameters {
url: string;
metadata: MultiscaleMeshMetadata;

static RPC_ID = 'graphene/MultiscaleMeshSource';
Expand Down
Loading

0 comments on commit 958d7a5

Please sign in to comment.