Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Perform data indexing and computations on-demand when required #55

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Perform data indexing and computations on-demand when required #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a7a51cf
Perform data indexing and computations on-demand when required
OssamaRafique May 4, 2023
6b9b039
Typo Fixed
OssamaRafique May 4, 2023
9715483
Merge branch 'main' into only-index-when-required
OssamaRafique May 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion app/scripts/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ class App {
this.visualization.addEventListener("pointHovered", (event) =>
console.log("pointHovered", event)
);
this.visualization.addEventListener("pointClicked", (event) =>
console.log("pointClicked", event)
);
this.visualization.addEventListener("pan", (event) =>
console.log("pan", event)
);
Expand All @@ -46,7 +49,7 @@ class App {

let selem = document.getElementById("specification-select");
selem.value = "tsne-10th";
selem.dispatchEvent(new Event('change'));
selem.dispatchEvent(new Event("change"));

document.getElementById("refresh-specification").click();
}
Expand Down
60 changes: 40 additions & 20 deletions cypress/integration/data-processor.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,8 @@ describe("Box selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One hacky way I've done testing of cached operations is to do something like this:

dataProcessor.expensiveOperationThatIsCached();
for(let i = 0; i < 1000; i++) { // If it's not cached, this will take very long
  dataProcessor.expensiveOperationThatIsCached(); // Should return immediately. 
}

.then(() => {
const { points: allPoints } = dataProcessor.selectBox([1, 1, 7, 7]);

Expand Down Expand Up @@ -218,7 +219,8 @@ describe("Box selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
const { points: allPoints } = dataProcessor.selectBox([1, 1, 7, 7]);
expect(allPoints).to.have.lengthOf(
Expand Down Expand Up @@ -266,7 +268,8 @@ describe("Box selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
const allPoints = dataProcessor.selectBox([1, 1, 7, 7]).points;
expect(allPoints).to.have.lengthOf(
Expand Down Expand Up @@ -312,7 +315,8 @@ describe("Box selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
const allPoints = dataProcessor.selectBox([1, 1, 7, 7]).points;
expect(allPoints).to.have.lengthOf(
Expand Down Expand Up @@ -358,7 +362,8 @@ describe("Box selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
const allPoints = dataProcessor.selectBox([1, 1, 7, 7]).points;
expect(allPoints).to.have.lengthOf(
Expand Down Expand Up @@ -396,7 +401,8 @@ describe("Box selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
const allPoints = dataProcessor.selectBox([
testGenomeScale.toClipSpaceFromString("chr1:1"),
Expand Down Expand Up @@ -475,7 +481,8 @@ describe("Box selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
const allPoints = dataProcessor.selectBox([
testGenomeScale.toClipSpaceFromString("chr1:1"),
Expand Down Expand Up @@ -554,7 +561,8 @@ describe("Box selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
const allPoints = dataProcessor.selectBox([
testGenomeScale.toClipSpaceFromString("chr1:1"),
Expand Down Expand Up @@ -641,7 +649,8 @@ describe("Lasso selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
expect(
dataProcessor.selectLasso([1, 1, 2, 1, 3, 2, 2, 2.5]).points
Expand All @@ -655,7 +664,8 @@ describe("Lasso selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
expect(
dataProcessor.selectLasso([1, 1, 2, 1, 3, 2, 2, 2.5]).points
Expand All @@ -669,7 +679,8 @@ describe("Lasso selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
expect(
dataProcessor.selectLasso([1, 1, 2, 1, 3, 2, 2, 2.5]).points
Expand All @@ -683,7 +694,8 @@ describe("Lasso selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
expect(
dataProcessor.selectLasso([1, 1, 2, 1, 3, 2, 2, 2.5]).points
Expand All @@ -697,7 +709,8 @@ describe("Lasso selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
expect(
dataProcessor.selectLasso([1, 1, 2, 1, 3, 2, 2, 2.5]).points
Expand All @@ -711,7 +724,8 @@ describe("Lasso selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
expect(
dataProcessor.selectLasso([
Expand All @@ -738,7 +752,8 @@ describe("Lasso selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
expect(
dataProcessor.selectLasso([
Expand All @@ -765,7 +780,8 @@ describe("Lasso selection", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
expect(
dataProcessor.selectLasso([
Expand Down Expand Up @@ -797,7 +813,8 @@ describe("Get closest point", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
let closest = dataProcessor.getClosestPoint([1.1, 1.1]).closestPoint;
expect(closest.category).to.eq("a");
Expand All @@ -818,7 +835,8 @@ describe("Get closest point", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
let closest = dataProcessor.getClosestPoint([1, 1]);
expect(closest.closestPoint.category).to.eq("a");
Expand Down Expand Up @@ -846,7 +864,8 @@ describe("Get closest point", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
let closest = dataProcessor.getClosestPoint([1.1, 1.1]);
expect(closest.closestPoint.category).to.eq("a");
Expand Down Expand Up @@ -874,7 +893,8 @@ describe("Get closest point", () => {
);

cy.wrap(dataProcessor)
.should("have.property", "index")
.should("have.property", "specificationHelper")
.then(() => dataProcessor.indexDataIfNotAlreadyIndexed())
.then(() => {
let closest = dataProcessor.getClosestPoint([
testGenomeScale.toClipSpaceFromString("chr1:101"),
Expand Down
44 changes: 27 additions & 17 deletions src/epiviz.gl/data-processor-worker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,41 @@
import DataProcessor from "./data-processor";

self.onmessage = (message) => {
function handleSelection(type, points) {
const selectionMethod = type === "selectBox" ? "selectBox" : "selectLasso";
const selection = self.processor[selectionMethod](points);
postMessage({
type,
selection,
bounds: points,
});
}

function handlePoint(type, point) {
const result = self.processor.getClosestPoint(point);
postMessage({
type,
...result,
});
}

switch (message.data.type) {
case "init":
self.processor = new DataProcessor(message.data.specification);
break;
case "selectBox":
postMessage({
type: message.data.type,
selection: self.processor.selectBox(message.data.points),
bounds: message.data.points,
});
break;
case "selectLasso":
postMessage({
type: message.data.type,
selection: self.processor.selectLasso(message.data.points),
bounds: message.data.points,
});
break;
case "getClosestPoint":
case "getClickPoint":
const result = self.processor.getClosestPoint(message.data.point);
postMessage({
type: message.data.type,
...result,
});
self.processor.indexDataIfNotAlreadyIndexed();
if (
message.data.type === "selectBox" ||
message.data.type === "selectLasso"
) {
handleSelection(message.data.type, message.data.points);
} else {
handlePoint(message.data.type, message.data.point);
}
break;
default:
console.error(`Received unknown message type: ${message.type}`);
Expand Down
44 changes: 29 additions & 15 deletions src/epiviz.gl/data-processor.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,31 @@ class DataProcessor {

console.log("Loading data...");

new SpecificationProcessor(specification, this.indexData.bind(this));
new SpecificationProcessor(
specification,
this.specificationCallback.bind(this)
);
}

/**
* Callback function that occurs after the specification processor has loaded the appropriate data
*
* @param {SpecificationProcessor} specificationHelper that is built in the constructor
*/
indexData(specificationHelper) {

specificationCallback(specificationHelper) {
this.specificationHelper = specificationHelper;
}

/**
* Indexes the data in the specificationHelper and stores it in the data and index fields of the DataProcessor
* object.
*/
indexDataIfNotAlreadyIndexed() {
// If the data has already been indexed or specificationHelper hasn't been built yet, do nothing
if (this.index || !this.specificationHelper) return;
// Otherwise, index the data
const specificationHelper = this.specificationHelper;
let totalPoints = 0;

for (const track of specificationHelper.tracks) {
Expand Down Expand Up @@ -114,15 +130,14 @@ class DataProcessor {
* @returns closest point or undefined
*/
getClosestPoint(point) {
let indices = this.index.neighbors(point[0], point[1], 1, 0)
let pointToReturn =
this.data[indices];
let indices = this.index.neighbors(point[0], point[1], 1, 0);
let pointToReturn = this.data[indices];
let distance = 0;
let isInside = true;
if (pointToReturn === undefined) {
indices = this.index.neighbors(point[0], point[1], 1, 5)
if(indices.length === 0) {
indices = this.index.neighbors(point[0], point[1], 1)
indices = this.index.neighbors(point[0], point[1], 1, 5);
if (indices.length === 0) {
indices = this.index.neighbors(point[0], point[1], 1);
}
pointToReturn = this.data[indices];
distance = Math.sqrt(
Expand All @@ -146,12 +161,11 @@ class DataProcessor {
const largerX = Math.max(points[0], points[2]);
const largerY = Math.max(points[1], points[3]);

let indices = this.index
.search(smallerX, smallerY, largerX, largerY)

let tpoints = indices.map((i) => this.data[i]);
let indices = this.index.search(smallerX, smallerY, largerX, largerY);

let tpoints = indices.map((i) => this.data[i]);

return {indices, "points": tpoints};
return { indices, points: tpoints };
}

/**
Expand Down Expand Up @@ -199,12 +213,12 @@ class DataProcessor {
simplifiedBoundingPolygon
);

if (tbool) findices.push(candidatePoints.indices[i])
if (tbool) findices.push(candidatePoints.indices[i]);

return tbool;
});

return {"indices": findices, "points": fpoints}
return { indices: findices, points: fpoints };
}
}

Expand Down
7 changes: 7 additions & 0 deletions src/epiviz.gl/utilities.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,10 @@ const getQuadraticBezierCurveForPoints = (P0, P1, P2) => {
return (t) => [x(t), y(t)];
};

const POINT_HOVERED_EVENT_NAME = "pointHovered";
const POINT_CLICKED_EVENT_NAME = "pointClicked";
const ON_SELECTION_END_EVENT_NAME = "onSelectionEnd";

export {
scale,
rgbToHex,
Expand All @@ -275,4 +279,7 @@ export {
getQuadraticBezierCurveForPoints,
DEFAULT_WIDTH,
DEFAULT_HEIGHT,
ON_SELECTION_END_EVENT_NAME,
POINT_HOVERED_EVENT_NAME,
POINT_CLICKED_EVENT_NAME,
};
Loading