Skip to content

Commit

Permalink
feat: implements batch file upload with promises
Browse files Browse the repository at this point in the history 
- Updated `fetchBatchSize` in `DataPackage.js` to replace `batchModeValue`.
- Added `uploadBatchSize` configuration in `AppModel.js`.
- Implemented `uploadFilesInBatch` method in `DataItemView.js` to handle batch file uploads using promises.
- Ensured that the `_.each` loop completes before proceeding to batch processing.

Closes #2224
  • Loading branch information
@vchendrix
vchendrix committed Dec 19, 2024
1 parent 3ba5300 commit c270e18
Show file tree
Hide file tree
Showing 3 changed files with 212 additions and 123 deletions.
179 changes: 90 additions & 89 deletions src/js/collections/DataPackage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -404,106 +404,107 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5",
* @param {number} [batchSize=10] - The number of models to fetch in each batch.
* @param {number} [timeout=5000] - The timeout for each fetch request in milliseconds.
* @param {number} [maxRetries=3] - The maximum number of retries for each fetch request.
* @since 0.0.0
*/
fetchMemberModels(models, index = 0, batchSize = 10, timeout = 5000, maxRetries = 3) {
// Update the number of file metadata items being loaded
this.packageModel.set("numLoadingFileMetadata", models.length - index);
// Update the number of file metadata items being loaded
this.packageModel.set("numLoadingFileMetadata", models.length - index);

// If the index is greater than or equal to the length of the models array, stop fetching
if (index >= models.length) {
this.triggerComplete();
return;
}
// If the index is greater than or equal to the length of the models array, stop fetching
if (index >= models.length) {
this.triggerComplete();
return;
}

// If batchSize is 0, set it to the total number of models
if (batchSize == 0) batchSize = models.length;

const collection = this;
// Slice the models array to get the current batch
const batch = models.slice(index, index + batchSize);

// Create an array of promises for fetching each model in the batch
const fetchPromises = batch.map((memberModel) => {
return new Promise((resolve, reject) => {
const attemptFetch = (retriesLeft) => {
// Create a promise for the fetch request
const fetchPromise = new Promise((fetchResolve, fetchReject) => {
memberModel.fetch({
success: () => {
// Once the model is synced, handle the response
memberModel.once("sync", (oldModel) => {
const newModel = collection.getMember(oldModel);

// If the type of the old model is different from the new model
if (oldModel.type != newModel.type) {
if (newModel.type == "DataPackage") {
// If the new model is a DataPackage, replace the old model with the new one
oldModel.trigger("replace", newModel);
fetchResolve();
} else {
// Otherwise, fetch the new model and replace the old model with the new one
newModel.set("synced", false);
newModel.fetch();
newModel.once("sync", (fetchedModel) => {
fetchedModel.set("synced", true);
collection.remove(oldModel);
collection.add(fetchedModel);
oldModel.trigger("replace", newModel);
// If batchSize is 0, set it to the total number of models
if (batchSize == 0) batchSize = models.length;

const collection = this;
// Slice the models array to get the current batch
const batch = models.slice(index, index + batchSize);

// Create an array of promises for fetching each model in the batch
const fetchPromises = batch.map((memberModel) => {
return new Promise((resolve, reject) => {
const attemptFetch = (retriesLeft) => {
// Create a promise for the fetch request
const fetchPromise = new Promise((fetchResolve, fetchReject) => {
memberModel.fetch({
success: () => {
// Once the model is synced, handle the response
memberModel.once("sync", (oldModel) => {
const newModel = collection.getMember(oldModel);

// If the type of the old model is different from the new model
if (oldModel.type != newModel.type) {
if (newModel.type == "DataPackage") {
// If the new model is a DataPackage, replace the old model with the new one
oldModel.trigger("replace", newModel);
fetchResolve();
} else {
// Otherwise, fetch the new model and replace the old model with the new one
newModel.set("synced", false);
newModel.fetch();
newModel.once("sync", (fetchedModel) => {
fetchedModel.set("synced", true);
collection.remove(oldModel);
collection.add(fetchedModel);
oldModel.trigger("replace", newModel);
if (newModel.type == "EML") collection.trigger("add:EML");
fetchResolve();
});
}
} else {
// If the type of the old model is the same as the new model, merge the new model into the collection
newModel.set("synced", true);
collection.add(newModel, { merge: true });
if (newModel.type == "EML") collection.trigger("add:EML");
fetchResolve();
});
}
}
});
},
error: (model, response) => fetchReject(new Error(response.statusText))
});
});

// Create a promise for the timeout
const timeoutPromise = new Promise((_, timeoutReject) => {
setTimeout(() => timeoutReject(new Error("Fetch timed out")), timeout);
});

// Race the fetch promise against the timeout promise
Promise.race([fetchPromise, timeoutPromise])
.then(resolve)
.catch((error) => {
if (retriesLeft > 0) {
// Retry the fetch if there are retries left
console.warn(`Retrying fetch for model: ${memberModel.id}, retries left: ${retriesLeft}, error: ${error}`);
attemptFetch(retriesLeft - 1);
} else {
// If the type of the old model is the same as the new model, merge the new model into the collection
newModel.set("synced", true);
collection.add(newModel, { merge: true });
if (newModel.type == "EML") collection.trigger("add:EML");
fetchResolve();
// Reject the promise if all retries are exhausted
console.error(`Failed to fetch model: ${memberModel.id} after ${maxRetries} retries, error: ${error}`);
reject(error);
}
});
},
error: (model, response) => fetchReject(new Error(response.statusText))
});
});
};

// Create a promise for the timeout
const timeoutPromise = new Promise((_, timeoutReject) => {
setTimeout(() => timeoutReject(new Error("Fetch timed out")), timeout);
// Start the fetch attempt with the maximum number of retries
attemptFetch(maxRetries);
});
});

// Race the fetch promise against the timeout promise
Promise.race([fetchPromise, timeoutPromise])
.then(resolve)
.catch((error) => {
if (retriesLeft > 0) {
// Retry the fetch if there are retries left
console.warn(`Retrying fetch for model: ${memberModel.id}, retries left: ${retriesLeft}, error: ${error}`);
attemptFetch(retriesLeft - 1);
} else {
// Reject the promise if all retries are exhausted
console.error(`Failed to fetch model: ${memberModel.id} after ${maxRetries} retries, error: ${error}`);
reject(error);
}
});
};

// Start the fetch attempt with the maximum number of retries
attemptFetch(maxRetries);
});
});

// Once all fetch promises are resolved, fetch the next batch
Promise.allSettled(fetchPromises).then((results) => {
const errors = results.filter(result => result.status === "rejected");
if (errors.length > 0) {
console.error("Error fetching member models:", errors);
}
// Fetch the next batch of models
this.fetchMemberModels.call(collection, models, index + batchSize, batchSize, timeout, maxRetries);
}).catch((error) => {
console.error("Error fetching member models:", error);
});
},
// Once all fetch promises are resolved, fetch the next batch
Promise.allSettled(fetchPromises).then((results) => {
const errors = results.filter(result => result.status === "rejected");
if (errors.length > 0) {
console.error("Error fetching member models:", errors);
}
// Fetch the next batch of models
this.fetchMemberModels.call(collection, models, index + batchSize, batchSize, timeout, maxRetries);
}).catch((error) => {
console.error("Error fetching member models:", error);
});
},

/**
* Overload fetch calls for a DataPackage
Expand Down Expand Up @@ -685,7 +686,7 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5",
//Don't fetch each member model if the fetchModels property on this Collection is set to false
if( this.fetchModels !== false ){
// Start fetching member models
this.fetchMemberModels.call(this, models, 0, MetacatUI.appModel.get("batchModeValue"));
this.fetchMemberModels.call(this, models, 0, MetacatUI.appModel.get("batchSizeFetch"));
}

} catch (error) {
Expand Down
23 changes: 21 additions & 2 deletions src/js/models/AppModel.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1998,7 +1998,7 @@ define(['jquery', 'underscore', 'backbone'],
packageFormat: 'application%2Fbagit-1.0',

/**
* Whether to batch requests to the DataONE API. This is an experimental feature
* Whether to batch fetch requests from the DataONE API. This is an experimental feature
* and should be used with caution. If set to a number greater than 0, MetacatUI will
* batch requests to the DataONE API and send them in groups of this size. This can
* improve performance when making many requests to the DataONE API, but can also
Expand All @@ -2011,8 +2011,27 @@ define(['jquery', 'underscore', 'backbone'],
* @type {number}
* @default 0
* @example 20
* @since 0.0.0
*/
batchModeValue: 0,
batchSizeFetch: 0,

/**
* Whether to batch uploads to the DataONE API. This is an experimental feature
* and should be used with caution. If set to a number greater than 0, MetacatUI will
* batch uploads to the DataONE API and send them in groups of this size. This can
* improve performance when uploading many files to the DataONE API, but can also
* cause issues if the requests are too large or if the DataONE API is not able to
* handle the batched requests.
*
* Currently, this feature is only used in the DataPackageModel when uploading files
* to the DataONE API.
*
* @type {number}
* @default 0
* @example 20
* @since 0.0.0
*/
batchSizeUpload: 0
}, MetacatUI.AppConfig),

defaultView: "data",
Expand Down
Loading

0 comments on commit c270e18

Please sign in to comment.