From 94cf7137df2fcd4ab4851c6452f2c3d8ffd0e3c1 Mon Sep 17 00:00:00 2001 From: Isla Li Date: Mon, 8 Apr 2024 16:37:36 -0700 Subject: [PATCH 1/2] issue-16: fix download bug in data product delivery --- onc/+onc/DataProductFile.m | 56 ++++++++++++++++++-------------- onc/+util/extractFileExtension.m | 13 ++++++++ onc/+util/save_as_file.m | 39 ++++++++++++++-------- 3 files changed, 69 insertions(+), 39 deletions(-) create mode 100644 onc/+util/extractFileExtension.m diff --git a/onc/+onc/DataProductFile.m b/onc/+onc/DataProductFile.m index bc918e5..1966d1f 100644 --- a/onc/+onc/DataProductFile.m +++ b/onc/+onc/DataProductFile.m @@ -57,22 +57,19 @@ % * maxRetries: (int) As provided by the Onc class % % Returns: (integer) The final response's HTTP status code + log = onc.DPLogger(); - this.status = 202; - saveResult = 0; + request = matlab.net.http.RequestMessage; + uri = matlab.net.URI(this.baseUrl); + uri.Query = matlab.net.QueryParameter(this.filters); + fullUrl = char(uri); + options = matlab.net.http.HTTPOptions('ConnectTimeout', timeout); while this.status == 202 - % prepare HTTP request - request = matlab.net.http.RequestMessage; - uri = matlab.net.URI(this.baseUrl); - uri.Query = matlab.net.QueryParameter(this.filters); - fullUrl = char(uri); - options = matlab.net.http.HTTPOptions('ConnectTimeout', timeout, 'ConvertResponse', false); - % run and time request if this.showInfo, log.printLine(sprintf('Requesting URL:\n %s', fullUrl)); end tic - response = send(request, uri, options); + response = request.send(uri,options); duration = toc; this.downloadUrl = fullUrl; @@ -89,6 +86,7 @@ s = this.status; if s == 200 % File downloaded, get filename from header and save + this.downloaded = true; filename = this.extractNameFromHeader(response); this.fileName = filename; @@ -98,34 +96,42 @@ if length(lengthData) == 1 this.fileSize = str2double(lengthData.Value); else - this.fileSize = strlength(response.Body.Data); + ext = util.extractFileExtension(filename); + if strcmp(ext, 'xml') + this.fileSize = length(xmlwrite(response.Body.Data)); + else + this.fileSize = strlength(response.Body.Data); + end + end + try + saveResult = util.save_as_file(response.Body.Data, outPath, filename, 'overwrite', overwrite); + catch ME + if strcmp(ME.identifier, 'onc:FileExistsError') + log.printLine(sprintf('Skipping "%s": File already exists\n', this.fileName)); + this.status = 777; + saveResult = -2; + this.downloaded = false; + else + rethrow(ME); + end end - %save_as_file doesn't work properly! Use urlwrite instead - [~, saveResult] = urlwrite(uri.EncodedURI,fullfile(outPath, filename)); - %saveResult = util.save_as_file(response, outPath, filename, overwrite); this.downloadingTime = round(duration, 3); % log status if saveResult == 0 - log.printLine(sprintf('Downloaded "%s"', this.fileName)); - elseif saveResult == -2 - log.printLine(sprintf('Skipping "%s": File already exists', this.fileName)); - this.status = 777; + log.printLine(sprintf('Downloaded "%s"\n', this.fileName)); end elseif s == 202 % Still processing, wait and retry - log.printResponse(jsondecode(response.Body.Data)); + log.printResponse(response.Body.Data); pause(pollPeriod); elseif s == 204 % No data found - log.printLine('No data found.'); + log.printLine('No data found.\n'); elseif s == 400 % API Error util.print_error(response, fullUrl); - err = struct( ... - 'message', sprintf('The request failed with HTTP status %d\n', this.status), ... - 'identifier', 'DataProductFile:HTTP400'); - error(err); + throw(util.prepare_exception(s, double(response.Body.Data.errors.errorCode))); elseif s == 404 % Index too high, no more files to download else @@ -139,7 +145,7 @@ endStatus = this.status; end - function filename = extractNameFromHeader(this, response) + function filename = extractNameFromHeader(~,response) %% Return the file name extracted from the HTTP response % % * response (object) The successful (200) httr response obtained from a download request diff --git a/onc/+util/extractFileExtension.m b/onc/+util/extractFileExtension.m new file mode 100644 index 0000000..a86ff8a --- /dev/null +++ b/onc/+util/extractFileExtension.m @@ -0,0 +1,13 @@ +function ext = extractFileExtension(filename) + % get extension from filename (string or char array) + % this function is called by save_as_file.m to decide which download method to use + + filename = char(filename); + possibleExtensionStartIndex = strfind(filename, '.'); + if ~isempty(possibleExtensionStartIndex) + extensionStartIndex = possibleExtensionStartIndex(end); + ext = filename(extensionStartIndex + 1:end); + else + ext = ''; + end +end \ No newline at end of file diff --git a/onc/+util/save_as_file.m b/onc/+util/save_as_file.m index 456c730..801f76d 100644 --- a/onc/+util/save_as_file.m +++ b/onc/+util/save_as_file.m @@ -6,7 +6,7 @@ % @param fileName Name of the file to save % @param overwrite If true will overwrite files with the same name % @return (numeric) Result code from {0: done, -1: error, -2: fileExists} -function endCode = save_as_file(response, filePath, fileName, varargin) +function endCode = save_as_file(dataToWrite, filePath, fileName, varargin) [overwrite] = util.param(varargin, 'overwrite', false); fullPath = fileName; @@ -25,27 +25,38 @@ try matlabVersion = version('-release'); year = str2double(matlabVersion(1:end-1)); - if year >= 2021 - f = fopen(fullPath, 'w','n','ISO-8859-1'); - else - f = fopen(fullPath, 'w','n'); - end - - if f ~= -1 - fwrite(f, response.Body.Data); + ext = util.extractFileExtension(fileName); + + % if result is an image file or .xml file, use other save methods instead of fwrite. + if strcmp(ext, 'png') || strcmp(ext, 'jpg') + imwrite(dataToWrite, fullPath); + elseif strcmp(ext, 'xml') + xmlwrite(fullPath, dataToWrite); else - endCode = -1; - return; + % open output file + if year >= 2021 + f = fopen(fullPath, 'w','n','ISO-8859-1'); + else + f = fopen(fullPath, 'w','n'); + end + + % write result if open file successfully + if f ~= -1 + fwrite(f, char(dataToWrite)); + else + endCode = -1; + return; + end + fclose(f); end - fclose(f); catch ex disp(ex); endCode = -1; return; end else - endCode = -2; - return; + % if file exists and overwrite is false, raise exception + throw(MException('onc:FileExistsError', 'Data product file exists in destination but overwrite is set to false')); end endCode = 0; From f2eddac8e64140df35ae224abe52bd302268f173 Mon Sep 17 00:00:00 2001 From: Isla Li Date: Thu, 18 Apr 2024 15:53:29 -0700 Subject: [PATCH 2/2] issue-16: Replaced util function with built-in function fileparts --- onc/+onc/DataProductFile.m | 10 ++++------ onc/+util/extractFileExtension.m | 13 ------------- onc/+util/save_as_file.m | 8 +++----- 3 files changed, 7 insertions(+), 24 deletions(-) delete mode 100644 onc/+util/extractFileExtension.m diff --git a/onc/+onc/DataProductFile.m b/onc/+onc/DataProductFile.m index 4d0ee2d..01ffc33 100644 --- a/onc/+onc/DataProductFile.m +++ b/onc/+onc/DataProductFile.m @@ -95,15 +95,13 @@ % Obtain filesize from headers, or fallback to body string length lengthData = response.getFields('Content-Length'); + [~, ~, ext] = fileparts(filename); if length(lengthData) == 1 this.fileSize = str2double(lengthData.Value); + elseif strcmp(ext, '.xml') + this.fileSize = length(xmlwrite(response.Body.Data)); else - ext = util.extractFileExtension(filename); - if strcmp(ext, 'xml') - this.fileSize = length(xmlwrite(response.Body.Data)); - else - this.fileSize = strlength(response.Body.Data); - end + this.fileSize = strlength(response.Body.Data); end try saveResult = util.save_as_file(response.Body.Data, outPath, filename, 'overwrite', overwrite); diff --git a/onc/+util/extractFileExtension.m b/onc/+util/extractFileExtension.m deleted file mode 100644 index a86ff8a..0000000 --- a/onc/+util/extractFileExtension.m +++ /dev/null @@ -1,13 +0,0 @@ -function ext = extractFileExtension(filename) - % get extension from filename (string or char array) - % this function is called by save_as_file.m to decide which download method to use - - filename = char(filename); - possibleExtensionStartIndex = strfind(filename, '.'); - if ~isempty(possibleExtensionStartIndex) - extensionStartIndex = possibleExtensionStartIndex(end); - ext = filename(extensionStartIndex + 1:end); - else - ext = ''; - end -end \ No newline at end of file diff --git a/onc/+util/save_as_file.m b/onc/+util/save_as_file.m index 0b25928..94fe773 100644 --- a/onc/+util/save_as_file.m +++ b/onc/+util/save_as_file.m @@ -25,13 +25,11 @@ try matlabVersion = version('-release'); year = str2double(matlabVersion(1:end-1)); - - ext = util.extractFileExtension(fileName); - + [~, ~, ext] = fileparts(fileName); % if result is an image file or .xml file, use other save methods instead of fwrite. - if strcmp(ext, 'png') || strcmp(ext, 'jpg') + if strcmp(ext, '.png') || strcmp(ext, '.jpg') imwrite(dataToWrite, fullPath); - elseif strcmp(ext, 'xml') + elseif strcmp(ext, '.xml') xmlwrite(fullPath, dataToWrite); else % open output file