From 44bd1a88bd8b42db50bea56fd12125b6cafe23bb Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Mon, 11 Mar 2024 12:06:47 -0700
Subject: [PATCH 01/35] issue5: add support to parameter waitComplete in
rundataproduct
---
onc/+onc/OncDelivery.m | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/onc/+onc/OncDelivery.m b/onc/+onc/OncDelivery.m
index 8cc98ca..09c779d 100644
--- a/onc/+onc/OncDelivery.m
+++ b/onc/+onc/OncDelivery.m
@@ -113,23 +113,26 @@
% run timed run request
tic
- status = 202;
- while status == 202
+ flag = 'queued';
+ while ~strcmp(flag,'complete') && ~strcmp(flag,'cancelled')
[response, info] = this.doRequest(url, filters);
status = info.status;
r.requestCount = r.requestCount + 1;
% guard against failed request
if util.is_failed_response(response, status)
- r = response;
throw(util.prepare_exception(status));
end
-
% repeat only if waitComplete
if waitComplete
log.printResponse(response);
- if status == 202, pause(this.pollPeriod); end
+ if status == 202
+ pause(this.pollPeriod);
+ end
+ else
+ break;
end
+ flag = response.status;
end
duration = toc;
fprintf('\n')
From 1731231876c4a2d5df1a96c2f832495bc07b96c2 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Mon, 11 Mar 2024 12:08:34 -0700
Subject: [PATCH 02/35] issue5: set waitComplete default value to true
---
onc/+onc/OncDelivery.m | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/onc/+onc/OncDelivery.m b/onc/+onc/OncDelivery.m
index 09c779d..20ba84f 100644
--- a/onc/+onc/OncDelivery.m
+++ b/onc/+onc/OncDelivery.m
@@ -104,7 +104,7 @@
%
% Documentation: https://wiki.oceannetworks.ca/display/CLIBS/Data+product+download+methods
- if ~exist('waitComplete','var'), waitComplete = false; end
+ if ~exist('waitComplete','var'), waitComplete = true; end
url = sprintf('%sapi/dataProductDelivery', this.baseUrl);
log = onc.DPLogger();
From 6a5ff03c79ac3ff504e860ac15704dfd6c5ed888 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Wed, 13 Mar 2024 15:43:19 -0700
Subject: [PATCH 03/35] issue-8: fixed never reached branch and update error
response exceptions
---
onc/+util/do_request.m | 25 ++++++---------
onc/+util/prepare_exception.m | 10 +++---
onc/+util/print_error.m | 60 ++++++++++++++++++++++-------------
3 files changed, 53 insertions(+), 42 deletions(-)
diff --git a/onc/+util/do_request.m b/onc/+util/do_request.m
index 0b667d7..0b57dc9 100644
--- a/onc/+util/do_request.m
+++ b/onc/+util/do_request.m
@@ -25,7 +25,9 @@
% run and time request
if showInfo, fprintf('\nRequesting URL:\n %s\n', fullUrl); end
tic
- response = send(request, uri, options);
+ %response = send(request, uri, options);
+ response = request.send(uri,options);
+
duration = toc;
% print duration
@@ -47,22 +49,13 @@
case 202
% Accepted, no need to print error, handle manually
result = response.Body.Data;
- case 400
- % Bad request
- result = response.Body.Data;
- util.print_error(response, fullUrl);
- case 401
- % Unauthorized
- fprintf('\nERROR 401: Unauthorized. Please verify your user token.\n')
- result = response.Body.Data;
- case 503
- % Down for maintenance
- fprintf('\nERROR 503: Service unavailable.\nWe could be down for maintenance; ');
- fprintf('visit https://data.oceannetworks.ca for more information.\n')
- result = struct('errors', [struct('errorCode', 503, 'message', 'Service Unavailable')]);
otherwise
- result = response.Body.Data;
- fprintf('\nERROR: The request failed with HTTP error %d\n', status)
+ util.print_error(response, fullUrl);
+ if status == 400 || status == 401
+ throw(util.prepare_exception(status,double(response.Body.Data.errors.errorCode)));
+ else
+ throw(util.prepare_exception(status));
+ end
end
% prepare info.size only if the response is a file, otherwise 0
diff --git a/onc/+util/prepare_exception.m b/onc/+util/prepare_exception.m
index 119698b..3dc9a84 100644
--- a/onc/+util/prepare_exception.m
+++ b/onc/+util/prepare_exception.m
@@ -1,15 +1,17 @@
-function ex = prepare_exception(status)
+function ex = prepare_exception(status, errorCode)
%% Prepares a throwable exception object for the response
%
% * response: (struct) Response as returned by do_request()
- % - status: @TODO
+ % - status: http response code
+ % - errorCode: optional
+ % specific error code returned in field errors (only for 400 and 401)
%
% Returns: (MException) @TODO
switch status
case 400
- ex = MException('onc:http400', 'HTTP 400: Invalid request parameters');
+ ex = MException(sprintf('onc:http400:error%d', errorCode), 'HTTP 400: Invalid request parameters');
case 401
- ex = MException('onc:http401', 'HTTP 401: Invalid token');
+ ex = MException(sprintf('onc:http401:error%d', errorCode), 'HTTP 401: Invalid token');
case 404
ex = MException('onc:http404', 'HTTP 404: Not found');
case 410
diff --git a/onc/+util/print_error.m b/onc/+util/print_error.m
index 039cf14..e1507f1 100644
--- a/onc/+util/print_error.m
+++ b/onc/+util/print_error.m
@@ -4,32 +4,48 @@ function print_error(response, url)
status = double(response.StatusCode);
if status == 400
+ % Bad request
fprintf('\nERROR 400 - Bad Request:\n %s\n\n', url)
- payload = response.Body.Data;
+ print_error_message(response.Body.Data);
- % Make sure the payload was parsed automatically
- if ~isstruct(payload)
- payload = jsondecode(payload);
- end
+ elseif status == 401
+ % Unauthorized
+ fprintf('\nERROR 401: Unauthorized. Please verify your user token.\n')
+ print_error_message(response.Body.Data)
- if isfield(payload, 'errors')
- for i = 1 : numel(payload.errors)
- e = payload.errors(i);
- msg = e.errorMessage;
- parameters = e.parameter;
- fprintf(' Parameter "%s" -> %s\n', string(parameters), msg)
- end
- fprintf('\n');
+ elseif status == 404
+ %Not Found
+ fprintf('\nERROR 404: Not Found \n')
- elseif status == 401
- fprintf('\nERROR 401: Unauthorized - %s\n', url)
- fprintf('Please check that your Web Services API token is valid. Find your token in your registered profile at https://data.oceannetworks.ca.\n')
+ elseif status == 500
+ % Internal Server Error
+ fprintf('\nERROR 500: Internal Server Error - %s\n', url)
+ fprintf('The API failed to process your request. You might want to retry later in case this is a temporary issue (i.e. Maintenance).\n')
+
+ elseif status == 503
+ % Down for maintenance
+ fprintf('\nERROR 503: Service unavailable.\nWe could be down for maintenance; ');
+ fprintf('visit https://data.oceannetworks.ca for more information.\n')
+
+ else
+ fprintf('\nERROR: The request failed with HTTP error %d\n', status);
+ end
+end
- elseif status == 500
- fprintf('\nERROR 500: Internal Server Error - %s\n', url)
- fprintf('The API failed to process your request. You might want to retry later in case this is a temporary issue (i.e. Maintenance).\n')
- else
- fprintf('\nERROR %d: The request failed.\n', status)
- end
+% helper function
+function print_error_message(payload)
+ % Make sure the payload was parsed automatically
+ if ~isstruct(payload)
+ payload = jsondecode(payload);
end
+
+ %if isfield(payload, 'errors')
+ for i = 1 : numel(payload.errors)
+ e = payload.errors(i);
+ msg = e.errorMessage;
+ parameters = e.parameter;
+ fprintf(' Parameter "%s" -> %s\n', string(parameters), msg)
+ end
+ fprintf('\n');
end
+
From 162ad59a3b2b5d00495aec4f10def32e0f81ace6 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Fri, 22 Mar 2024 14:39:41 -0700
Subject: [PATCH 04/35] issue7: rewrite test suites
---
onc/tests/suites/Test01_Locations.m | 123 +++----
onc/tests/suites/Test02_Deployments.m | 102 +++---
onc/tests/suites/Test03_DeviceCategories.m | 83 ++---
onc/tests/suites/Test04_Devices.m | 108 +++---
onc/tests/suites/Test05_Properties.m | 82 ++---
.../suites/Test06_DataProductDiscovery.m | 96 +++---
onc/tests/suites/Test07_DataProductDelivery.m | 315 +++++++-----------
onc/tests/suites/Test08_RealTime.m | 259 ++++++++++----
onc/tests/suites/Test09_ArchiveFiles.m | 228 ++++++++-----
9 files changed, 704 insertions(+), 692 deletions(-)
diff --git a/onc/tests/suites/Test01_Locations.m b/onc/tests/suites/Test01_Locations.m
index 03443ba..29cad1b 100644
--- a/onc/tests/suites/Test01_Locations.m
+++ b/onc/tests/suites/Test01_Locations.m
@@ -1,82 +1,93 @@
% OncTest Locations test suite
-% Contains test cases for the locations discovery service.
+% Contains test cases for the locations / locationtrees discovery service.
-classdef Test01_Locations < TestDiscovery
+classdef Test01_Locations < matlab.unittest.TestCase
+ properties
+ onc
+ end
- %% Public Methods
- methods
-
- function obj = Test01_Locations()
- % Constructor
- obj@TestDiscovery();
- obj.expectedFields('getLocations') = ["deployments", "locationName", "depth", "bbox", "description", "hasDeviceData", "lon", "locationCode", "hasPropertyData", "lat", "dataSearchURL"];
+ methods (TestClassSetup)
+ function classSetup(this)
+ config = globals();
+ this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
-
+
%% Test methods
methods (Test)
- %% General test cases
-
- function testGetAllLocations(this)
- % Make an unfiltered locations request
- % verifies: expected fields, minimum rows
- locations = this.o.getLocations();
- verify_fields(this, locations, this.expectedFields('getLocations'));
- this.verify_min_length(locations, 500);
+ %% getLocations test cases
+ function testInvalidTimeRangeGreaterStartTime(this)
+ filters = {'locationCode', 'FGPD','dateFrom', '2020-01-01', 'dateTo', '2019-01-01'};
+ verifyError(this, @() this.onc.getLocations(filters), 'onc:http400:error23');
end
-
- function testISODateRange(this)
- % Test a date range with format ISO8601
- filters = {'dateFrom', '2014-02-24T00:00:01.000Z', 'dateTo', '2014-03-24T00:00:01.000Z'};
- locations = this.testSingleFilter('getLocations', filters, 100, NaN);
+
+ function testInvalidTimeRangeFutureStartTime(this)
+ filters = {'locationCode', 'FGPD', 'dateFrom', '2050-01-01'};
+ verifyError(this, @() this.onc.getLocations(filters), 'onc:http400:error25');
end
- function testFilterIncludeChildren(this)
- % Test filter includeChildren, verify children were obtained
- filters = {'includeChildren', 'true', 'locationCode', 'SAAN'};
- locations = this.testSingleFilter('getLocations', filters, 30, NaN);
+ function testInvalidParamValue(this)
+ filters = {'locationCode', 'XYZ123'};
+ verifyError(this, @() this.onc.getLocations(filters), 'onc:http400:error127');
end
- function testWrongLocationCode(this)
- % try an invalid locationCode, verify error structure
- locations = this.o.getLocations({'locationCode', 'CQS34543BG'});
- verify_error_response(this, locations);
+ function testInvalidParamName(this)
+ filters = {'fakeParamName', 'FGPD'};
+ verifyError(this, @() this.onc.getLocations(filters), 'onc:http400:error129');
end
- function testNoLocationsFound(this)
- % try a locations query with 0 results, verify result is an empty 0x0 matrix
- locations = this.o.getLocations({'locationCode', 'SAAN', 'dateTo', '1995-03-24T00:00:01.000Z'});
- verifyEqual(this, size(locations), [0 0]);
-
+ function testNoData(this)
+ filters = {'locationCode', 'FGPD', 'dateTo', '1900-01-01'};
+ verifyError(this, @() this.onc.getLocations(filters), 'onc:http404');
end
- %% Single filter test cases
- % These tests invoke getLocations with a single filter, for every supported filter
- % Verifications according to tests documentation at: https://internal.oceannetworks.ca/x/xYI2Ag
- function testFilterLocationCode(this)
- locations = this.testSingleFilter('getLocations', {'locationCode', 'CQSBG'}, 1, 1);
- verifyEqual(this, locations(1).locationName, 'Bubbly Gulch');
+ function testValidParams(this)
+ filters = {'locationCode', 'FGPD', 'dateFrom', '2005-09-17', 'dateTo', '2020-09-17'};
+ locations = this.onc.getLocations(filters);
+ verifyTrue(this, length(locations) >= 1);
+ expectedLocationsFields = ["deployments", "locationName", "depth", "bbox", ...
+ "description", "hasDeviceData", "lon", "locationCode",...
+ "hasPropertyData", "lat", "dataSearchURL"];
+ verify_fields(this, locations(1), expectedLocationsFields);
+ expectedBboxFields = ["maxDepth", "maxLat", "maxLon", "minDepth", "minLat", "minLon"];
+ verify_fields(this, locations(1).bbox, expectedBboxFields);
end
- function testFilterLocationName(this)
- locations = this.testSingleFilter('getLocations', {'locationName', 'Bubbly Gulch'}, 1, 1);
- verifyEqual(this, locations(1).locationCode, 'CQSBG');
- end
+ %% Location tree test cases
- function testFilterDeviceCategoryCode(this)
- locations = this.testSingleFilter('getLocations', {'deviceCategoryCode', 'CTD'}, 50, NaN);
+ function testTreeInvalidTimeRangeGreaterStartTime(this)
+ filters = {'locationCode', 'ARCT', 'dateFrom', '2020-01-01', 'dateTo', '2019-01-01'};
+ verifyError(this, @() this.onc.getLocationHierarchy(filters), 'onc:http400:error23');
end
-
- function testFilterDeviceCode(this)
- locations = this.testSingleFilter('getLocations', {'deviceCode', 'NORTEKADCP9917'}, 1, NaN);
+
+ function testTreeInvalidTimeRangeFutureStartTime(this)
+ filters = {'locationCode', 'ARCT', 'dateFrom', '2050-01-01'};
+ verifyError(this, @() this.onc.getLocationHierarchy(filters), 'onc:http400:error25');
end
-
- function testFilterPropertyCode(this)
- locations = this.testSingleFilter('getLocations', {'propertyCode', 'co2concentration'}, 1, NaN);
+
+ function testTreeInvalidParamValue(this)
+ filters = {'locationCode', 'XYZ123'};
+ verifyError(this, @() this.onc.getLocationHierarchy(filters), 'onc:http400:error127');
+ end
+
+ function testTreeInvalidParamName(this)
+ filters = {'fakeParamName', 'ARCT'};
+ verifyError(this, @() this.onc.getLocationHierarchy(filters), 'onc:http400:error129');
end
- function testFilterDataProductCode(this)
- locations = this.testSingleFilter('getLocations', {'dataProductCode', 'MP4V'}, 20, NaN);
+ function testTreeNoData(this)
+ filters = {'locationCode', 'ARCT', 'dateTo', '1900-01-01'};
+ verifyError(this, @() this.onc.getLocationHierarchy(filters), 'onc:http404');
+ end
+
+ function testTreeValidParams(this)
+ filters = {'locationCode', 'ARCT', 'deviceCategoryCode', 'VIDEOCAM'};
+ locations = this.onc.getLocationHierarchy(filters);
+ verifyTrue(this, length(locations) >= 1);
+ expectedLocationsFields = ["locationName","children","description","hasDeviceData",...
+ "locationCode","hasPropertyData"];
+ verify_fields(this, locations(1), expectedLocationsFields);
+ verifyTrue(this, length(locations(1).children) >= 1);
end
end
end
\ No newline at end of file
diff --git a/onc/tests/suites/Test02_Deployments.m b/onc/tests/suites/Test02_Deployments.m
index 05f1461..edaa2fd 100644
--- a/onc/tests/suites/Test02_Deployments.m
+++ b/onc/tests/suites/Test02_Deployments.m
@@ -1,66 +1,58 @@
% Deployments test suite
% Contains test cases for the deployments discovery service.
-classdef Test02_Deployments < TestDiscovery
-
- %% Public Methods
- methods
-
- function obj = Test02_Deployments()
- % Constructor
- obj@TestDiscovery();
- obj.expectedFields('getDeployments') = ["begin", "depth", "deviceCode", "end", "hasDeviceData", "heading", "lat", "locationCode", "lon", "pitch", "roll"];
+classdef Test02_Deployments < matlab.unittest.TestCase
+ properties
+ onc
+ end
+
+ methods (TestClassSetup)
+ function classSetup(this)
+ config = globals();
+ this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
%% Test methods
methods (Test)
%% General test cases
-
- function testGetAllDeployments(this)
- % Make an unfiltered deployments request
- % verifies: expected fields, minimum rows
- deployments = this.o.getDeployments();
- verify_fields(this, deployments, this.expectedFields('getDeployments'));
- this.verify_min_length(deployments, 500);
- end
-
- function testISODateRange(this)
- % Test a date range with format ISO8601
- filters = {'dateFrom', '2014-02-24T00:00:01.000Z', 'dateTo', '2014-03-24T00:00:01.000Z'};
- deployments = this.testSingleFilter('getDeployments', filters, 100, NaN);
- end
-
- function testWrongLocationCode(this)
- % try an invalid locationCode, verify error structure
- deployments = this.o.getDeployments({'locationCode', 'CQS34543BG'});
- verify_error_response(this, deployments);
- end
-
- function testNoDeploymentsFound(this)
- % try a deployments query with 0 results, verify result is an empty 0x0 matrix
- deployments = this.o.getDeployments({'locationCode', 'SAAN', 'dateTo', '1995-03-24T00:00:01.000Z'});
- verifyEqual(this, size(deployments), [0 0]);
-
- end
- %% Single filter test cases
- % These tests invoke getdeployments with a single filter, for every supported filter
- % Verifications according to tests documentation at: https://internal.oceannetworks.ca/x/xYI2Ag
-
- function testFilterLocationCode(this)
- deployments = this.testSingleFilter('getDeployments', {'locationCode', 'CQSBG'}, 2, NaN);
- end
-
- function testFilterDeviceCategoryCode(this)
- deployments = this.testSingleFilter('getDeployments', {'deviceCategoryCode', 'CTD'}, 50, NaN);
- end
-
- function testFilterDeviceCode(this)
- deployments = this.testSingleFilter('getDeployments', {'deviceCode', 'NORTEKADCP9917'}, 1, NaN);
- end
-
- function testFilterPropertyCode(this)
- deployments = this.testSingleFilter('getDeployments', {'propertyCode', 'co2concentration'}, 1, NaN);
- end
+ function testInvalidTimeRangeGreaterStartTime(this)
+ filters = {'locationCode', 'BACAX', 'deviceCategoryCode', 'CTD', 'dateFrom', '2020-01-01', 'dateTo', '2019-01-01'};
+ verifyError(this, @() this.onc.getDeployments(filters), 'onc:http400:error23');
+ end
+
+ function testInvalidTimeRangeFutureStartTime(this)
+ filters = {'locationCode', 'BACAX', 'deviceCategoryCode', 'CTD', 'dateFrom', '2050-01-01'};
+ verifyError(this, @() this.onc.getDeployments(filters), 'onc:http400:error25');
+ end
+
+ function testInvalidParamValue(this)
+ filters = {'locationCode', 'XYZ123', 'deviceCategoryCode', 'CTD'};
+ verifyError(this, @() this.onc.getDeployments(filters), 'onc:http400:error127');
+ end
+
+ function testInvalidParamName(this)
+ filters = {'fakeParamName', 'BACAX', 'deviceCategoryCode', 'CTD'};
+ verifyError(this, @() this.onc.getDeployments(filters), 'onc:http400:error129');
+ end
+
+ function testNoData(this)
+ filters = {'locationCode', 'BACAX', 'deviceCategoryCode', 'CTD', 'dateTo', '1900-01-01'};
+ verifyError(this, @() this.onc.getDeployments(filters), 'onc:http400:error127');
+ end
+
+ function testValidParams(this)
+ filters = {'locationCode', 'BACAX', 'deviceCategoryCode', 'CTD', 'dateFrom', '2005-09-17', 'dateTo', '2015-09-17T13:00:00.000Z'};
+ deployments = this.onc.getDeployments(filters);
+ verifyTrue(this, length(deployments) >= 1);
+ expectedDeploymentFields = ["begin", "citation", "depth", "deviceCategoryCode", ...
+ "deviceCode", "end", "hasDeviceData", "heading", ...
+ "lat", "locationCode", "lon", "pitch", "roll"];
+ verify_fields(this, deployments(1), expectedDeploymentFields);
+ expectedCitationFields = ["citation", "doi", "landingPageUrl", "queryPid"];
+ verify_fields(this, deployments(1).citation, expectedCitationFields);
+ end
+
+
end
end
diff --git a/onc/tests/suites/Test03_DeviceCategories.m b/onc/tests/suites/Test03_DeviceCategories.m
index f2a4882..9d4bd56 100644
--- a/onc/tests/suites/Test03_DeviceCategories.m
+++ b/onc/tests/suites/Test03_DeviceCategories.m
@@ -1,67 +1,52 @@
% DeviceCategories test suite
% Contains test cases for the deviceCategories discovery service.
-classdef Test03_DeviceCategories < TestDiscovery
-
- %% Public Methods
- methods
-
- function obj = Test03_DeviceCategories()
- % Constructor
- obj@TestDiscovery();
- obj.expectedFields('getDeviceCategories') = ["description", "deviceCategoryCode", "deviceCategoryName", "hasDeviceData", "longDescription"];
+classdef Test03_DeviceCategories < matlab.unittest.TestCase
+ properties
+ onc
+ end
+
+ methods (TestClassSetup)
+ function classSetup(this)
+ config = globals();
+ this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
+
%% Test methods
methods (Test)
%% General test cases
- function testGetAllDeviceCategories(this)
+ function testInvalidParamValue(this)
% Make an unfiltered deviceCategories request
% verifies: expected fields, minimum rows
- deviceCategories = this.o.getDeviceCategories();
- verify_fields(this, deviceCategories, this.expectedFields('getDeviceCategories'));
- this.verify_min_length(deviceCategories, 100);
+ filters = {'deviceCategoryCode','XYZ123'};
+ verifyError(this, @() this.onc.getDeviceCategories(filters), 'onc:http400:error127');
end
- function testWrongDeviceCategoryCode(this)
+ function testInvalidParamName(this)
% try an invalid locationCode, verify error structure
- deviceCategories = this.o.getDeviceCategories({'deviceCategoryCode', 'XYZ321'});
- verify_error_response(this, deviceCategories);
- end
-
- function testNoDeviceCategoriesFound(this)
- % try a deviceCategories query with 0 results, verify result is an empty 0x0 matrix
- deviceCategories = this.o.getDeviceCategories({'locationCode', 'SAAN', 'propertyCode', 'co2concentration'});
- verifyEqual(this, size(deviceCategories), [0 0]);
- end
-
- %% Single filter test cases
- % These tests invoke getDeviceCategories with a single filter, for every supported filter
- % Verifications according to tests documentation at: https://internal.oceannetworks.ca/x/xYI2Ag
-
- function testFilterDeviceCategoryCode(this)
- deviceCategories = this.testSingleFilter('getDeviceCategories', {'deviceCategoryCode', 'ADCP1200KHZ'}, 1, 1);
- verifyEqual(this, deviceCategories(1).deviceCategoryCode, 'ADCP1200KHZ');
- end
-
- function testFilterDeviceCategoryName(this)
- deviceCategories = this.testSingleFilter('getDeviceCategories', {'deviceCategoryName', 'Current Profiler 1200'}, 1, 1);
- verifyEqual(this, deviceCategories(1).deviceCategoryCode, 'ADCP1200KHZ');
- end
-
- function testFilterDescription(this)
- deviceCategories = this.testSingleFilter('getDeviceCategories', {'description', '3D Camera'}, 1, 1);
- verifyEqual(this, deviceCategories(1).deviceCategoryCode, 'CAMERA_3D');
- end
-
- function testFilterLocationCode(this)
- deviceCategories = this.testSingleFilter('getDeviceCategories', {'locationCode', 'CQSBG'}, 1, NaN);
- end
-
- function testFilterPropertyCode(this)
- deviceCategories = this.testSingleFilter('getDeviceCategories', {'propertyCode', 'co2concentration'}, 1, NaN);
+ filters = {'fakeParamName', 'CTD'};
+ verifyError(this, @() this.onc.getDeviceCategories(filters), 'onc:http400:error129');
+ end
+
+ function testNoData(this)
+ % try a deviceCategories query with 0 results, verify result message
+ filters = {'deviceCategoryCode', 'CTD', 'deviceCategoryName', 'Conductivity','description','TemperatureXXX'};
+ verifyError(this, @() this.onc.getDeviceCategories(filters), 'onc:http404');
+ end
+
+ function testValidParams(this)
+ filters = {'deviceCategoryCode', 'CTD', 'deviceCategoryName', 'Conductivity', 'description', 'Temperature'};
+
+ deviceCategories = this.onc.getDeviceCategories(filters);
+ verifyTrue(this, length(deviceCategories) >= 1);
+ expectedDeviceCategoriesFields = ["cvTerm", "description", "deviceCategoryCode", ...
+ "deviceCategoryName", "hasDeviceData", "longDescription"];
+ verify_fields(this, deviceCategories(1), expectedDeviceCategoriesFields);
+ expectedCvTermFields = ["uri", "vocabulary"];
+ verify_fields(this, deviceCategories(1).cvTerm.deviceCategory, expectedCvTermFields);
end
end
end
diff --git a/onc/tests/suites/Test04_Devices.m b/onc/tests/suites/Test04_Devices.m
index f0f4d4b..66b3bbb 100644
--- a/onc/tests/suites/Test04_Devices.m
+++ b/onc/tests/suites/Test04_Devices.m
@@ -1,76 +1,58 @@
% Devices test suite
% Contains test cases for the devices discovery service.
-classdef Test04_Devices < TestDiscovery
+classdef Test04_Devices < matlab.unittest.TestCase
- %% Public Methods
- methods
-
- function obj = Test04_Devices()
- % Constructor
- obj@TestDiscovery();
- obj.expectedFields('getDevices') = ["dataRating", "deviceCode", "deviceId", "deviceLink", "deviceName"];
+ properties
+ onc
+ end
+
+ methods (TestClassSetup)
+ function classSetup(this)
+ config = globals();
+ this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
%% Test methods
methods (Test)
%% General test cases
-
- function testGetAllDevices(this)
- % Make an unfiltered Ddvices request
- % verifies: expected fields, minimum rows
- devices = this.o.getDevices();
- verify_fields(this, devices, this.expectedFields('getDevices'));
- this.verify_min_length(devices, 300);
- end
-
- function testISODateRange(this)
- % Test a date range with format ISO8601
- filters = {'dateFrom', '2014-02-24T00:00:01.000Z', 'dateTo', '2014-03-24T00:00:01.000Z'};
- devices = this.testSingleFilter('getDevices', filters, 100, NaN);
- end
-
- function testWrongDeviceCode(this)
- % try an invalid locationCode, verify error structure
- devices = this.o.getDevices({'deviceCode', 'XYZ321'});
- verify_error_response(this, devices);
- end
-
- function testNoDevicesFound(this)
- % try a Devices query with 0 results, verify result is an empty 0x0 matrix
- devices = this.o.getDevices({'locationCode', 'SAAN', 'dateTo', '1995-03-24T00:00:01.000Z'});
- verifyEqual(this, size(devices), [0 0]);
-
- end
- %% Single filter test cases
- % These tests invoke getDevices with a single filter, for every supported filter
- % Verifications according to tests documentation at: https://internal.oceannetworks.ca/x/xYI2Ag
-
- function testFilterDeviceCode(this)
- devices = this.testSingleFilter('getDevices', {'deviceCode', 'NORTEKADCP9917'}, 1, 1);
- verifyEqual(this, devices(1).deviceCode, 'NORTEKADCP9917');
- end
-
- function testFilterDeviceName(this)
- devices = this.testSingleFilter('getDevices', {'deviceName', 'Nortek Aquadopp HR-Profiler 2965'}, 1, 1);
- verifyEqual(this, devices(1).deviceCode, 'BC_POD1_AD2M');
- end
-
- function testFilterLocationCode(this)
- devices = this.testSingleFilter('getDevices', {'locationCode', 'CQSBG'}, 1, NaN);
- end
-
- function testFilterDeviceCategoryCode(this)
- devices = this.testSingleFilter('getDevices', {'deviceCategoryCode', 'CTD'}, 100, NaN);
- end
-
- function testFilterPropertyCode(this)
- devices = this.testSingleFilter('getDevices', {'propertyCode', 'co2concentration'}, 2, NaN);
- end
-
- function testFilterDataProductCode(this)
- devices = this.testSingleFilter('getDevices', {'dataProductCode', 'MP4V'}, 20, NaN);
+ function testInvalidTimeRangeGreaterStartTime(this)
+ filters = {'deviceCode', 'BPR-Folger-59','dateFrom', '2020-01-01', 'dateTo', '2019-01-01'};
+ verifyError(this, @() this.onc.getDevices(filters), 'onc:http400:error23');
+ end
+
+ function testInvalidTimeRangeFutureStartTime(this)
+ filters = {'deviceCode', 'BPR-Folger-59', 'dateFrom', '2050-01-01'};
+ verifyError(this, @() this.onc.getDevices(filters), 'onc:http400:error25');
+ end
+
+ function testInvalidParamValue(this)
+ filters = {'deviceCode', 'XYZ123'};
+ verifyError(this, @() this.onc.getDevices(filters), 'onc:http400:error127');
+ end
+
+ function testInvalidParamName(this)
+ filters = {'fakeParamName', 'BPR-Folger-59'};
+ verifyError(this, @() this.onc.getDevices(filters), 'onc:http400:error129');
+ end
+
+ function testNoData(this)
+ filters = {'deviceCode', 'BPR-Folger-59', 'dateTo', '1900-01-01'};
+ verifyError(this, @() this.onc.getDevices(filters), 'onc:http404');
+ end
+
+ function testValidParams(this)
+ filters = {'deviceCode', 'BPR-Folger-59', 'dateFrom', '2005-09-17', 'dateTo', '2020-09-17'};
+ devices = this.onc.getDevices(filters);
+ verifyTrue(this, length(devices) >= 1);
+ expectedDevicesFields = ["cvTerm", "dataRating", "deviceCategoryCode", "deviceCode", ...
+ "deviceId", "deviceLink", "deviceName", "hasDeviceData"];
+ verify_fields(this, devices(1), expectedDevicesFields);
+ expectedCvTermFields = ["uri", "vocabulary"];
+ verify_fields(this, devices(1).cvTerm.device, expectedCvTermFields);
+ expectedDataRatingFields = ["dateFrom", "dateTo", "samplePeriod", "sampleSize"];
+ verify_fields(this, devices(1).dataRating, expectedDataRatingFields);
end
end
end
diff --git a/onc/tests/suites/Test05_Properties.m b/onc/tests/suites/Test05_Properties.m
index 6381a87..d4346b7 100644
--- a/onc/tests/suites/Test05_Properties.m
+++ b/onc/tests/suites/Test05_Properties.m
@@ -1,15 +1,15 @@
% Properties test suite
% Contains test cases for the properties discovery service.
-classdef Test05_Properties < TestDiscovery
-
- %% Public Methods
- methods
-
- function obj = Test05_Properties()
- % Constructor
- obj@TestDiscovery();
- obj.expectedFields('getProperties') = ["description", "hasDeviceData", "hasPropertyData", "propertyCode", "propertyName", "uom"];
+classdef Test05_Properties < matlab.unittest.TestCase
+ properties
+ onc
+ end
+
+ methods (TestClassSetup)
+ function classSetup(this)
+ config = globals();
+ this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
@@ -17,55 +17,35 @@
methods (Test)
%% General test cases
- function testGetAllProperties(this)
- % Make an unfiltered properties request
+ function testInvalidParamValue(this)
+ % Make an unfiltered deviceCategories request
% verifies: expected fields, minimum rows
- properties = this.o.getProperties();
- verify_fields(this, properties, this.expectedFields('getProperties'));
- this.verify_min_length(properties, 150);
- end
-
- function testWrongPropertyCode(this)
- % try an invalid propertyCode, verify error structure
- properties = this.o.getProperties({'propertyCode', 'XYZ321'});
- verify_error_response(this, properties);
- end
-
- function testNoPropertiesFound(this)
- % try a properties query with 0 results, verify result is an empty 0x0 matrix
- properties = this.o.getProperties({'locationCode', 'SAAN', 'deviceCategoryCode', 'POWER_SUPPLY'});
- verifyEqual(this, size(properties), [0 0]);
-
- end
- %% Single filter test cases
- % These tests invoke getProperties with a single filter, for every supported filter
- % Verifications according to tests documentation at: https://internal.oceannetworks.ca/x/xYI2Ag
-
- function testFilterPropertyCode(this)
- properties = this.testSingleFilter('getProperties', {'propertyCode', 'absolutehumidity'}, 1, 1);
- verifyEqual(this, properties(1).propertyCode, 'absolutehumidity');
- end
-
- function testFilterPropertyName(this)
- properties = this.testSingleFilter('getProperties', {'propertyName', 'Bender Electrical Resistance'}, 1, 1);
- verifyEqual(this, properties(1).propertyCode, 'benderelectricalresistance');
- end
-
- function testFilterDescription(this)
- properties = this.testSingleFilter('getProperties', {'description', 'Kurtosis Statistical Analysis'}, 1, 1);
- verifyEqual(this, properties(1).propertyCode, 'kurtosisstatisticalanalysis');
+ filters = {'propertyCode','XYZ123'};
+ verifyError(this, @() this.onc.getProperties(filters), 'onc:http400:error127');
end
- function testFilterLocationCode(this)
- properties = this.testSingleFilter('getProperties', {'locationCode', 'ROVMP'}, 10, NaN);
+ function testInvalidParamName(this)
+ % try an invalid locationCode, verify error structure
+ filters = {'fakeParamName', 'conductivity'};
+ verifyError(this, @() this.onc.getProperties(filters), 'onc:http400:error129');
end
- function testFilterDeviceCategoryCode(this)
- properties = this.testSingleFilter('getProperties', {'deviceCategoryCode', 'CTD'}, 10, NaN);
+ function testNoData(this)
+ % try a deviceCategories query with 0 results, verify result message
+ filters = {'propertyCode', 'conductivity', 'locationCode', 'SAAN'};
+ verifyError(this, @() this.onc.getProperties(filters), 'onc:http404');
end
- function testFilterDeviceCode(this)
- properties = this.testSingleFilter('getProperties', {'deviceCode', 'ALECACTW-CAR0014'}, 3, NaN);
+ function testValidParams(this)
+ filters = {'propertyCode', 'conductivity', 'locationCode', 'BACAX', 'deviceCategoryCode', 'CTD'};
+
+ properties = this.onc.getProperties(filters);
+ verifyTrue(this, length(properties) >= 1);
+ expectedPropertiesFields = ["cvTerm", "description", "hasDeviceData", ...
+ "hasPropertyData", "propertyCode", "propertyName", "uom"];
+ verify_fields(this, properties(1), expectedPropertiesFields);
+ expectedCvTermFields = ["uri", "vocabulary"];
+ verify_fields(this, properties(1).cvTerm.uom, expectedCvTermFields);
end
end
end
diff --git a/onc/tests/suites/Test06_DataProductDiscovery.m b/onc/tests/suites/Test06_DataProductDiscovery.m
index d5b6769..111ab3e 100644
--- a/onc/tests/suites/Test06_DataProductDiscovery.m
+++ b/onc/tests/suites/Test06_DataProductDiscovery.m
@@ -1,15 +1,15 @@
% DataProductsDiscovery test suite
% Contains test cases for the dataProducts discovery service.
-classdef Test06_DataProductDiscovery < TestDiscovery
-
- %% Public Methods
- methods
-
- function obj = Test06_DataProductDiscovery()
- % Constructor
- obj@TestDiscovery();
- obj.expectedFields('getDataProducts') = ["dataProductCode", "dataProductName", "extension", "hasDeviceData", "hasPropertyData", "helpDocument"];
+classdef Test06_DataProductDiscovery < matlab.unittest.TestCase
+ properties
+ onc
+ end
+
+ methods (TestClassSetup)
+ function classSetup(this)
+ config = globals();
+ this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
@@ -17,54 +17,38 @@
methods (Test)
%% General test cases
- function testGetAllDataProducts(this)
- % Make an unfiltered dataProducts request
+ function testInvalidParamValue(this)
+ % Make an unfiltered deviceCategories request
% verifies: expected fields, minimum rows
- dataProducts = this.o.getDataProducts();
- verify_fields(this, dataProducts, this.expectedFields('getDataProducts'));
- this.verify_min_length(dataProducts, 100);
- end
-
- function testWrongDataProductCode(this)
- % try an invalid dataProductCode, verify error structure
- dataProducts = this.o.getDataProducts({'dataProductCode', 'XYZ321'});
- verify_error_response(this, dataProducts);
- end
-
- function testNoDataProductsFound(this)
- % try a dataProducts query with 0 results, verify result is an empty 0x0 matrix
- dataProducts = this.o.getDataProducts({'locationCode', 'SAAN', 'deviceCategoryCode', 'POWER_SUPPLY'});
- verifyEqual(this, size(dataProducts), [0 0]);
-
- end
- %% Single filter test cases
- % These tests invoke getDataProducts with a single filter, for every supported filter
- % Verifications according to tests documentation at: https://internal.oceannetworks.ca/x/xYI2Ag
-
- function testFilterDataProductCode(this)
- dataProducts = this.testSingleFilter('getDataProducts', {'dataProductCode', 'CPD'}, 1, 1);
- verifyEqual(this, dataProducts(1).dataProductCode, 'CPD');
- end
-
- function testFilterExtension(this)
- dataProducts = this.testSingleFilter('getDataProducts', {'extension', 'cor'}, 1, 2);
- verifyEqual(this, dataProducts(1).extension, 'cor');
- end
-
- function testFilterLocationCode(this)
- dataProducts = this.testSingleFilter('getDataProducts', {'locationCode', 'SAAN'}, 1, NaN);
- end
-
- function testFilterDeviceCategoryCode(this)
- dataProducts = this.testSingleFilter('getDataProducts', {'deviceCategoryCode', 'CTD'}, 20, NaN);
- end
-
- function testFilterDeviceCode(this)
- dataProducts = this.testSingleFilter('getDataProducts', {'deviceCode', 'BC_POD1_AD2M'}, 5, NaN);
- end
-
- function testFilterPropertyCode(this)
- dataProducts = this.testSingleFilter('getDataProducts', {'propertyCode', 'oxygen'}, 10, NaN);
+ filters = {'dataProductCode','XYZ123'};
+ verifyError(this, @() this.onc.getDataProducts(filters), 'onc:http400:error127');
+ end
+
+ function testInvalidParamName(this)
+ % try an invalid locationCode, verify error structure
+ filters = {'fakeParamName', 'HSD'};
+ verifyError(this, @() this.onc.getDataProducts(filters), 'onc:http400:error129');
+ end
+
+ function testNoData(this)
+ % try a deviceCategories query with 0 results, verify result message
+ filters = {'dataProductCode', 'HSD', 'extension', 'txt'};
+ verifyError(this, @() this.onc.getDataProducts(filters), 'onc:http404');
+ end
+
+ function testValidParams(this)
+ filters = {'dataProductCode', 'HSD', 'extension', 'png'};
+
+ dataProducts = this.onc.getDataProducts(filters);
+ verifyTrue(this, length(dataProducts) >= 1);
+ expectedDataProductFields = ["dataProductCode", "dataProductName", "dataProductOptions", ...
+ "extension", "hasDeviceData", "hasPropertyData", "helpDocument"];
+ verify_fields(this, dataProducts(1), expectedDataProductFields);
+ expectedDataProductOptions = ["allowableRange", "allowableValues", "defaultValue", ...
+ "documentation", "option", "suboptions"];
+ verify_fields(this, dataProducts(1).dataProductOptions(7), expectedDataProductOptions);
+ expectedDataProductOptionsAllowableRange = ["lowerBound", "onlyIntegers", "unitOfMeasure", "upperBound"];
+ verify_fields(this, dataProducts(1).dataProductOptions(7).allowableRange, expectedDataProductOptionsAllowableRange);
end
end
end
diff --git a/onc/tests/suites/Test07_DataProductDelivery.m b/onc/tests/suites/Test07_DataProductDelivery.m
index b5abfd2..db393ae 100644
--- a/onc/tests/suites/Test07_DataProductDelivery.m
+++ b/onc/tests/suites/Test07_DataProductDelivery.m
@@ -4,225 +4,148 @@
classdef Test07_DataProductDelivery < matlab.unittest.TestCase
%% Private Properties
properties (SetAccess = private)
+ onc
+ outPath
maxRetries % default max Number of orderDataProduct() retries
-
- % dummy filters for downloading a data product with 1 file
- F_DUMMY1 = struct( ...
- 'dataProductCode', 'TSSD', ...
- 'extension', 'csv', ...
- 'locationCode', 'BACAX', ...
- 'deviceCategoryCode', 'ADCP2MHZ', ...
- 'dateFrom', '2016-07-27T00:00:00.000Z', ...
- 'dateTo', '2016-07-27T00:00:30.000Z', ...
- 'dpo_dataGaps', '0', ...
- 'dpo_qualityControl', '1', ...
- 'dpo_resample', 'none');
-
- % dummy filters for downloading a data product with 2 files
- F_DUMMY2 = struct( ...
+ Params = struct( ...
'dataProductCode', 'TSSP', ...
'extension', 'png', ...
- 'locationCode', 'CRIP.C1', ...
- 'deviceCategoryCode', 'CTD', ...
- 'dateFrom', '2019-03-20T00:00:00.000Z', ...
- 'dateTo', '2019-03-20T00:30:00.000Z', ...
+ 'dateFrom', '2019-08-29', ...
+ 'dateTo', '2019-08-30', ...
+ 'locationCode', 'CRIP.C1', ...
+ 'deviceCategoryCode', 'CTD', ... ...
'dpo_qualityControl', '1', ...
- 'dpo_resample', 'none');
-
- % fake filter
- F_FAKE = struct( ...
- 'dataProductCode', 'FAKECODE', ...
- 'extension', 'XYZ', ...
- 'locationCode', 'AREA51', ...
- 'deviceCategoryCode', 'AK47', ...
- 'dateFrom', '2019-03-20T00:00:00.000Z', ...
- 'dateTo', '2019-03-20T00:30:00.000Z', ...
- 'dpo_qualityControl', '1', ...
- 'dpo_resample', 'none');
+ 'dpo_resample', 'none'...
+ );
+
+ expectedFields = struct('url', 'char', ...
+ 'status', 'char', ...
+ 'size', 'int', ...
+ 'file', 'char', ...
+ 'index', 'char', ...
+ 'downloaded' , 'logical', ...
+ 'requestCount', 'int', ...
+ 'fileDownloadTime', 'double');
end
methods (TestClassSetup)
- function prepareSuite(testCase)
- s = rmdir('tests/output/07', 's'); % delete directory contents
+ function classSetup(this)
+ config = globals();
+ this.outPath = 'output';
+ this.onc = Onc(config.token, config.production, config.showInfo, this.outPath, config.timeout);
+ this.maxRetries = 100;
end
+
end
methods (TestClassTeardown)
- function cleanSuite(testCase)
- s = rmdir('tests/output/07', 's'); % delete directory contents
+ function cleanSuite(this)
+ if isfolder(this.outPath)
+ rmdir(this.outPath, 's');
+ end
end
end
%% Protected Methods
methods (Access = protected)
-
- function validStruct = verifyRowStructure(this, data)
- validStruct = verify_fields(this, data, ["url", "status", "status", "size", "file", "index", "downloaded", "requestCount", "fileDownloadTime"]);
- end
-
- % Validates that row at index has the status and downloaded provided
- function validateRow(this, rows, varargin)
- [index, status, downloaded] = util.param(varargin, 'index', '1', 'status', 'complete', 'downloaded', false);
-
- % grab the row at index
- row = rows(1);
- for i = 1: numel(rows)
- if rows(i).index == index
- row = rows(i);
- break;
- end
- end
- verifyInstanceOf(this, row, 'struct');
-
- % verify properties
- verifyEqual(this, row.status, status);
- verifyEqual(this, row.downloaded, downloaded);
- end
-
- function cleanDirectory(this, outPath)
- % Deletes all files in output directory
- delete(strcat(outPath, '/*'));
- end
-
- function onc = prepareOnc(this, outPath)
- global config;
- onc = Onc(config.token, config.production, config.showInfo, outPath, config.timeout);
- end
+ function updateOncOutPath(this, outPath)
+ this.onc = Onc(this.onc.token, this.onc.production, this.onc.showInfo, outPath, this.onc.timeout);
+ end
end
- %% Public Methods
- methods
- function this = Test07_DataProductDelivery()
- this.maxRetries = 100;
- end
- end
- %% Test methods
+ %% Test method
+ %% Testing order method
methods (Test)
- function test01_order_product_links_only(this)
- % Order a dummy data product, only obtain the download links
- onc = this.prepareOnc('output/07/01');
- this.cleanDirectory(onc.outPath);
- result = onc.orderDataProduct(this.F_DUMMY1, 100, true, false);
- rows = result.downloadResults;
-
- verifyLength(this, rows, 1);
- if verifyRowStructure(this, rows(1))
- this.validateRow(rows, 'index', '1', 'status', 'complete', 'downloaded', false);
- verify_files_in_path(this, onc.outPath, 0);
- end
-
- end
-
- function test02_order_links_with_metadata(this)
- onc = this.prepareOnc('output/07/02');
- this.cleanDirectory(onc.outPath);
- result = onc.orderDataProduct(this.F_DUMMY1, 100, true, true);
- rows = result.downloadResults;
- verifyLength(this, rows, 2);
- if verifyRowStructure(this, rows(1))
- this.validateRow(rows, 'index', '1', 'status', 'complete', 'downloaded', false);
- this.validateRow(rows, 'index', 'meta', 'status', 'complete', 'downloaded', false);
- end
- verify_files_in_path(this, onc.outPath, 0);
+ function testInvalidParamValue(this)
+ filters = this.Params;
+ filters.dataProductCode ='XYZ123';
+ verifyError(this, @() this.onc.orderDataProduct(filters, this.maxRetries), 'onc:http400:error127');
+ end
+
+ function testValidDefault(this)
+ this.updateOncOutPath('output/testValidDefault');
+ result = this.onc.orderDataProduct(this.Params);
+ verifyTrue(this, length(result.downloadResults) == 3, ...
+ 'The first two are png files, and the third one is the metadata.');
+ assertEqual(this, result.downloadResults(1).status, 'complete');
+ assertEqual(this, result.downloadResults(1).index, '1');
+ assertTrue(this, result.downloadResults(1).downloaded);
+
+
+ assertEqual(this, result.downloadResults(3).status, 'complete');
+ assertEqual(this, result.downloadResults(3).index, 'meta');
+ assertTrue(this, result.downloadResults(3).downloaded);
+
+ verify_files_in_path(this, this.onc.outPath, 3);
+ verify_field_value_type(this, result.downloadResults(1), this.expectedFields);
+ end
+
+ function testValidNoMetadata(this)
+ this.updateOncOutPath('output/testValidNoMetadata');
+ result = this.onc.orderDataProduct(this.Params, 'includeMetadataFile', false);
+ verifyTrue(this, length(result.downloadResults) == 2);
+ assertEqual(this, result.downloadResults(1).status, 'complete');
+ assertEqual(this, result.downloadResults(1).index, '1');
+ assertTrue(this, result.downloadResults(1).downloaded);
+
+ verify_files_in_path(this, this.onc.outPath, 2, 'The first two are png files.');
+ verify_field_value_type(this, result.downloadResults(1), this.expectedFields);
+ end
+
+ function testValidResultsOnly(this)
+ this.updateOncOutPath('output/testValidResultsOnly');
+ result = this.onc.orderDataProduct(this.Params, 'downloadResultsOnly', true);
+ verifyTrue(this, length(result.downloadResults) == 3, ...
+ 'The first two are png files, and the third one is the metadata.');
+ assertEqual(this, result.downloadResults(1).status, 'complete');
+ assertEqual(this, result.downloadResults(1).index, '1');
+ assertTrue(this, ~result.downloadResults(1).downloaded);
+
+
+ assertEqual(this, result.downloadResults(3).status, 'complete');
+ assertEqual(this, result.downloadResults(3).index, 'meta');
+ assertTrue(this, ~result.downloadResults(3).downloaded);
+
+ verify_files_in_path(this, this.onc.outPath, 0, 'No files should be downloaded when download_results_only is True.');
+ verify_field_value_type(this, result.downloadResults(1), this.expectedFields);
+
+
+ end
+
+ function testValidManual(this)
+ this.updateOncOutPath('output/testValidManual');
+ requestId = this.onc.requestDataProduct(this.Params).dpRequestId;
+ runId = this.onc.runDataProduct(requestId).runIds(1);
+ data = this.onc.downloadDataProduct(runId);
+ verifyTrue(this, length(data) == 3, ...
+ 'The first two are png files, and the third one is the metadata.');
+ assertEqual(this, data(1).status, 'complete');
+ assertEqual(this, data(1).index, '1');
+ assertTrue(this, data(1).downloaded);
+
+
+ assertEqual(this, data(3).status, 'complete');
+ assertEqual(this, data(3).index, 'meta');
+ assertTrue(this, data(3).downloaded);
+
+ verify_files_in_path(this, this.onc.outPath, 3);
+ verify_field_value_type(this, data(1), this.expectedFields);
+
end
-
- function test03_order_and_download(this)
- onc = this.prepareOnc('output/07/03');
- this.cleanDirectory(onc.outPath);
- result = onc.orderDataProduct(this.F_DUMMY1, 100, false, false);
- rows = result.downloadResults;
- verifyLength(this, rows, 1);
- if verifyRowStructure(this, rows(1))
- this.validateRow(rows, 'index', '1', 'status', 'complete', 'downloaded', true);
- end
- verify_files_in_path(this, onc.outPath, 1);
- end
-
- function test04_order_and_download_multiple(this)
- onc = this.prepareOnc('output/07/04');
- this.cleanDirectory(onc.outPath);
- result = onc.orderDataProduct(this.F_DUMMY2, 100, false, false);
- rows = result.downloadResults;
- verifyLength(this, rows, 2);
- if verifyRowStructure(this, rows(1))
- this.validateRow(rows, 'index', '1', 'status', 'complete', 'downloaded', true);
- this.validateRow(rows, 'index', '2', 'status', 'complete', 'downloaded', true);
- end
- verify_files_in_path(this, onc.outPath, 2);
- end
-
- function test05_order_and_download_with_metadata(this)
- onc = this.prepareOnc('output/07/05');
- this.cleanDirectory(onc.outPath);
- result = onc.orderDataProduct(this.F_DUMMY1, 100, false, true);
- rows = result.downloadResults;
- verifyLength(this, rows, 2);
- if verifyRowStructure(this, rows(1))
- this.validateRow(rows, 'index', '1', 'status', 'complete', 'downloaded', true);
- this.validateRow(rows, 'index', 'meta', 'status', 'complete', 'downloaded', true);
- end
- verify_files_in_path(this, onc.outPath, 2);
- end
-
- function test06_order_and_download_multiple_with_metadata(this)
- onc = this.prepareOnc('output/07/06');
- this.cleanDirectory(onc.outPath);
- result = onc.orderDataProduct(this.F_DUMMY2, 100, false, true);
- rows = result.downloadResults;
- verifyLength(this, rows, 3);
- if verifyRowStructure(this, rows(1))
- this.validateRow(rows, 'index', '1', 'status', 'complete', 'downloaded', true);
- this.validateRow(rows, 'index', '2', 'status', 'complete', 'downloaded', true);
- this.validateRow(rows, 'index', 'meta', 'status', 'complete', 'downloaded', true);
- end
- verify_files_in_path(this, onc.outPath, 3);
- end
-
- function test07_wrong_order_request_argument(this)
- onc = this.prepareOnc('output/07/07');
- this.cleanDirectory(onc.outPath);
- verifyError(this, @() onc.orderDataProduct(this.F_FAKE, 100, true, false), 'onc:http400');
- end
-
- function test08_manual_request_run_and_download(this)
- onc = this.prepareOnc('output/07/08');
- this.cleanDirectory(onc.outPath);
- reqId = onc.requestDataProduct(this.F_DUMMY1).dpRequestId;
- runId = onc.runDataProduct(reqId).runIds(1);
- rows = onc.downloadDataProduct( ...
- runId, 'maxRetries', 0, 'downloadResultsOnly', false, 'includeMetadataFile', true);
- verifyLength(this, rows, 2);
- this.validateRow(rows, 'index', '1', 'status', 'complete', 'downloaded', true);
- this.validateRow(rows, 'index', 'meta', 'status', 'complete', 'downloaded', true);
- verify_files_in_path(this, onc.outPath, 2);
- end
-
- function test09_manual_request_run_and_download_results_only(this)
- onc = this.prepareOnc('output/07/09');
- this.cleanDirectory(onc.outPath);
- reqId = onc.requestDataProduct(this.F_DUMMY1).dpRequestId;
- runId = onc.runDataProduct(reqId).runIds(1);
- rows = onc.downloadDataProduct(runId, 'downloadResultsOnly', true, 'includeMetadataFile', true);
- verifyLength(this, rows, 2);
- this.validateRow(rows, 'index', '1', 'status', 'complete', 'downloaded', false);
- this.validateRow(rows, 'index', 'meta', 'status', 'complete', 'downloaded', false);
- verify_files_in_path(this, onc.outPath, 0);
- end
-
- function test10_manual_run_with_wrong_argument(this)
- onc = this.prepareOnc('output/07/10');
- this.cleanDirectory(onc.outPath);
- verifyError(this, @() onc.runDataProduct(1234568790), 'onc:http400');
+
+ %% Testing run method
+ function testInvalidRequestId(this)
+ verifyError(this, @() this.onc.runDataProduct(1234567890), 'onc:http400:error127');
end
-
- function test11_manual_download_with_wrong_argument(this)
- onc = this.prepareOnc('output/07/11');
- this.cleanDirectory(onc.outPath);
- verifyError(this, ...
- @() onc.downloadDataProduct(1234568790, 'downloadResultsOnly', false, 'includeMetadataFile', true), ...
- 'DataProductFile:HTTP400');
+
+ %% Testing download method
+ function testInvalidRunId(this)
+ verifyError(this, @() this.onc.downloadDataProduct(1234567890), 'onc:http400:error127');
end
end
+
+
+
end
diff --git a/onc/tests/suites/Test08_RealTime.m b/onc/tests/suites/Test08_RealTime.m
index 6ff7626..982d675 100644
--- a/onc/tests/suites/Test08_RealTime.m
+++ b/onc/tests/suites/Test08_RealTime.m
@@ -1,111 +1,222 @@
classdef Test08_RealTime < matlab.unittest.TestCase
properties (SetAccess = private)
onc
- F_SCALAR1 = struct('locationCode', 'CRIP.C1', 'deviceCategoryCode', 'CTD', 'propertyCode', 'density', ...
- 'dateFrom', '2018-03-24T00:00:00.000Z', 'dateTo', '2018-03-24T00:00:15.000Z');
- % 3 pages of temperature, by location
- F_SCALAR2 = struct('locationCode', 'CRIP.C1', 'deviceCategoryCode', 'CTD', 'propertyCode', 'density', ...
- 'dateFrom', '2018-03-24T00:00:00.000Z', 'dateTo', '2018-03-24T00:00:15.000Z', 'rowLimit', 5);
- % 3 pages of temperature
- F_SCALAR3 = struct('deviceCode', 'BARIX001', 'dateFrom', '2017-06-08T00:00:00.000Z', 'dateTo', 'PT7M', 'rowLimit', 5);
- F_RAW1 = struct('locationCode', 'CRIP.C1', 'deviceCategoryCode', 'CTD', ...
- 'dateFrom', '2018-03-24T00:00:00.000Z', 'dateTo', '2018-03-24T00:00:10.000Z');
- F_RAW3 = struct('locationCode', 'CRIP.C1', 'deviceCategoryCode', 'CTD', ...
- 'dateFrom', '2018-03-24T00:00:00.000Z', 'dateTo', '2018-03-24T00:00:15.000Z', 'rowLimit', 5);
- F_RAWDEV1 = struct('deviceCode', 'BARIX001', 'dateFrom', '2017-06-08T00:00:00.000Z', 'dateTo', 'PT5S');
- F_WRONG_FILTERS = struct('locationCode', 'ONION', 'deviceCategoryCode', 'POTATO', 'propertyCode', 'BANANA', ...
- 'dateFrom', '2018-03-24T00:00:00.000Z', 'dateTo', '2018-03-24T00:00:10.000Z');
- F_NODATA = struct('locationCode', 'CRIP.C1 ', 'deviceCategoryCode', 'CTD', ...
- 'dateFrom', '2015-03-24T00:00:00.000Z', 'dateTo', '2015-03-24T00:00:10.000Z');
+ outPath = 'output';
+ paramsLocation = struct('locationCode', 'NCBC',...
+ 'deviceCategoryCode', 'BPR',...
+ 'propertyCode', 'seawatertemperature,totalpressure',...
+ 'dateFrom', '2019-11-23T00:00:00.000Z',...
+ 'dateTo', '2019-11-23T00:01:00.000Z',...
+ 'rowLimit', 80000);
+
+ paramsRawLocation = struct('locationCode', 'NCBC',...
+ 'deviceCategoryCode', 'BPR',...
+ 'dateFrom', '2019-11-23T00:00:00.000Z',...
+ 'dateTo', '2019-11-23T00:01:00.000Z',...
+ 'rowLimit', 80000, ...
+ 'sizeLimit', 20, ...
+ 'convertHexToDecimal', false);
+
+ paramsLocationMultiPages = struct('locationCode', 'NCBC',...
+ 'deviceCategoryCode', 'BPR',...
+ 'propertyCode', 'seawatertemperature,totalpressure',...
+ 'dateFrom', '2019-11-23T00:00:00.000Z',...
+ 'dateTo', '2019-11-23T00:01:00.000Z',...
+ 'rowLimit', 25);
+
+ paramsRawLocationMultiPages = struct('locationCode', 'NCBC',...
+ 'deviceCategoryCode', 'BPR',...
+ 'dateFrom', '2019-11-23T00:00:00.000Z',...
+ 'dateTo', '2019-11-23T00:01:00.000Z',...
+ 'rowLimit', 25, ...
+ 'sizeLimit', 20, ...
+ 'convertHexToDecimal', false);
+
+ paramsDevice = struct('deviceCode', 'BPR-Folger-59', ...
+ 'dateFrom', '2019-11-23T00:00:00.000Z', ...
+ 'dateTo', '2019-11-23T00:01:00.000Z', ...
+ 'rowLimit', 80000);
+
+ paramsDeviceMultiPages = struct('deviceCode', 'BPR-Folger-59', ...
+ 'dateFrom', '2019-11-23T00:00:00.000Z', ...
+ 'dateTo', '2019-11-23T00:01:00.000Z', ...
+ 'rowLimit', 25);
end
methods (TestClassSetup)
- function prepareSuite(testCase)
- s = rmdir('tests/output/08', 's'); % delete directory contents
+ function classSetup(this)
+ config = globals();
+ this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
methods (TestClassTeardown)
- function cleanSuite(testCase)
- s = rmdir('tests/output/08', 's'); % delete directory contents
- end
- end
-
- %% Public Methods
- methods
- % Constructor
- function this = Test08_RealTime()
- global config;
- this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
+ function cleanSuite(this)
+ if isfolder(this.outPath)
+ rmdir(this.outPath, 's');
+ end
end
end
%% Test methods
methods (Test)
- %% General test cases
+ %% Testing scalardata location
+
+ function testLocationInvalidParamValue(this)
+ filters = this.paramsLocation;
+ filters.locationCode = 'XYZ123';
+ verifyError(this, @() this.onc.getDirectByLocation(filters), 'onc:http400:error127');
+ end
- function test01_scalar_by_location_1_page(this)
- response = this.onc.getDirectByLocation(this.F_SCALAR1);
- sensorData = response.sensorData(1);
- verify_has_field(this, sensorData, 'data');
- verify_field_value(this, sensorData, 'sensorCode', 'Sensor8_Voltage');
- verify_no_next_page(this, response);
+ function testLocationInvalidParamName(this)
+ filters = this.paramsLocation;
+ filters.fakeParamName = 'NCBC';
+ verifyError(this, @() this.onc.getDirectByLocation(filters), 'onc:http400:error129');
end
- function test02_scalar_by_location_3_pages(this)
- response = this.onc.getDirectByLocation(this.F_SCALAR2, true);
- sensorData = response.sensorData(1);
- verify_has_field(this, sensorData, 'data');
- verify_field_value(this, sensorData, 'sensorCode', 'Sensor8_Voltage');
- verifyLength(this, sensorData.data.values, 15);
- verify_no_next_page(this, response);
+ function testLocationNoData(this)
+ filters = this.paramsLocation;
+ filters.dateFrom = '2000-01-01';
+ filters.dateTo = '2000-01-02';
+ verifyError(this, @() this.onc.getDirectByLocation(filters), 'onc:http400:error127');
end
- function test03_scalar_by_location_no_results(this)
- response = this.onc.getDirectByLocation(this.F_NODATA);
- verifyLength(this, response.sensorData, 0);
+ function testLocationValidParamsOnePage(this)
+ result = this.onc.getDirectByLocation(this.paramsLocation);
+ resultAllPages = this.onc.getDirectByLocation(this.paramsLocationMultiPages, 'allPages', true);
+ assertTrue(this, length(result.sensorData(1).data.values) > this.paramsLocationMultiPages.rowLimit, ...
+ 'Test should return at least `rowLimit` rows for each sensor.');
+ assertEmpty(this, result.next, 'Test should return only one page.');
+ assertEqual(this, resultAllPages.sensorData(1).data, result.sensorData(1).data, ...
+ 'Test should concatenate rows for all pages.');
+ assertEmpty(this, resultAllPages.next, 'Test should return only one page.');
+ end
+
+ function testLocationValidParamsMultiplePages(this)
+ result = this.onc.getDirectByLocation(this.paramsLocationMultiPages);
+ assertEqual(this, length(result.sensorData(1).data.values), this.paramsLocationMultiPages.rowLimit, ...
+ 'Test should only return `rowLimit` rows for each sensor.');
+ assertTrue(this, ~isempty(result.next), 'Test should return multiple pages.');
end
- function test04_scalar_by_location_wrong_filters(this)
- result = this.onc.getDirectByLocation(this.F_WRONG_FILTERS);
- verify_error_response(this, result);
+ %% Testing rawdata location
+ function testRawLocationInvalidParamValue(this)
+ filters = this.paramsRawLocation;
+ filters.locationCode = 'XYZ123';
+ verifyError(this, @() this.onc.getDirectRawByLocation(filters), 'onc:http400:error127');
+ end
+
+ function testRawLocationInvalidParamName(this)
+ filters = this.paramsRawLocation;
+ filters.fakeParamName = 'NCBC';
+ verifyError(this, @() this.onc.getDirectRawByLocation(filters), 'onc:http400:error129');
end
- function test05_raw_by_location_1_page(this)
- response = this.onc.getDirectRawByLocation(this.F_RAW1);
- verifyLength(this, response.data.readings, 10);
- verify_no_next_page(this, response);
+ function testRawLocationNoData(this)
+ filters = this.paramsRawLocation;
+ filters.dateFrom = '2000-01-01';
+ filters.dateTo = '2000-01-02';
+ verifyError(this, @() this.onc.getDirectRawByLocation(filters), 'onc:http400:error127');
end
- function test06_raw_by_location_3_pages(this)
- response = this.onc.getDirectRawByLocation(this.F_RAW3, true);
- verifyLength(this, response.data.readings, 15);
- verify_no_next_page(this, response);
+ function testRawLocationValidParamsOnePage(this)
+ result = this.onc.getDirectRawByLocation(this.paramsRawLocation);
+ resultAllPages = this.onc.getDirectRawByLocation(this.paramsRawLocationMultiPages, 'allPages', true);
+ assertTrue(this, length(result.data.readings) > this.paramsRawLocationMultiPages.rowLimit, ...
+ 'Test should return at least `rowLimit` rows');
+ assertEmpty(this, result.next, 'Test should return only one page.');
+ assertEqual(this, resultAllPages.data, result.data, ...
+ 'Test should concatenate rows for all pages.');
+ assertEmpty(this, resultAllPages.next, 'Test should return only one page.');
+ end
+
+ function testRawLocationValidParamsMultiplePages(this)
+ result = this.onc.getDirectRawByLocation(this.paramsRawLocationMultiPages);
+ assertEqual(this, length(result.data.readings), this.paramsRawLocationMultiPages.rowLimit, ...
+ 'Test should only return `rowLimit` rows for each sensor.');
+ assertTrue(this, ~isempty(result.next), 'Test should return multiple pages.');
+ end
+
+ %% Testing scalardata device
+
+ function testDeviceInvalidParamValue(this)
+ filters = this.paramsDevice;
+ filters.deviceCode = 'XYZ123';
+ verifyError(this, @() this.onc.getDirectByDevice(filters), 'onc:http400:error127');
end
- function test07_raw_by_device_1_page(this)
- response = this.onc.getDirectRawByDevice(this.F_RAWDEV1);
- verifyLength(this, response.data.readings, 47);
- verify_no_next_page(this, response);
+ function testDeviceInvalidParamName(this)
+ filters = this.paramsDevice;
+ filters.fakeParamName = 'BPR-Folger-59';
+ verifyError(this, @() this.onc.getDirectByDevice(filters), 'onc:http400:error129');
end
- function test08_raw_no_results(this)
- response = this.onc.getDirectRawByLocation(this.F_NODATA);
- verifyLength(this, response.data.readings, 0);
+ function testDeviceNoData(this)
+ filters = this.paramsDevice;
+ filters.dateFrom = '2000-01-01';
+ filters.dateTo = '2000-01-02';
+ result = this.onc.getDirectByDevice(filters);
+ assertEmpty(this, result.sensorData);
end
- function test09_raw_by_location_wrong_filters(this)
- result = this.onc.getDirectRawByLocation(this.F_WRONG_FILTERS);
- verify_error_response(this, result);
+ function testDeviceValidParamsOnePage(this)
+ result = this.onc.getDirectByDevice(this.paramsDevice);
+ resultAllPages = this.onc.getDirectByDevice(this.paramsDeviceMultiPages, 'allPages', true);
+ assertTrue(this, length(result.sensorData(1).data.values) > this.paramsDeviceMultiPages.rowLimit, ...
+ 'Test should return at least `rowLimit` rows for each sensor.');
+ assertEmpty(this, result.next, 'Test should return only one page.');
+ assertEqual(this, resultAllPages.sensorData(1).data, result.sensorData(1).data, ...
+ 'Test should concatenate rows for all pages.');
+ assertEmpty(this, resultAllPages.next, 'Test should return only one page.');
+ end
+
+ function testDeviceValidParamsMultiplePages(this)
+ result = this.onc.getDirectByDevice(this.paramsDeviceMultiPages);
+ assertEqual(this, length(result.sensorData(1).data.values), this.paramsDeviceMultiPages.rowLimit, ...
+ 'Test should only return `rowLimit` rows for each sensor.');
+ assertTrue(this, ~isempty(result.next), 'Test should return multiple pages.');
end
- function test10_scalar_by_device_6_pages(this)
- response = this.onc.getDirectByDevice(this.F_SCALAR3, true);
- sensorData = response.sensorData(1);
- verify_has_field(this, sensorData, 'data');
- verify_field_value(this, sensorData, 'sensorCode', 'analog_input501');
- verifyLength(this, sensorData.data.values, 14);
- verify_no_next_page(this, response);
+ %% Testing rawdata device
+ %{
+ % waiting for python updates
+ function testRawDeviceInvalidParamValue(this)
+ filters = this.paramsDevice;
+ filters.deviceCode = 'XYZ123';
+ verifyError(this, @() this.onc.getDirectRawByDevice(filters), 'onc:http400:error127');
+ end
+
+ function testRawDeviceInvalidParamName(this)
+ filters = this.paramsDevice;
+ filters.fakeParamName = 'BPR-Folger-59';
+ verifyError(this, @() this.onc.getDirectRawByDevice(filters), 'onc:http400:error129');
+ end
+
+ function testRawDeviceNoData(this)
+ filters = this.paramsDevice;
+ filters.dateFrom = '2000-01-01';
+ filters.dateTo = '2000-01-02';
+ result = this.onc.getDirectByDevice(filters);
+ assertEmpty(this, result.sensorData);
+ end
+
+ function testRawDeviceValidParamsOnePage(this)
+ result = this.onc.getDirectByDevice(this.paramsDevice);
+ resultAllPages = this.onc.getDirectByDevice(this.paramsDeviceMultiPages, 'allPages', true);
+ assertTrue(this, length(result.sensorData(1).data.values) > this.paramsDeviceMultiPages.rowLimit, ...
+ 'Test should return at least `rowLimit` rows for each sensor.');
+ assertEmpty(this, result.next, 'Test should return only one page.');
+ assertEqual(this, resultAllPages.sensorData(1).data, result.sensorData(1).data, ...
+ 'Test should concatenate rows for all pages.');
+ assertEmpty(this, resultAllPages.next, 'Test should return only one page.');
+ end
+
+ function testRawDeviceValidParamsMultiplePages(this)
+ result = this.onc.getDirectByDevice(this.paramsDeviceMultiPages);
+ assertEqual(this, length(result.sensorData(1).data.values), this.paramsDeviceMultiPages.rowLimit, ...
+ 'Test should only return `rowLimit` rows for each sensor.');
+ assertTrue(this, ~isempty(result.next), 'Test should return multiple pages.');
end
+ %}
end
end
\ No newline at end of file
diff --git a/onc/tests/suites/Test09_ArchiveFiles.m b/onc/tests/suites/Test09_ArchiveFiles.m
index a362dda..2a30a94 100644
--- a/onc/tests/suites/Test09_ArchiveFiles.m
+++ b/onc/tests/suites/Test09_ArchiveFiles.m
@@ -1,139 +1,183 @@
classdef Test09_ArchiveFiles < matlab.unittest.TestCase
properties (SetAccess = private)
onc
- F_LOCATION1 = struct("locationCode", "RISS", "deviceCategoryCode", "VIDEOCAM", "dateFrom", "2016-12-01T00:00:00.000Z", "dateTo", "2016-12-01T01:00:00.000Z");
- F_LOCATION3 = struct("locationCode", "RISS", "deviceCategoryCode", "VIDEOCAM", "dateFrom", "2016-12-01T00:00:00.000Z", "dateTo", "2016-12-01T01:00:00.000Z", "rowLimit", 5);
- F_LOCATIONFULL = struct("locationCode", "RISS", "deviceCategoryCode", "VIDEOCAM", "dateFrom", "2016-12-01T00:00:00.000Z", "dateTo", "2016-12-01T01:00:00.000Z", "extension", "mp4");
- F_LOC_RETURN1 = struct("locationCode", "RISS", "deviceCategoryCode", "VIDEOCAM", "dateFrom", "2016-12-01T00:00:00.000Z", "dateTo", "2016-12-01T01:00:00.000Z", "rowLimit", 5, "returnOptions", "archiveLocation");
- F_LOC_RETURN2 = struct("locationCode", "RISS", "deviceCategoryCode", "VIDEOCAM", "dateFrom", "2016-12-01T00:00:00.000Z", "dateTo", "2016-12-03T00:00:00.000Z", "rowLimit", 50, "returnOptions", "all", "extension", "mp4");
- F_DEVICE1 = struct("dateFrom", "2010-01-01T00:00:00.000Z", "dateTo", "2010-01-01T00:02:00.000Z");
- F_DEVICE1EXT = struct("deviceCode", "NAXYS_HYD_007", "dateFrom", "2010-01-01T00:00:00.000Z", "dateTo", "2010-01-01T00:02:00.000Z", "extension", "mp3");
- F_GETDIRECT_DEV = struct("dateFrom", "2010-01-01T00:00:00.000Z", "dateTo", "2010-01-01T00:00:30.000Z", "deviceCode", "NAXYS_HYD_007", "returnOptions", "all");
- F_GETDIRECT_LOC = struct("dateFrom", "2016-12-01T00:00:00.000Z", "dateTo", "2016-12-01T01:00:00.000Z", "locationCode", "RISS", "deviceCategoryCode", "VIDEOCAM", "extension", "mp4");
- F_FAKE = struct("locationCode", "AREA51");
+ outPath = 'output';
+ paramsLocation = struct( ...
+ 'locationCode', 'NCBC',...
+ 'deviceCategoryCode', 'BPR',...
+ 'dateFrom', '2019-11-23',...
+ 'dateTo', '2019-11-26',...
+ 'fileExtension', 'txt',...
+ 'rowLimit', 80000,...
+ 'page', 1);
+
+ paramsLocationMultiPages = struct( ...
+ 'locationCode', 'NCBC',...
+ 'deviceCategoryCode', 'BPR',...
+ 'dateFrom', '2019-11-23',...
+ 'dateTo', '2019-11-26',...
+ 'fileExtension', 'txt',...
+ 'rowLimit', 2,...
+ 'page', 1);
+
+ paramsDevice = struct('deviceCode', 'BPR-Folger-59',...
+ 'dateFrom', '2019-11-23',...
+ 'dateTo', '2019-11-26',...
+ 'fileExtension', 'txt',...
+ 'rowLimit', 80000,...
+ 'page', 1);
+
+ paramsDeviceMultiPages = struct('deviceCode', 'BPR-Folger-59',...
+ 'dateFrom', '2019-11-23',...
+ 'dateTo', '2019-11-26',...
+ 'fileExtension', 'txt',...
+ 'rowLimit', 2,...
+ 'page', 1);
end
methods (TestClassSetup)
- function prepareSuite(testCase)
- s = rmdir('tests/output/09', 's'); % delete directory contents
+ function classSetup(this)
+ config = globals();
+ this.onc = Onc(config.token, config.production, config.showInfo, this.outPath, config.timeout);
end
+
end
methods (TestClassTeardown)
- function cleanSuite(testCase)
- s = rmdir('tests/output/09', 's'); % delete directory contents
+ function cleanSuite(this)
+ if isfolder(this.outPath)
+ rmdir(this.outPath, 's');
+ end
end
end
- %% Public Methods
- methods
- % Constructor
- function this = Test09_ArchiveFiles()
- % prepare generic onc object
- global config;
- this.onc = Onc(config.token, config.production, config.showInfo, 'output/09', config.timeout);
- end
-
- function onc = prepareOnc(this, outPath)
- global config;
- if ~isempty(outPath)
- delete(sprintf('%s/*', outPath));
- end
- onc = Onc(config.token, config.production, config.showInfo, outPath, config.timeout);
- end
+ %% Protected helper method
+ methods (Access = protected)
+ function updateOncOutPath(this, outPath)
+ this.onc = Onc(this.onc.token, this.onc.production, this.onc.showInfo, outPath, this.onc.timeout);
+ end
end
%% Test methods
methods (Test)
- %% General test cases
-
- function test01_list_by_location_1_page(this)
- result = this.onc.getListByLocation(this.F_LOCATION1);
- verifyLength(this, result.files, 15);
+ %% Testing location
+ function testLocationInvalidParamValue(this)
+ filters = this.paramsLocation;
+ filters.locationCode = 'XYZ123';
+ verifyError(this, @() this.onc.getListByLocation(filters), 'onc:http400:error127');
+ end
+
+ function testLocationInvalidParamName(this)
+ filters = this.paramsLocation;
+ filters.fakeParamName = 'NCBC';
+ verifyError(this, @() this.onc.getListByLocation(filters), 'onc:http400:error129');
end
- function test02_list_by_location_3_pages(this)
- result = this.onc.getListByLocation(this.F_LOCATION1, true);
- verifyLength(this, result.files, 15);
+ function testLocationNoData(this)
+ filters = this.paramsLocation;
+ filters.dateFrom = '2000-01-01';
+ filters.dateTo = '2000-01-02';
+ verifyError(this, @() this.onc.getListByLocation(filters), 'onc:http400:error127');
end
- function test03_list_by_location_1_page_filter_ext(this)
- result = this.onc.getListByLocation(this.F_LOCATIONFULL);
- verifyLength(this, result.files, 1);
+ function testLocationValidParamsOnePage(this)
+ result = this.onc.getListByLocation(this.paramsLocation);
+ resultAllPages = this.onc.getListByLocation(this.paramsLocationMultiPages, 'allPages', true);
+ assertTrue(this, length(result.files) > this.paramsLocationMultiPages.rowLimit, ...
+ 'Test should return at least `rowLimit` rows.');
+ assertEmpty(this, result.next, 'Test should return only one page.');
+ assertEqual(this, resultAllPages.files, result.files, ...
+ 'Test should concatenate rows for all pages.');
+ assertEmpty(this, resultAllPages.next, 'Test should return only one page.');
end
- function test04_list_by_location_wrong_filters(this)
- result = this.onc.getListByLocation(this.F_DEVICE1);
- verify_error_response(this, result);
+ function testLocationValidParamsMultiplePages(this)
+ result = this.onc.getListByLocation(this.paramsLocationMultiPages);
+ assertEqual(this, length(result.files), this.paramsLocationMultiPages.rowLimit, ...
+ 'Test should only return `rowLimit` rows.');
+ assertTrue(this, ~isempty(result.next), 'Test should return multiple pages.');
end
- function test05_list_by_device_1_page_filter_ext(this)
- result = this.onc.getListByDevice(this.F_DEVICE1EXT);
- verifyLength(this, result.files, 4);
+ %% Testing device
+
+ function testDeviceInvalidParamValue(this)
+ filters = this.paramsDevice;
+ filters.deviceCode = 'XYZ123';
+ verifyError(this, @() this.onc.getListByDevice(filters), 'onc:http400:error127');
end
- function test06_get_file(this)
- onc9 = this.prepareOnc('output/09/06');
- onc9.getFile('NAXYS_HYD_007_20091231T235919.476Z-spect-small.png');
- verify_files_in_path(this, onc9.outPath, 1);
+ function testDeviceInvalidParamsMissingRequired(this)
+ filters = rmfield(this.paramsDevice, 'deviceCode');
+ verifyError(this, @() this.onc.getListByDevice(filters), 'onc:http400:error128');
end
- function test07_direct_files_device_returnOptions(this)
- onc9 = this.prepareOnc('output/09/07');
- result = onc9.getDirectFiles(this.F_GETDIRECT_DEV);
- verify_files_in_path(this, onc9.outPath, 12);
- verifyLength(this, result.downloadResults, 12);
+ function testDeviceInvalidParamName(this)
+ filters = this.paramsDevice;
+ filters.fakeParamName = 'BPR-Folger-59';
+ verifyError(this, @() this.onc.getListByDevice(filters), 'onc:http400:error129');
end
- function test08_direct_files_location_overwrite(this)
- onc9 = this.prepareOnc('output/09/08');
- result = onc9.getDirectFiles(this.F_GETDIRECT_LOC);
- verifyLength(this, result.downloadResults, 1);
- verify_field_value(this, result.downloadResults(1), 'status', 'completed');
- verify_files_in_path(this, onc9.outPath, 1);
- result = onc9.getDirectFiles(this.F_GETDIRECT_LOC);
- verifyLength(this, result.downloadResults, 1);
- verify_field_value(this, result.downloadResults(1), 'status', 'skipped');
- verify_files_in_path(this, onc9.outPath, 1);
+ function testDeviceNoData(this)
+ filters = this.paramsDevice;
+ filters.dateFrom = '2000-01-01';
+ filters.dateTo = '2000-01-02';
+ result = this.onc.getListByDevice(filters);
+ assertEmpty(this, result.files);
end
- function test09_getfile_wrong_filename(this)
- onc9 = this.prepareOnc('output/09/09');
- result = onc9.getFile('FAKEFILE.XYZ');
- verify_error_response(this, result);
+ function testDeviceValidParamsOnePage(this)
+ result = this.onc.getListByDevice(this.paramsDevice);
+ resultAllPages = this.onc.getListByDevice(this.paramsDeviceMultiPages, 'allPages', true);
+ assertTrue(this, length(result.files) > this.paramsDeviceMultiPages.rowLimit, ...
+ 'Test should return at least `rowLimit` rows.');
+ assertEmpty(this, result.next, 'Test should return only one page.');
+ assertEqual(this, resultAllPages.files, result.files, ...
+ 'Test should concatenate rows for all pages.');
+ assertEmpty(this, resultAllPages.next, 'Test should return only one page.');
end
- function test10_direct_files_no_source(this)
- onc9 = this.prepareOnc('output/09/10');
- verifyError(this, ...
- @() onc9.getDirectFiles(this.F_FAKE), ...
- 'Archive:InvalidFilters');
+ function testDeviceValidParamsMultiplePages(this)
+ result = this.onc.getListByDevice(this.paramsDeviceMultiPages);
+ assertEqual(this, length(result.files), this.paramsDeviceMultiPages.rowLimit, ...
+ 'Test should only return `rowLimit` rows.');
+ assertTrue(this, ~isempty(result.next), 'Test should return multiple pages.');
end
+
+ %% Testing download
- function test11_list_by_device_wrong_filters(this)
- result = this.onc.getListByDevice(this.F_FAKE);
- verify_error_response(this, result);
+ function testDownloadInvalidParamValue(this)
+ verifyError(this, @() this.onc.getFile('FAKEFILE.XYZ'), 'onc:http400:error96');
end
- function test12_list_by_location_3_pages_archiveLocations(this)
- result = this.onc.getListByLocation(this.F_LOC_RETURN1, true);
- verifyLength(this, result.files, 15);
- verify_has_field(this, result.files(1), 'archiveLocation');
+ function testDownloadInvalidParamsMissingRequired(this)
+ verifyError(this, @() this.onc.getFile(), 'onc:http400:error128');
end
- function test13_list_by_device_3_pages_extension_all(this)
- result = this.onc.getListByLocation(this.F_LOC_RETURN2, true);
- verifyLength(this, result.files, 2);
- verify_has_field(this, result.files(1), 'uncompressedFileSize')
+ function testDownloadValidParams(this)
+ this.updateOncOutPath('output/testDownloadValidParams');
+ filename = 'BPR-Folger-59_20191123T000000.000Z.txt';
+ this.onc.getFile(filename);
+ assertTrue(this, exist([this.onc.outPath '/' filename], 'file') == 2);
+ verifyError(this, @() this.onc.getFile(filename), 'onc:FileExistsError');
+ this.onc.getFile(filename, 'overwrite', true);
+ verify_files_in_path(this, this.onc.outPath, 1);
end
- function test14_save_file_empty_outpath(this)
- filename = 'NAXYS_HYD_007_20091231T235919.476Z-spect-small.png';
- onc9 = this.prepareOnc('');
- result = onc9.getFile(filename);
- verifyTrue(this, isfile(filename));
- delete(filename); % clean up
+ %% Testing direct download
+
+ function testDirectDownloadValidParamsOnePage(this)
+ this.updateOncOutPath('output/testDirectDownloadValidParamsOnePage');
+ data = this.onc.getListByLocation(this.paramsLocation);
+ result = this.onc.getDirectFiles(this.paramsLocation);
+ verify_files_in_path(this, this.onc.outPath, length(data.files));
+ assertEqual(this, result.stats.fileCount, length(data.files));
end
+ function testDirectDownloadValidParamsMultiplePages(this)
+ this.updateOncOutPath('output/testDirectDownloadValidParamsMultiplePages');
+ result = this.onc.getDirectFiles(this.paramsLocationMultiPages);
+ verify_files_in_path(this, this.onc.outPath, this.paramsLocationMultiPages.rowLimit);
+ assertEqual(this, result.stats.fileCount, this.paramsLocationMultiPages.rowLimit)
+ end
end
end
\ No newline at end of file
From 39d4a0f4fe9f9cd53374c44f4eb29e8d4b9f2686 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Mon, 25 Mar 2024 15:21:07 -0700
Subject: [PATCH 05/35] issue7: Modify tests so that can be run with matlab's
unittest library functions, change config from global variable to mat, abd
update instructions
---
onc/tests/README.md | 15 +++---
onc/tests/globals.m | 47 ++++++++++---------
onc/tests/load_config.m | 12 +++++
onc/tests/runAllTests.m | 10 ----
onc/tests/runTestCase.m | 10 ----
onc/tests/runTestSuite.m | 36 --------------
onc/tests/suites/Test01_Locations.m | 2 +-
onc/tests/suites/Test02_Deployments.m | 2 +-
onc/tests/suites/Test03_DeviceCategories.m | 2 +-
onc/tests/suites/Test04_Devices.m | 2 +-
onc/tests/suites/Test05_Properties.m | 2 +-
.../suites/Test06_DataProductDiscovery.m | 2 +-
onc/tests/suites/Test07_DataProductDelivery.m | 2 +-
onc/tests/suites/Test08_RealTime.m | 2 +-
onc/tests/suites/Test09_ArchiveFiles.m | 2 +-
15 files changed, 55 insertions(+), 93 deletions(-)
create mode 100644 onc/tests/load_config.m
delete mode 100644 onc/tests/runAllTests.m
delete mode 100644 onc/tests/runTestCase.m
delete mode 100644 onc/tests/runTestSuite.m
diff --git a/onc/tests/README.md b/onc/tests/README.md
index 4b88b7c..443550f 100644
--- a/onc/tests/README.md
+++ b/onc/tests/README.md
@@ -24,20 +24,23 @@ Although each test suite inherits from matlab.unittest.TestCase, each is written
2. Use MATLAB's command window to run all tests, using the commands described below:
*Running all tests:*
+ in folder api-matlab-client, run command:
+ runtests('onc/tests/suites', 'IncludeSubfolders', true);
- runAllTests
-
+(if it doesn't display test results table at the end, all tests pass. You can also remove semicolon to see test results)
+
*Running a test suite:*
- runTestSuite
+ runtests()
i.e.:
- runTestSuite 1
+ runtests("Test01_Locations");
*Running a test case:*
+
+ runtests(/)
- runTestCase
i.e.:
- runTestCase Test01_Locations testGetAllLocations
+ runtests("Test01_Locations/testInvalidTimeRangeGreaterStartTime")
**DEVELOPING TESTS**
diff --git a/onc/tests/globals.m b/onc/tests/globals.m
index cbf416d..9aedd11 100644
--- a/onc/tests/globals.m
+++ b/onc/tests/globals.m
@@ -1,25 +1,28 @@
-import matlab.unittest.TestSuite;
-clc;
+function config = globals()
+ clc;
+
+ addpath(genpath(''));
+ addpath(genpath('util'));
+ addpath(genpath('suites'));
+ addpath(genpath('../'));
+
+ % Change the current folder to the folder of this m-file.
-addpath(genpath(''));
-addpath(genpath('util'));
-addpath(genpath('suites'));
-addpath(genpath('../'));
+ % grab token from "TOKEN" file
+ f = fopen('TOKEN','r');
+ if f > 0
+ line = fgetl(f);
+ fclose(f);
+ else
+ line = getenv('TOKEN_STRING');
+ end
-% Change the current folder to the folder of this m-file.
-if(~isdeployed)
- cd(fileparts(which(mfilename)));
-end
-
-% configuration used by default unless test cases say otherwise
-global config;
-config.production = true;
-config.showInfo = false;
-config.outPath = 'output';
-config.timeout = 60;
+ % Set and save config
+ config.production = true;
+ config.showInfo = false;
+ config.outPath = 'output';
+ config.timeout = 60;
+ config.token = strtrim(line);
-% grab token from "TOKEN" file
-f = fopen('TOKEN','r');
-line = fgetl(f);
-fclose(f);
-config.token = strtrim(line);
+ save('config.mat', 'config')
+end
diff --git a/onc/tests/load_config.m b/onc/tests/load_config.m
new file mode 100644
index 0000000..4cb62ce
--- /dev/null
+++ b/onc/tests/load_config.m
@@ -0,0 +1,12 @@
+function config = load_config()
+ % Change the current folder to the folder of this m-file.
+ if(~isdeployed)
+ cd(fileparts(which(mfilename)));
+ end
+
+ % If exist config load directly else run globals to get config
+ if exist('config.mat', 'file')
+ load('config.mat', 'config');
+ else
+ config = globals();
+ end
diff --git a/onc/tests/runAllTests.m b/onc/tests/runAllTests.m
deleted file mode 100644
index 229497b..0000000
--- a/onc/tests/runAllTests.m
+++ /dev/null
@@ -1,10 +0,0 @@
-function runAllTests()
- import matlab.unittest.TestSuite;
- run('globals.m');
-
- % Runs all tests suites and displays results
- suiteFolder = TestSuite.fromFolder('./suites');
- testResults = run(suiteFolder);
- rt = table(testResults);
- disp(rt);
-end
\ No newline at end of file
diff --git a/onc/tests/runTestCase.m b/onc/tests/runTestCase.m
deleted file mode 100644
index 2a309f6..0000000
--- a/onc/tests/runTestCase.m
+++ /dev/null
@@ -1,10 +0,0 @@
-function runTestCase(suiteName, caseName)
- import matlab.unittest.TestSuite;
- run('globals.m');
-
- % Run a single test case
- obj = eval(suiteName);
- testResults = run(obj, caseName); % Method name
- rt = table(testResults);
- disp(rt);
-end
\ No newline at end of file
diff --git a/onc/tests/runTestSuite.m b/onc/tests/runTestSuite.m
deleted file mode 100644
index 55a3f31..0000000
--- a/onc/tests/runTestSuite.m
+++ /dev/null
@@ -1,36 +0,0 @@
-function runTestSuite(suite)
- import matlab.unittest.TestSuite;
- run('globals.m');
-
- % Run a single test suite
- testResults = [];
- suite = str2num(suite);
-
- switch suite
- case 1
- testResults = run(Test01_Locations);
- case 2
- testResults = run(Test02_Deployments);
- case 3
- testResults = run(Test03_DeviceCategories);
- case 4
- testResults = run(Test04_Devices);
- case 5
- testResults = run(Test05_Properties);
- case 6
- testResults = run(Test06_DataProductDiscovery);
- case 7
- testResults = run(Test07_DataProductDelivery);
- case 8
- testResults = run(Test08_RealTime);
- case 9
- testResults = run(Test09_ArchiveFiles);
- otherwise
- throw(MException('onc:BadSuiteNumber', sprintf('Wrong suite number: %d', suite)));
-
- end
-
- rt = table(testResults);
- fprintf('\n');
- disp(rt);
-end
\ No newline at end of file
diff --git a/onc/tests/suites/Test01_Locations.m b/onc/tests/suites/Test01_Locations.m
index 29cad1b..4d6e2dd 100644
--- a/onc/tests/suites/Test01_Locations.m
+++ b/onc/tests/suites/Test01_Locations.m
@@ -8,7 +8,7 @@
methods (TestClassSetup)
function classSetup(this)
- config = globals();
+ config = load_config();
this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
diff --git a/onc/tests/suites/Test02_Deployments.m b/onc/tests/suites/Test02_Deployments.m
index edaa2fd..0089967 100644
--- a/onc/tests/suites/Test02_Deployments.m
+++ b/onc/tests/suites/Test02_Deployments.m
@@ -8,7 +8,7 @@
methods (TestClassSetup)
function classSetup(this)
- config = globals();
+ config = load_config();
this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
diff --git a/onc/tests/suites/Test03_DeviceCategories.m b/onc/tests/suites/Test03_DeviceCategories.m
index 9d4bd56..3575ad6 100644
--- a/onc/tests/suites/Test03_DeviceCategories.m
+++ b/onc/tests/suites/Test03_DeviceCategories.m
@@ -8,7 +8,7 @@
methods (TestClassSetup)
function classSetup(this)
- config = globals();
+ config = load_config();
this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
diff --git a/onc/tests/suites/Test04_Devices.m b/onc/tests/suites/Test04_Devices.m
index 66b3bbb..95ad925 100644
--- a/onc/tests/suites/Test04_Devices.m
+++ b/onc/tests/suites/Test04_Devices.m
@@ -9,7 +9,7 @@
methods (TestClassSetup)
function classSetup(this)
- config = globals();
+ config = load_config();
this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
diff --git a/onc/tests/suites/Test05_Properties.m b/onc/tests/suites/Test05_Properties.m
index d4346b7..c1803e9 100644
--- a/onc/tests/suites/Test05_Properties.m
+++ b/onc/tests/suites/Test05_Properties.m
@@ -8,7 +8,7 @@
methods (TestClassSetup)
function classSetup(this)
- config = globals();
+ config = load_config();
this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
diff --git a/onc/tests/suites/Test06_DataProductDiscovery.m b/onc/tests/suites/Test06_DataProductDiscovery.m
index 111ab3e..a29a628 100644
--- a/onc/tests/suites/Test06_DataProductDiscovery.m
+++ b/onc/tests/suites/Test06_DataProductDiscovery.m
@@ -8,7 +8,7 @@
methods (TestClassSetup)
function classSetup(this)
- config = globals();
+ config = load_config();
this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
diff --git a/onc/tests/suites/Test07_DataProductDelivery.m b/onc/tests/suites/Test07_DataProductDelivery.m
index db393ae..ff5073a 100644
--- a/onc/tests/suites/Test07_DataProductDelivery.m
+++ b/onc/tests/suites/Test07_DataProductDelivery.m
@@ -30,7 +30,7 @@
methods (TestClassSetup)
function classSetup(this)
- config = globals();
+ config = load_config();
this.outPath = 'output';
this.onc = Onc(config.token, config.production, config.showInfo, this.outPath, config.timeout);
this.maxRetries = 100;
diff --git a/onc/tests/suites/Test08_RealTime.m b/onc/tests/suites/Test08_RealTime.m
index 982d675..1ac730c 100644
--- a/onc/tests/suites/Test08_RealTime.m
+++ b/onc/tests/suites/Test08_RealTime.m
@@ -45,7 +45,7 @@
methods (TestClassSetup)
function classSetup(this)
- config = globals();
+ config = load_config();
this.onc = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
end
end
diff --git a/onc/tests/suites/Test09_ArchiveFiles.m b/onc/tests/suites/Test09_ArchiveFiles.m
index 2a30a94..bedef55 100644
--- a/onc/tests/suites/Test09_ArchiveFiles.m
+++ b/onc/tests/suites/Test09_ArchiveFiles.m
@@ -37,7 +37,7 @@
methods (TestClassSetup)
function classSetup(this)
- config = globals();
+ config = load_config();
this.onc = Onc(config.token, config.production, config.showInfo, this.outPath, config.timeout);
end
From 2bbede4bf32f39d21284b59586873d6ecafd1148 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Mon, 25 Mar 2024 15:28:26 -0700
Subject: [PATCH 06/35] issue7: Remove, modify and add util files for testing
as needed
---
onc/tests/util/TestDiscovery.m | 48 -------------------
onc/tests/util/verify_cell_array_length.m | 17 -------
onc/tests/util/verify_downloaded_file.m | 6 ---
onc/tests/util/verify_error_response.m | 8 ----
onc/tests/util/verify_field_value.m | 7 +--
onc/tests/util/verify_field_value_not_empty.m | 7 ---
onc/tests/util/verify_field_value_type.m | 21 ++++++++
onc/tests/util/verify_files_in_path.m | 9 ++--
onc/tests/util/verify_has_field.m | 6 ---
onc/tests/util/verify_min_length.m | 4 --
onc/tests/util/verify_no_next_page.m | 7 ---
11 files changed, 31 insertions(+), 109 deletions(-)
delete mode 100644 onc/tests/util/TestDiscovery.m
delete mode 100644 onc/tests/util/verify_cell_array_length.m
delete mode 100644 onc/tests/util/verify_downloaded_file.m
delete mode 100644 onc/tests/util/verify_error_response.m
delete mode 100644 onc/tests/util/verify_field_value_not_empty.m
create mode 100644 onc/tests/util/verify_field_value_type.m
delete mode 100644 onc/tests/util/verify_has_field.m
delete mode 100644 onc/tests/util/verify_min_length.m
delete mode 100644 onc/tests/util/verify_no_next_page.m
diff --git a/onc/tests/util/TestDiscovery.m b/onc/tests/util/TestDiscovery.m
deleted file mode 100644
index 385035e..0000000
--- a/onc/tests/util/TestDiscovery.m
+++ /dev/null
@@ -1,48 +0,0 @@
-classdef TestDiscovery < matlab.unittest.TestCase
-% Discovery services generic test model
-% Contains common utility functions for discovery test cases
- %% Private Properties
- properties (SetAccess = private)
- o % ONC object
- expectedFields % Map of string array with field names expected in a locations response
- end
-
- %% Public Methods
- methods
-
- function obj = TestDiscovery()
- % Constructor
- global config;
- obj.o = Onc(config.token, config.production, config.showInfo, config.outPath, config.timeout);
-
- obj.expectedFields = containers.Map;
- end
- end
-
- %% Protected Methods
- methods (Access = protected)
-
- function verify_min_length(this, data, min)
- % Verifies that data has a min length
- verifyGreaterThanOrEqual(this, length(data), min);
- end
-
- function result = testSingleFilter(this, methodName, filters, minRows, maxRows)
- % Generic single filter test with validation for expected
- % filters and min/max result rows
- % methodName {String} Name of the method to invoke on ONC object
- % filters {CellArray} Cell array of strings, with key, value pairs
- % minRows {Number} Min number of rows expected or NaN for no limit
- % maxRows {Number} Max number of rows expected or NaN for no limit
- result = this.o.(methodName)(filters);
- fields = this.expectedFields(methodName);
- verify_fields(this, result, fields);
- if ~isnan(minRows)
- verifyGreaterThanOrEqual(this, length(result), minRows);
- end
- if ~isnan(maxRows)
- verifyLessThanOrEqual(this, length(result), maxRows);
- end
- end
- end
-end
\ No newline at end of file
diff --git a/onc/tests/util/verify_cell_array_length.m b/onc/tests/util/verify_cell_array_length.m
deleted file mode 100644
index 1ec177a..0000000
--- a/onc/tests/util/verify_cell_array_length.m
+++ /dev/null
@@ -1,17 +0,0 @@
-function validLength = verify_cell_array_length(testCase, data, minRows, maxRows)
- % verifies that a 1D cell array meets min and max dimensions
- % throws verification soft failures in obj if rules are not met
- % if min or max are NaN, they are not evaluated
- % returns number of rows
- validLength = true;
- [~, l] = size(data);
-
- if ~isnan(minRows)
- if (l < minRows), validLength = false; end
- verifyGreaterThanOrEqual(testCase, l, minRows);
- end
- if ~isnan(maxRows)
- if (l > maxRows), validLength = false; end
- verifyLessThanOrEqual(testCase, l, maxRows);
- end
-end
diff --git a/onc/tests/util/verify_downloaded_file.m b/onc/tests/util/verify_downloaded_file.m
deleted file mode 100644
index 8c376d8..0000000
--- a/onc/tests/util/verify_downloaded_file.m
+++ /dev/null
@@ -1,6 +0,0 @@
-function fileExists = verify_downloaded_file(testCase, path)
-%verify_downloaded_file Verifies that the file in path exists
-% path {string} is an absolute file path
- fileExists = (exist(path, 'file') == 2);
- verifyTrue(testCase, fileExists);
-end
diff --git a/onc/tests/util/verify_error_response.m b/onc/tests/util/verify_error_response.m
deleted file mode 100644
index d0e38b6..0000000
--- a/onc/tests/util/verify_error_response.m
+++ /dev/null
@@ -1,8 +0,0 @@
-function verify_error_response(obj, response)
- verifyTrue(obj, isfield(response, "errors"))
- if isfield(response, "errors")
- names = fieldnames(response.errors(1));
- verifyTrue(obj, ismember("errorCode", names))
- verifyTrue(obj, ismember("errorMessage", names))
- end
-end
\ No newline at end of file
diff --git a/onc/tests/util/verify_field_value.m b/onc/tests/util/verify_field_value.m
index 074475f..bebf169 100644
--- a/onc/tests/util/verify_field_value.m
+++ b/onc/tests/util/verify_field_value.m
@@ -2,8 +2,9 @@ function verify_field_value(testCase, data, fieldName, expectedValue)
%VERIFYFIELDVALUE Verifies that the fieldName has the expected value only if it exists
% Does nothing if fieldName doesnt exist
% If value is not as expected, throws a soft failure on testCase
- if (isfield(data, fieldName))
- verifyEqual(testCase, data.(fieldName), expectedValue);
- end
+
+% First test if field exist then test value
+ verifyTrue(testCase, isfield(data, fieldName));
+ verifyEqual(testCase, data.(fieldName), expectedValue);
end
diff --git a/onc/tests/util/verify_field_value_not_empty.m b/onc/tests/util/verify_field_value_not_empty.m
deleted file mode 100644
index b403298..0000000
--- a/onc/tests/util/verify_field_value_not_empty.m
+++ /dev/null
@@ -1,7 +0,0 @@
-function verify_field_value_not_empty(testCase, data, fieldName)
-%verify_field_value_not_empty Throws a verification failure if data.fieldName exists but is an empty string
-% Doesn't do anything if the field doesn't exist
- if isfield(data, fieldName)
- verifyNotEqual(testCase, data.(fieldName), '');
- end
-end
diff --git a/onc/tests/util/verify_field_value_type.m b/onc/tests/util/verify_field_value_type.m
new file mode 100644
index 0000000..05f59d3
--- /dev/null
+++ b/onc/tests/util/verify_field_value_type.m
@@ -0,0 +1,21 @@
+function verify_field_value_type(testCase, dataStruct, expectedStruct)
+ % Verify that dataStruct contains correct fields and the corresponding values
+ % are of the correct variable type
+ %
+ % Example input:
+ % dataStruct = struct('fieldName1', data1, 'fieldName2', data2, ...);
+ % expectedStruct = struct('fieldName1', 'int', 'fieldName2', 'char', ...);
+
+ expectedFieldNames = fieldnames(expectedStruct);
+ for i = 1 : length(expectedFieldNames)
+ currField = expectedFieldNames{i};
+ verifyTrue(testCase, isfield(dataStruct, currField));
+ if strcmp(expectedStruct.(currField), 'int')
+ verifyTrue(testCase, isa(dataStruct.(currField), 'double'));
+ % using floor to test if this is an integer
+ verifyTrue(testCase, floor(dataStruct.(currField)) == dataStruct.(currField));
+ else
+ verifyTrue(testCase, isa(dataStruct.(currField), expectedStruct.(currField)));
+ end
+ end
+end
\ No newline at end of file
diff --git a/onc/tests/util/verify_files_in_path.m b/onc/tests/util/verify_files_in_path.m
index e245c76..5946a45 100644
--- a/onc/tests/util/verify_files_in_path.m
+++ b/onc/tests/util/verify_files_in_path.m
@@ -1,5 +1,5 @@
% verifies that path has exactly n files
-function verify_files_in_path(obj, path, n)
+function verify_files_in_path(obj, path, n, msg)
files = dir(sprintf('%s/*.*', path));
% remove . and ..
@@ -9,6 +9,9 @@ function verify_files_in_path(obj, path, n)
count = count + 1;
end
end
-
- verifyEqual(obj, count, n);
+ if exist('msg', 'var')
+ verifyEqual(obj, count, n, msg);
+ else
+ verifyEqual(obj, count, n);
+ end
end
\ No newline at end of file
diff --git a/onc/tests/util/verify_has_field.m b/onc/tests/util/verify_has_field.m
deleted file mode 100644
index 714578c..0000000
--- a/onc/tests/util/verify_has_field.m
+++ /dev/null
@@ -1,6 +0,0 @@
-function verify_has_field(testCase, data, fieldName)
-%VERIFYFIELDVALUE Verifies that the data is a structire with field fieldName
-% Throws a soft failure if the field is not a member
- verifyTrue(testCase, isfield(data, fieldName));
-end
-
diff --git a/onc/tests/util/verify_min_length.m b/onc/tests/util/verify_min_length.m
deleted file mode 100644
index 0e51f26..0000000
--- a/onc/tests/util/verify_min_length.m
+++ /dev/null
@@ -1,4 +0,0 @@
-function verify_min_length(data, min)
- % Verifies that data has a min length
- verifyGreaterThanOrEqual(length(data), min);
-end
diff --git a/onc/tests/util/verify_no_next_page.m b/onc/tests/util/verify_no_next_page.m
deleted file mode 100644
index 00c4963..0000000
--- a/onc/tests/util/verify_no_next_page.m
+++ /dev/null
@@ -1,7 +0,0 @@
-function verify_no_next_page(testCase, data)
-%VERIFY_NO_NEXT_PAGE only passes if data is a response where next is null
- verify_has_field(testCase, data, 'next');
- verifyTrue(testCase, isempty(data.next));
-
-end
-
From da4495eac7d210bf03d6cfe9b2c448482d5197e3 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Mon, 25 Mar 2024 15:35:17 -0700
Subject: [PATCH 07/35] issue8: Clean up codes
---
onc/+util/do_request.m | 4 ++--
onc/+util/print_error.m | 1 -
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/onc/+util/do_request.m b/onc/+util/do_request.m
index 0b57dc9..ba02eb3 100644
--- a/onc/+util/do_request.m
+++ b/onc/+util/do_request.m
@@ -25,7 +25,6 @@
% run and time request
if showInfo, fprintf('\nRequesting URL:\n %s\n', fullUrl); end
tic
- %response = send(request, uri, options);
response = request.send(uri,options);
duration = toc;
@@ -52,7 +51,8 @@
otherwise
util.print_error(response, fullUrl);
if status == 400 || status == 401
- throw(util.prepare_exception(status,double(response.Body.Data.errors.errorCode)));
+ errorStruct = response.Body.Data;
+ throw(util.prepare_exception(status, double(errorStruct.errors.errorCode)));
else
throw(util.prepare_exception(status));
end
diff --git a/onc/+util/print_error.m b/onc/+util/print_error.m
index e1507f1..d803508 100644
--- a/onc/+util/print_error.m
+++ b/onc/+util/print_error.m
@@ -39,7 +39,6 @@ function print_error_message(payload)
payload = jsondecode(payload);
end
- %if isfield(payload, 'errors')
for i = 1 : numel(payload.errors)
e = payload.errors(i);
msg = e.errorMessage;
From 2bdca3772c4630c0d3e234c8d28e2f26e3c3aaa9 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Mon, 25 Mar 2024 15:39:53 -0700
Subject: [PATCH 08/35] issue5: Modify documentation
---
onc/+onc/OncDelivery.m | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/onc/+onc/OncDelivery.m b/onc/+onc/OncDelivery.m
index 20ba84f..5e7cc89 100644
--- a/onc/+onc/OncDelivery.m
+++ b/onc/+onc/OncDelivery.m
@@ -98,7 +98,7 @@
% runDataProduct (dpRequestId)
%
% * dpRequestId (int) Request id obtained by requestDataProduct()
- % - waitComplete @TODO
+ % - waitComplete: wait until dp finish when set to true (default)
%
% Returns: (struct) information of the run process
%
From f8543d9d89bb3047c48f139ba7687ccc25421021 Mon Sep 17 00:00:00 2001
From: IslaL <142735981+IslaL@users.noreply.github.com>
Date: Tue, 26 Mar 2024 09:28:33 -0700
Subject: [PATCH 09/35] remove commented line
---
onc/tests/globals.m | 1 -
1 file changed, 1 deletion(-)
diff --git a/onc/tests/globals.m b/onc/tests/globals.m
index 9aedd11..23100a7 100644
--- a/onc/tests/globals.m
+++ b/onc/tests/globals.m
@@ -6,7 +6,6 @@
addpath(genpath('suites'));
addpath(genpath('../'));
- % Change the current folder to the folder of this m-file.
% grab token from "TOKEN" file
f = fopen('TOKEN','r');
From 838ad491b53e295c50ec1663163c66d0ae9abdab Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Tue, 26 Mar 2024 10:11:16 -0700
Subject: [PATCH 10/35] issue4: set up MATLAB workflow
---
.github/workflows/matlab.yml | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
create mode 100644 .github/workflows/matlab.yml
diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml
new file mode 100644
index 0000000..3aed878
--- /dev/null
+++ b/.github/workflows/matlab.yml
@@ -0,0 +1,36 @@
+name: Run MATLAB Tests
+
+on:
+ pull_request:
+ branches: main
+ push:
+ branches: main
+ workflow_dispatch:
+
+jobs:
+ tests:
+ name: MATLAB Test
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ matlab: [ "R2022b"]
+ os: [ubuntu-latest, windows-latest]
+ env:
+ MLM_LICENSE_FILE: ${{ secrets.MATLAB_LICENSE }}
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@main
+
+ - name: Set up MATLAB
+ uses: matlab-actions/setup-matlab@v2
+ with:
+ release: ${{ matrix.matlab }}
+ cache: true
+
+ - name: Run tests
+ uses: matlab-actions/run-tests@v2
+ with:
+ source-folder: 'onc'
+ env:
+ TOKEN_STRING: ${{ secrets.TOKEN_STRING }}
+
From 179f8d6c8bc97addb3498415d2041f7ffaf7d582 Mon Sep 17 00:00:00 2001
From: IslaL <142735981+IslaL@users.noreply.github.com>
Date: Tue, 26 Mar 2024 13:22:52 -0700
Subject: [PATCH 11/35] Update variable name line to token, TOKEN_STRING to
TOKEN
---
onc/tests/globals.m | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/onc/tests/globals.m b/onc/tests/globals.m
index 23100a7..baffae5 100644
--- a/onc/tests/globals.m
+++ b/onc/tests/globals.m
@@ -10,10 +10,10 @@
% grab token from "TOKEN" file
f = fopen('TOKEN','r');
if f > 0
- line = fgetl(f);
+ token = fgetl(f);
fclose(f);
else
- line = getenv('TOKEN_STRING');
+ token = getenv('TOKEN');
end
% Set and save config
@@ -21,7 +21,7 @@
config.showInfo = false;
config.outPath = 'output';
config.timeout = 60;
- config.token = strtrim(line);
+ config.token = strtrim(token);
save('config.mat', 'config')
end
From ac7fd85bb33d93be9b7016edea9ffd54be7c7e11 Mon Sep 17 00:00:00 2001
From: IslaL <142735981+IslaL@users.noreply.github.com>
Date: Tue, 26 Mar 2024 13:24:00 -0700
Subject: [PATCH 12/35] Update variable name TOKEN_STRING to TOKEN
---
.github/workflows/matlab.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml
index 3aed878..f8ab354 100644
--- a/.github/workflows/matlab.yml
+++ b/.github/workflows/matlab.yml
@@ -13,7 +13,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
- matlab: [ "R2022b"]
+ matlab: ["R2022b"]
os: [ubuntu-latest, windows-latest]
env:
MLM_LICENSE_FILE: ${{ secrets.MATLAB_LICENSE }}
@@ -32,5 +32,5 @@ jobs:
with:
source-folder: 'onc'
env:
- TOKEN_STRING: ${{ secrets.TOKEN_STRING }}
+ TOKEN: ${{ secrets.TOKEN }}
From 0294950fdcb67ff2f28c0858b063569e90ad1486 Mon Sep 17 00:00:00 2001
From: IslaL <142735981+IslaL@users.noreply.github.com>
Date: Tue, 26 Mar 2024 13:28:28 -0700
Subject: [PATCH 13/35] Update globals.m to make sure it saves config file in
the correct folder
---
onc/tests/globals.m | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/onc/tests/globals.m b/onc/tests/globals.m
index baffae5..9293dbe 100644
--- a/onc/tests/globals.m
+++ b/onc/tests/globals.m
@@ -1,12 +1,16 @@
function config = globals()
clc;
-
+
addpath(genpath(''));
addpath(genpath('util'));
addpath(genpath('suites'));
addpath(genpath('../'));
-
+ % Change the current folder to the folder of this m-file.
+ if(~isdeployed)
+ cd(fileparts(which(mfilename)));
+ end
+
% grab token from "TOKEN" file
f = fopen('TOKEN','r');
if f > 0
From 7bf5cd7b72bcb93fe81d3136d9d8149a52213823 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Thu, 28 Mar 2024 16:06:52 -0700
Subject: [PATCH 14/35] issue-8: fixed minor bugs found when testing
---
onc/+onc/DataProductFile.m | 50 +++++++++++++++++++++++---------------
onc/+onc/MultiPage.m | 8 +++---
onc/+onc/OncArchive.m | 12 ++++-----
onc/+onc/OncDelivery.m | 11 ++-------
onc/+util/param.m | 7 +++---
onc/+util/save_as_file.m | 5 ++--
6 files changed, 48 insertions(+), 45 deletions(-)
diff --git a/onc/+onc/DataProductFile.m b/onc/+onc/DataProductFile.m
index bc918e5..9ff3024 100644
--- a/onc/+onc/DataProductFile.m
+++ b/onc/+onc/DataProductFile.m
@@ -57,22 +57,19 @@
% * maxRetries: (int) As provided by the Onc class
%
% Returns: (integer) The final response's HTTP status code
+
log = onc.DPLogger();
-
this.status = 202;
- saveResult = 0;
+ request = matlab.net.http.RequestMessage;
+ uri = matlab.net.URI(this.baseUrl);
+ uri.Query = matlab.net.QueryParameter(this.filters);
+ fullUrl = char(uri);
+ options = matlab.net.http.HTTPOptions('ConnectTimeout', timeout, 'ConvertResponse', false);
while this.status == 202
- % prepare HTTP request
- request = matlab.net.http.RequestMessage;
- uri = matlab.net.URI(this.baseUrl);
- uri.Query = matlab.net.QueryParameter(this.filters);
- fullUrl = char(uri);
- options = matlab.net.http.HTTPOptions('ConnectTimeout', timeout, 'ConvertResponse', false);
-
% run and time request
if this.showInfo, log.printLine(sprintf('Requesting URL:\n %s', fullUrl)); end
tic
- response = send(request, uri, options);
+ response = request.send(uri,options);
duration = toc;
this.downloadUrl = fullUrl;
@@ -89,6 +86,7 @@
s = this.status;
if s == 200
% File downloaded, get filename from header and save
+
this.downloaded = true;
filename = this.extractNameFromHeader(response);
this.fileName = filename;
@@ -101,31 +99,43 @@
this.fileSize = strlength(response.Body.Data);
end
%save_as_file doesn't work properly! Use urlwrite instead
- [~, saveResult] = urlwrite(uri.EncodedURI,fullfile(outPath, filename));
- %saveResult = util.save_as_file(response, outPath, filename, overwrite);
+ %[~, saveResult] = urlwrite(uri.EncodedURI,fullfile(outPath, filename));
+ % neither urlwrite nor save_as_file works properly, use websave
+
+
+ savefilename = fullfile(outPath, filename);
+ % Create folder if not exist
+ if ~exist(outPath,'dir')
+ mkdir(outPath)
+ end
+ % saveResult -2: File exists and set to not overwrite 0: done
+ if ~overwrite && exist(savefilename,'file') == 2
+ saveResult = -2;
+ else
+ websave(savefilename,uri.EncodedURI);
+ saveResult = 0;
+ end
this.downloadingTime = round(duration, 3);
% log status
if saveResult == 0
- log.printLine(sprintf('Downloaded "%s"', this.fileName));
+ log.printLine(sprintf('Downloaded "%s"\n', this.fileName));
elseif saveResult == -2
- log.printLine(sprintf('Skipping "%s": File already exists', this.fileName));
+ log.printLine(sprintf('Skipping "%s": File already exists\n', this.fileName));
this.status = 777;
end
elseif s == 202
% Still processing, wait and retry
log.printResponse(jsondecode(response.Body.Data));
+ %log.printResponse(response.Body.Data);
pause(pollPeriod);
elseif s == 204
% No data found
- log.printLine('No data found.');
+ log.printLine('No data found.\n');
elseif s == 400
% API Error
util.print_error(response, fullUrl);
- err = struct( ...
- 'message', sprintf('The request failed with HTTP status %d\n', this.status), ...
- 'identifier', 'DataProductFile:HTTP400');
- error(err);
+ throw(util.prepare_exception(s, double(jsondecode(response.Body.Data).errors.errorCode)));
elseif s == 404
% Index too high, no more files to download
else
@@ -139,7 +149,7 @@
endStatus = this.status;
end
- function filename = extractNameFromHeader(this, response)
+ function filename = extractNameFromHeader(~,response)
%% Return the file name extracted from the HTTP response
%
% * response (object) The successful (200) httr response obtained from a download request
diff --git a/onc/+onc/MultiPage.m b/onc/+onc/MultiPage.m
index 1c923c0..3fa8865 100644
--- a/onc/+onc/MultiPage.m
+++ b/onc/+onc/MultiPage.m
@@ -208,13 +208,13 @@
end
elseif isArchive
row0 = response.files(1);
+ rowEnd = response.files(end);
-
- if isa(row0, 'char')
+ if isa(row0, 'char') || iscell(row0)
% extract the date from the filename
regExp = '\\d{8}T\\d{6}d\\.\\d{3}Z';
- nameFirst = response.files(1);
- nameLast = response.files(end);
+ nameFirst = char(row0);
+ nameLast = char(rowEnd);
mFirst = regexp(nameFirst, regExp, 'once', 'match');
mLast = regexp(nameLast, regExp, 'once', 'match');
if isempty(mFirst) || isempty(mLast)
diff --git a/onc/+onc/OncArchive.m b/onc/+onc/OncArchive.m
index b5cba9e..bafe497 100644
--- a/onc/+onc/OncArchive.m
+++ b/onc/+onc/OncArchive.m
@@ -51,7 +51,9 @@
% Returns: (struct) Information on the download result
%
% Documentation: https://wiki.oceannetworks.ca/display/CLmatlab/Archive+file+download+methods
-
+ if ~exist('filename', 'var')
+ filename = '';
+ end
[overwrite, showMsg] = util.param(varargin, 'overwrite', false, 'showMsg', true);
url = this.serviceUrl('archivefiles');
@@ -61,10 +63,11 @@
[response, info] = ...
util.do_request(url, filters, 'timeout', this.timeout, 'showInfo', this.showInfo, ...
- 'rawResponse', true, 'showProgress', true);
+ 'showProgress', true);
if not(info.status == 200)
- fileInfo = jsondecode(response);
+ %fileInfo = jsondecode(response);
+ fileInfo = response;
return;
end
@@ -76,9 +79,6 @@
if saveStatus == 0
txtStatus = 'completed';
if showMsg, fprintf(' File was downloaded to "%s"\n', filename); end
- elseif saveStatus == -2
- if showMsg, fprintf(' File was skipped (already exists).\n'); end
- txtStatus = 'skipped';
end
fullUrl = this.getDownloadUrl(filename);
diff --git a/onc/+onc/OncDelivery.m b/onc/+onc/OncDelivery.m
index 5e7cc89..7a0b296 100644
--- a/onc/+onc/OncDelivery.m
+++ b/onc/+onc/OncDelivery.m
@@ -33,18 +33,11 @@
fileList = [];
% Request the product
- [rqData, status] = this.requestDataProduct(filters);
- if util.is_failed_response(rqData, status)
- r = rqData;
- return
- end
+ [rqData, ~] = this.requestDataProduct(filters);
% Run the product request
[runData, status] = this.runDataProduct(rqData.dpRequestId);
- if util.is_failed_response(runData, status)
- r = runData;
- return;
- end
+
if downloadResultsOnly
% Only run and return links
diff --git a/onc/+util/param.m b/onc/+util/param.m
index 72c56b8..b4cbe90 100644
--- a/onc/+util/param.m
+++ b/onc/+util/param.m
@@ -15,13 +15,14 @@
addOptional(p, name, defValue);
end
- if n == 2
- % Handle single element separately to avoid problem with struct param
+ if n == 2 && isstruct(varargin{2})
+ % Handle struct param seperately
name = names(1);
if length(args) == 1
+ % one struct given
varargout{1} = args{1};
else
- varargout{1} = varargin{2};
+ varargout{1} = varargin{2};
end
else
% parse using inputParser
diff --git a/onc/+util/save_as_file.m b/onc/+util/save_as_file.m
index 456c730..2a6274a 100644
--- a/onc/+util/save_as_file.m
+++ b/onc/+util/save_as_file.m
@@ -32,7 +32,7 @@
end
if f ~= -1
- fwrite(f, response.Body.Data);
+ fwrite(f, response);
else
endCode = -1;
return;
@@ -44,8 +44,7 @@
return;
end
else
- endCode = -2;
- return;
+ throw(MException('onc:FileExistsError', 'Data product file exists in destination but overwrite is set to false'));
end
endCode = 0;
From 94cf7137df2fcd4ab4851c6452f2c3d8ffd0e3c1 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Mon, 8 Apr 2024 16:37:36 -0700
Subject: [PATCH 15/35] issue-16: fix download bug in data product delivery
---
onc/+onc/DataProductFile.m | 56 ++++++++++++++++++--------------
onc/+util/extractFileExtension.m | 13 ++++++++
onc/+util/save_as_file.m | 39 ++++++++++++++--------
3 files changed, 69 insertions(+), 39 deletions(-)
create mode 100644 onc/+util/extractFileExtension.m
diff --git a/onc/+onc/DataProductFile.m b/onc/+onc/DataProductFile.m
index bc918e5..1966d1f 100644
--- a/onc/+onc/DataProductFile.m
+++ b/onc/+onc/DataProductFile.m
@@ -57,22 +57,19 @@
% * maxRetries: (int) As provided by the Onc class
%
% Returns: (integer) The final response's HTTP status code
+
log = onc.DPLogger();
-
this.status = 202;
- saveResult = 0;
+ request = matlab.net.http.RequestMessage;
+ uri = matlab.net.URI(this.baseUrl);
+ uri.Query = matlab.net.QueryParameter(this.filters);
+ fullUrl = char(uri);
+ options = matlab.net.http.HTTPOptions('ConnectTimeout', timeout);
while this.status == 202
- % prepare HTTP request
- request = matlab.net.http.RequestMessage;
- uri = matlab.net.URI(this.baseUrl);
- uri.Query = matlab.net.QueryParameter(this.filters);
- fullUrl = char(uri);
- options = matlab.net.http.HTTPOptions('ConnectTimeout', timeout, 'ConvertResponse', false);
-
% run and time request
if this.showInfo, log.printLine(sprintf('Requesting URL:\n %s', fullUrl)); end
tic
- response = send(request, uri, options);
+ response = request.send(uri,options);
duration = toc;
this.downloadUrl = fullUrl;
@@ -89,6 +86,7 @@
s = this.status;
if s == 200
% File downloaded, get filename from header and save
+
this.downloaded = true;
filename = this.extractNameFromHeader(response);
this.fileName = filename;
@@ -98,34 +96,42 @@
if length(lengthData) == 1
this.fileSize = str2double(lengthData.Value);
else
- this.fileSize = strlength(response.Body.Data);
+ ext = util.extractFileExtension(filename);
+ if strcmp(ext, 'xml')
+ this.fileSize = length(xmlwrite(response.Body.Data));
+ else
+ this.fileSize = strlength(response.Body.Data);
+ end
+ end
+ try
+ saveResult = util.save_as_file(response.Body.Data, outPath, filename, 'overwrite', overwrite);
+ catch ME
+ if strcmp(ME.identifier, 'onc:FileExistsError')
+ log.printLine(sprintf('Skipping "%s": File already exists\n', this.fileName));
+ this.status = 777;
+ saveResult = -2;
+ this.downloaded = false;
+ else
+ rethrow(ME);
+ end
end
- %save_as_file doesn't work properly! Use urlwrite instead
- [~, saveResult] = urlwrite(uri.EncodedURI,fullfile(outPath, filename));
- %saveResult = util.save_as_file(response, outPath, filename, overwrite);
this.downloadingTime = round(duration, 3);
% log status
if saveResult == 0
- log.printLine(sprintf('Downloaded "%s"', this.fileName));
- elseif saveResult == -2
- log.printLine(sprintf('Skipping "%s": File already exists', this.fileName));
- this.status = 777;
+ log.printLine(sprintf('Downloaded "%s"\n', this.fileName));
end
elseif s == 202
% Still processing, wait and retry
- log.printResponse(jsondecode(response.Body.Data));
+ log.printResponse(response.Body.Data);
pause(pollPeriod);
elseif s == 204
% No data found
- log.printLine('No data found.');
+ log.printLine('No data found.\n');
elseif s == 400
% API Error
util.print_error(response, fullUrl);
- err = struct( ...
- 'message', sprintf('The request failed with HTTP status %d\n', this.status), ...
- 'identifier', 'DataProductFile:HTTP400');
- error(err);
+ throw(util.prepare_exception(s, double(response.Body.Data.errors.errorCode)));
elseif s == 404
% Index too high, no more files to download
else
@@ -139,7 +145,7 @@
endStatus = this.status;
end
- function filename = extractNameFromHeader(this, response)
+ function filename = extractNameFromHeader(~,response)
%% Return the file name extracted from the HTTP response
%
% * response (object) The successful (200) httr response obtained from a download request
diff --git a/onc/+util/extractFileExtension.m b/onc/+util/extractFileExtension.m
new file mode 100644
index 0000000..a86ff8a
--- /dev/null
+++ b/onc/+util/extractFileExtension.m
@@ -0,0 +1,13 @@
+function ext = extractFileExtension(filename)
+ % get extension from filename (string or char array)
+ % this function is called by save_as_file.m to decide which download method to use
+
+ filename = char(filename);
+ possibleExtensionStartIndex = strfind(filename, '.');
+ if ~isempty(possibleExtensionStartIndex)
+ extensionStartIndex = possibleExtensionStartIndex(end);
+ ext = filename(extensionStartIndex + 1:end);
+ else
+ ext = '';
+ end
+end
\ No newline at end of file
diff --git a/onc/+util/save_as_file.m b/onc/+util/save_as_file.m
index 456c730..801f76d 100644
--- a/onc/+util/save_as_file.m
+++ b/onc/+util/save_as_file.m
@@ -6,7 +6,7 @@
% @param fileName Name of the file to save
% @param overwrite If true will overwrite files with the same name
% @return (numeric) Result code from {0: done, -1: error, -2: fileExists}
-function endCode = save_as_file(response, filePath, fileName, varargin)
+function endCode = save_as_file(dataToWrite, filePath, fileName, varargin)
[overwrite] = util.param(varargin, 'overwrite', false);
fullPath = fileName;
@@ -25,27 +25,38 @@
try
matlabVersion = version('-release');
year = str2double(matlabVersion(1:end-1));
- if year >= 2021
- f = fopen(fullPath, 'w','n','ISO-8859-1');
- else
- f = fopen(fullPath, 'w','n');
- end
-
- if f ~= -1
- fwrite(f, response.Body.Data);
+ ext = util.extractFileExtension(fileName);
+
+ % if result is an image file or .xml file, use other save methods instead of fwrite.
+ if strcmp(ext, 'png') || strcmp(ext, 'jpg')
+ imwrite(dataToWrite, fullPath);
+ elseif strcmp(ext, 'xml')
+ xmlwrite(fullPath, dataToWrite);
else
- endCode = -1;
- return;
+ % open output file
+ if year >= 2021
+ f = fopen(fullPath, 'w','n','ISO-8859-1');
+ else
+ f = fopen(fullPath, 'w','n');
+ end
+
+ % write result if open file successfully
+ if f ~= -1
+ fwrite(f, char(dataToWrite));
+ else
+ endCode = -1;
+ return;
+ end
+ fclose(f);
end
- fclose(f);
catch ex
disp(ex);
endCode = -1;
return;
end
else
- endCode = -2;
- return;
+ % if file exists and overwrite is false, raise exception
+ throw(MException('onc:FileExistsError', 'Data product file exists in destination but overwrite is set to false'));
end
endCode = 0;
From 6c46f9c89af8e5108def79eab49b380e1232df83 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Mon, 8 Apr 2024 16:46:45 -0700
Subject: [PATCH 16/35] issue-4: update workflow to test before merge to
release branches as well and add option to test against QA or PROD
---
.github/workflows/matlab.yml | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml
index f8ab354..35a5e17 100644
--- a/.github/workflows/matlab.yml
+++ b/.github/workflows/matlab.yml
@@ -2,9 +2,13 @@ name: Run MATLAB Tests
on:
pull_request:
- branches: main
+ branches:
+ - 'main'
+ - 'release-staging-*'
push:
- branches: main
+ branches:
+ - 'main'
+ - 'release-staging-*'
workflow_dispatch:
jobs:
@@ -33,4 +37,5 @@ jobs:
source-folder: 'onc'
env:
TOKEN: ${{ secrets.TOKEN }}
+ ONC_ENV: true
From 5af504e5bb5d24fbbbfd8a4e7a8cb4731dad2aba Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Tue, 9 Apr 2024 15:47:00 -0700
Subject: [PATCH 17/35] issue-14: remove commented lines
---
onc/+onc/DataProductFile.m | 5 -----
1 file changed, 5 deletions(-)
diff --git a/onc/+onc/DataProductFile.m b/onc/+onc/DataProductFile.m
index 9ff3024..1f961b7 100644
--- a/onc/+onc/DataProductFile.m
+++ b/onc/+onc/DataProductFile.m
@@ -98,10 +98,6 @@
else
this.fileSize = strlength(response.Body.Data);
end
- %save_as_file doesn't work properly! Use urlwrite instead
- %[~, saveResult] = urlwrite(uri.EncodedURI,fullfile(outPath, filename));
- % neither urlwrite nor save_as_file works properly, use websave
-
savefilename = fullfile(outPath, filename);
% Create folder if not exist
@@ -127,7 +123,6 @@
elseif s == 202
% Still processing, wait and retry
log.printResponse(jsondecode(response.Body.Data));
- %log.printResponse(response.Body.Data);
pause(pollPeriod);
elseif s == 204
% No data found
From 459c14eca4aced6ce1ae826599765a068cc12317 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Thu, 11 Apr 2024 14:35:45 -0700
Subject: [PATCH 18/35] issue-11: Implement restart, cancel, and status
functions and add tests
---
onc/+onc/OncDelivery.m | 79 ++++++++++++++++---
onc/+util/is_failed_response.m | 24 ------
onc/tests/globals.m | 8 +-
onc/tests/suites/Test07_DataProductDelivery.m | 62 ++++++++++++---
onc/tests/suites/Test08_RealTime.m | 25 +++---
5 files changed, 140 insertions(+), 58 deletions(-)
delete mode 100644 onc/+util/is_failed_response.m
diff --git a/onc/+onc/OncDelivery.m b/onc/+onc/OncDelivery.m
index 7a0b296..a24cf2a 100644
--- a/onc/+onc/OncDelivery.m
+++ b/onc/+onc/OncDelivery.m
@@ -76,12 +76,9 @@
fprintf('Requesting data product...\n');
[r, info] = this.doRequest(url, filters);
status = info.status;
- if not(util.is_failed_response(r, status))
- this.estimatePollPeriod(r);
- this.printProductRequest(r);
- else
- throw(util.prepare_exception(status));
- end
+ this.estimatePollPeriod(r);
+ this.printProductRequest(r);
+
end
function [r, status] = runDataProduct(this, dpRequestId, waitComplete)
@@ -106,16 +103,18 @@
% run timed run request
tic
+ cancelUrl = url + "?method=cancel&token="+string(this.token)+"&dpRequestId=" + string(dpRequestId);
+ if waitComplete
+ fprintf('\nTo cancel this data product, visit url:\n %s\n', cancelUrl);
+ else
+ fprintf('\nTo cancel this data product, please execute command ''onc.cancelDataProduct(%d)''\n', dpRequestId)
+ end
flag = 'queued';
while ~strcmp(flag,'complete') && ~strcmp(flag,'cancelled')
[response, info] = this.doRequest(url, filters);
status = info.status;
r.requestCount = r.requestCount + 1;
- % guard against failed request
- if util.is_failed_response(response, status)
- throw(util.prepare_exception(status));
- end
% repeat only if waitComplete
if waitComplete
log.printResponse(response);
@@ -140,6 +139,40 @@
r.runIds = [r.runIds, run.dpRunId];
end
end
+
+ function response = checkDataProduct(this, dpRequestId)
+ %% Check the status of a data product
+ %
+ %
+ % checkDataProduct (dpRequestId)
+ %
+ % * dpRequestId (int) Request id obtained by requestDataProduct()
+ %
+ % Returns: response (struct): status of this data product
+ %
+ url = sprintf('%sapi/dataProductDelivery', this.baseUrl);
+ filters = struct('method', 'status', 'token', this.token, 'dpRequestId', dpRequestId);
+ response = this.doRequest(url, filters);
+ end
+
+ function [response, info] = cancelDataProduct(this, dpRequestId)
+ %% Cancel a running data product
+ %
+ %
+ % cancelDataProduct (dpRequestId)
+ %
+ % * dpRequestId (int) Request id obtained by requestDataProduct()
+ %
+ % Returns: response (struct): cancel process status and message
+ % info (struct): cancel process http code and status
+ %
+ url = sprintf('%sapi/dataProductDelivery', this.baseUrl);
+ filters = struct('method', 'cancel', 'token', this.token, 'dpRequestId', dpRequestId);
+ [response, info] = this.doRequest(url, filters);
+ if isfield(response, 'status') && strcmp(response.status, 'cancelled') && info.status == 200
+ fprintf("Data product with request id %d and run id %d was successfully cancelled\n", dpRequestId, response.dpRunId);
+ end
+ end
function fileData = downloadDataProduct(this, runId, varargin)
%% Download a data product manually with a runId
@@ -169,8 +202,32 @@
fileData = this.downloadProductFiles(runId, metadata, maxRetries, overwrite);
end
end
+
+ function [response, info] = restartDataProduct(this, dpRequestId, waitComplete)
+ %% Restart a cancelled data product
+ %
+ %
+ % restartDataProduct (dpRequestId, waitComplete)
+ %
+ % * dpRequestId (int) Request id obtained by requestDataProduct()
+ % - waitComplete (optional): wait until dp finish when set to true (default)
+ %
+ % Returns: response (struct): restart process status and message
+ % info (struct): restart process http code and status
+ %
+ if ~exist('waitComplete','var'), waitComplete = true; end
+ url = sprintf('%sapi/dataProductDelivery', this.baseUrl);
+ filters = struct('method', 'restart', 'token', this.token, 'dpRequestId', dpRequestId);
+ [response, info] = this.doRequest(url, filters);
+ if isfield(response, 'status') && (strcmp(response.status, 'data product running') || strcmp(response.status, 'queued')) && info.status == 200
+ fprintf("Restarted data product with request id %d and run id %d\n", dpRequestId, response.dpRunId);
+ end
+ if waitComplete
+ [response, info] = this.runDataProduct(dpRequestId, true);
+ end
+ end
end
-
+
methods (Access = private, Hidden = true)
function fileList = downloadProductFiles(this, runId, varargin)
%% Download all data product files for provided run id
diff --git a/onc/+util/is_failed_response.m b/onc/+util/is_failed_response.m
deleted file mode 100644
index 744181c..0000000
--- a/onc/+util/is_failed_response.m
+++ /dev/null
@@ -1,24 +0,0 @@
-function isFailed = is_failed_response(response, status)
- %% Checks if a server response describes a failure
- %
- % * response: (struct) Response as returned by do_request()
- % - status: @TODO
- %
- % Returns: (Logical) true when the response is a failure
- isFailed = false;
-
- % Fail if HTTP status code is not a 2xx
- if exist('status', 'var')
- if status < 200 || status > 226
- isFailed = true;
- return
- end
- end
-
- % Fail if the response is an error description
- if isfield(response, "errors")
- names = fieldnames(response.errors(1));
- isFailed = ismember("errorCode", names) && ismember("errorMessage", names);
- end
-end
-
diff --git a/onc/tests/globals.m b/onc/tests/globals.m
index 9293dbe..daa37ac 100644
--- a/onc/tests/globals.m
+++ b/onc/tests/globals.m
@@ -11,7 +11,7 @@
cd(fileparts(which(mfilename)));
end
- % grab token from "TOKEN" file
+ % grab token from "TOKEN" file or get from env
f = fopen('TOKEN','r');
if f > 0
token = fgetl(f);
@@ -20,8 +20,12 @@
token = getenv('TOKEN');
end
+ % get environment from ONC_ENV or use QA as default
+ config.production = getenv('ONC_ENV');
+ if isempty(config.production)
+ config.production = false;
+ end
% Set and save config
- config.production = true;
config.showInfo = false;
config.outPath = 'output';
config.timeout = 60;
diff --git a/onc/tests/suites/Test07_DataProductDelivery.m b/onc/tests/suites/Test07_DataProductDelivery.m
index ff5073a..6d6f336 100644
--- a/onc/tests/suites/Test07_DataProductDelivery.m
+++ b/onc/tests/suites/Test07_DataProductDelivery.m
@@ -113,10 +113,40 @@ function testValidResultsOnly(this)
end
+
+ %% Testing run method
+ function testInvalidRequestId(this)
+ verifyError(this, @() this.onc.runDataProduct(1234567890), 'onc:http400:error127');
+ end
+
+ %% Testing download method
+ function testInvalidRunId(this)
+ verifyError(this, @() this.onc.downloadDataProduct(1234567890), 'onc:http400:error127');
+ end
+
+ %% Testing cancel method
+ function testCancelWithInvalidRequestId(this)
+ verifyError(this, @() this.onc.cancelDataProduct(1234567890), 'onc:http400:error127');
+ end
+
+ %% Testing status method
+ function testStatusWithInvalidRequestId(this)
+ verifyError(this, @() this.onc.checkDataProduct(1234567890), 'onc:http400:error127');
+ end
+
+ %% Testing restart method
+ function testRestartWithInvalidRequestId(this)
+ verifyError(this, @() this.onc.restartDataProduct(1234567890), 'onc:http400:error127');
+ end
+ %% Integration tests
function testValidManual(this)
this.updateOncOutPath('output/testValidManual');
requestId = this.onc.requestDataProduct(this.Params).dpRequestId;
+ statusBeforeDownload = this.onc.checkDataProduct(requestId);
+
+ assertEqual(this, statusBeforeDownload.searchHdrStatus, 'OPEN');
+
runId = this.onc.runDataProduct(requestId).runIds(1);
data = this.onc.downloadDataProduct(runId);
verifyTrue(this, length(data) == 3, ...
@@ -133,16 +163,30 @@ function testValidManual(this)
verify_files_in_path(this, this.onc.outPath, 3);
verify_field_value_type(this, data(1), this.expectedFields);
+ statusAfterDownload = this.onc.checkDataProduct(requestId);
+ assertEqual(this, statusAfterDownload.searchHdrStatus, 'COMPLETED');
end
-
- %% Testing run method
- function testInvalidRequestId(this)
- verifyError(this, @() this.onc.runDataProduct(1234567890), 'onc:http400:error127');
- end
-
- %% Testing download method
- function testInvalidRunId(this)
- verifyError(this, @() this.onc.downloadDataProduct(1234567890), 'onc:http400:error127');
+
+ function testValidCancelRestart(this)
+ this.updateOncOutPath('output/testValidCancelRestart');
+ requestId = this.onc.requestDataProduct(this.Params).dpRequestId;
+ runId = this.onc.runDataProduct(requestId, false).runIds(1);
+ responseCancel = this.onc.cancelDataProduct(requestId);
+
+ verify_field_value(this, responseCancel, 'dpRunId', runId);
+ verify_field_value(this, responseCancel, 'status', 'cancelled');
+
+ %update MATLAB:nonExistentField error to actual http400 error for this test
+ %after api service fixes the issue that this 400 error does not contain "errors" field
+ assertError(this, @() this.onc.downloadDataProduct(runId), 'MATLAB:nonExistentField')
+
+ runIdAfterRestart = this.onc.restartDataProduct(requestId).runIds(1);
+ assertEqual(this, runIdAfterRestart, runId);
+
+ responseDownload = this.onc.downloadDataProduct(runId);
+ assertEqual(this, length(responseDownload), 3, "The first two are png files, and the third one is the metadata");
+ verify_files_in_path(this, this.onc.outPath, 3);
+ verify_field_value_type(this, responseDownload(1), this.expectedFields);
end
end
diff --git a/onc/tests/suites/Test08_RealTime.m b/onc/tests/suites/Test08_RealTime.m
index 1ac730c..ce86798 100644
--- a/onc/tests/suites/Test08_RealTime.m
+++ b/onc/tests/suites/Test08_RealTime.m
@@ -163,7 +163,7 @@ function testDeviceValidParamsOnePage(this)
result = this.onc.getDirectByDevice(this.paramsDevice);
resultAllPages = this.onc.getDirectByDevice(this.paramsDeviceMultiPages, 'allPages', true);
assertTrue(this, length(result.sensorData(1).data.values) > this.paramsDeviceMultiPages.rowLimit, ...
- 'Test should return at least `rowLimit` rows for each sensor.');
+ 'Test should return at least `rowLimit` rows.');
assertEmpty(this, result.next, 'Test should return only one page.');
assertEqual(this, resultAllPages.sensorData(1).data, result.sensorData(1).data, ...
'Test should concatenate rows for all pages.');
@@ -178,8 +178,7 @@ function testDeviceValidParamsMultiplePages(this)
end
%% Testing rawdata device
- %{
- % waiting for python updates
+
function testRawDeviceInvalidParamValue(this)
filters = this.paramsDevice;
filters.deviceCode = 'XYZ123';
@@ -196,27 +195,29 @@ function testRawDeviceNoData(this)
filters = this.paramsDevice;
filters.dateFrom = '2000-01-01';
filters.dateTo = '2000-01-02';
- result = this.onc.getDirectByDevice(filters);
- assertEmpty(this, result.sensorData);
+ result = this.onc.getDirectRawByDevice(filters);
+ assertEmpty(this, result.data.lineTypes);
+ assertEmpty(this, result.data.readings);
+ assertEmpty(this, result.data.times);
end
function testRawDeviceValidParamsOnePage(this)
- result = this.onc.getDirectByDevice(this.paramsDevice);
- resultAllPages = this.onc.getDirectByDevice(this.paramsDeviceMultiPages, 'allPages', true);
- assertTrue(this, length(result.sensorData(1).data.values) > this.paramsDeviceMultiPages.rowLimit, ...
+ result = this.onc.getDirectRawByDevice(this.paramsDevice);
+ resultAllPages = this.onc.getDirectRawByDevice(this.paramsDeviceMultiPages, 'allPages', true);
+ assertTrue(this, length(result.data.readings) > this.paramsDeviceMultiPages.rowLimit, ...
'Test should return at least `rowLimit` rows for each sensor.');
assertEmpty(this, result.next, 'Test should return only one page.');
- assertEqual(this, resultAllPages.sensorData(1).data, result.sensorData(1).data, ...
+ assertEqual(this, resultAllPages.data, result.data, ...
'Test should concatenate rows for all pages.');
assertEmpty(this, resultAllPages.next, 'Test should return only one page.');
end
function testRawDeviceValidParamsMultiplePages(this)
- result = this.onc.getDirectByDevice(this.paramsDeviceMultiPages);
- assertEqual(this, length(result.sensorData(1).data.values), this.paramsDeviceMultiPages.rowLimit, ...
+ result = this.onc.getDirectRawByDevice(this.paramsDeviceMultiPages);
+ assertEqual(this, length(result.data.readings), this.paramsDeviceMultiPages.rowLimit, ...
'Test should only return `rowLimit` rows for each sensor.');
assertTrue(this, ~isempty(result.next), 'Test should return multiple pages.');
end
- %}
+
end
end
\ No newline at end of file
From f2eddac8e64140df35ae224abe52bd302268f173 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Thu, 18 Apr 2024 15:53:29 -0700
Subject: [PATCH 19/35] issue-16: Replaced util function with built-in function
fileparts
---
onc/+onc/DataProductFile.m | 10 ++++------
onc/+util/extractFileExtension.m | 13 -------------
onc/+util/save_as_file.m | 8 +++-----
3 files changed, 7 insertions(+), 24 deletions(-)
delete mode 100644 onc/+util/extractFileExtension.m
diff --git a/onc/+onc/DataProductFile.m b/onc/+onc/DataProductFile.m
index 4d0ee2d..01ffc33 100644
--- a/onc/+onc/DataProductFile.m
+++ b/onc/+onc/DataProductFile.m
@@ -95,15 +95,13 @@
% Obtain filesize from headers, or fallback to body string length
lengthData = response.getFields('Content-Length');
+ [~, ~, ext] = fileparts(filename);
if length(lengthData) == 1
this.fileSize = str2double(lengthData.Value);
+ elseif strcmp(ext, '.xml')
+ this.fileSize = length(xmlwrite(response.Body.Data));
else
- ext = util.extractFileExtension(filename);
- if strcmp(ext, 'xml')
- this.fileSize = length(xmlwrite(response.Body.Data));
- else
- this.fileSize = strlength(response.Body.Data);
- end
+ this.fileSize = strlength(response.Body.Data);
end
try
saveResult = util.save_as_file(response.Body.Data, outPath, filename, 'overwrite', overwrite);
diff --git a/onc/+util/extractFileExtension.m b/onc/+util/extractFileExtension.m
deleted file mode 100644
index a86ff8a..0000000
--- a/onc/+util/extractFileExtension.m
+++ /dev/null
@@ -1,13 +0,0 @@
-function ext = extractFileExtension(filename)
- % get extension from filename (string or char array)
- % this function is called by save_as_file.m to decide which download method to use
-
- filename = char(filename);
- possibleExtensionStartIndex = strfind(filename, '.');
- if ~isempty(possibleExtensionStartIndex)
- extensionStartIndex = possibleExtensionStartIndex(end);
- ext = filename(extensionStartIndex + 1:end);
- else
- ext = '';
- end
-end
\ No newline at end of file
diff --git a/onc/+util/save_as_file.m b/onc/+util/save_as_file.m
index 0b25928..94fe773 100644
--- a/onc/+util/save_as_file.m
+++ b/onc/+util/save_as_file.m
@@ -25,13 +25,11 @@
try
matlabVersion = version('-release');
year = str2double(matlabVersion(1:end-1));
-
- ext = util.extractFileExtension(fileName);
-
+ [~, ~, ext] = fileparts(fileName);
% if result is an image file or .xml file, use other save methods instead of fwrite.
- if strcmp(ext, 'png') || strcmp(ext, 'jpg')
+ if strcmp(ext, '.png') || strcmp(ext, '.jpg')
imwrite(dataToWrite, fullPath);
- elseif strcmp(ext, 'xml')
+ elseif strcmp(ext, '.xml')
xmlwrite(fullPath, dataToWrite);
else
% open output file
From 8f94aabc85b7dc085fe48df3b989b59f0e77bc62 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Thu, 18 Apr 2024 17:02:33 -0700
Subject: [PATCH 20/35] issue-11: update oncEnv from logical values to char
---
.github/workflows/matlab.yml | 2 +-
onc/tests/globals.m | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml
index 35a5e17..69ebcfc 100644
--- a/.github/workflows/matlab.yml
+++ b/.github/workflows/matlab.yml
@@ -37,5 +37,5 @@ jobs:
source-folder: 'onc'
env:
TOKEN: ${{ secrets.TOKEN }}
- ONC_ENV: true
+ ONC_ENV: 'prod'
diff --git a/onc/tests/globals.m b/onc/tests/globals.m
index daa37ac..be0c5a7 100644
--- a/onc/tests/globals.m
+++ b/onc/tests/globals.m
@@ -22,7 +22,9 @@
% get environment from ONC_ENV or use QA as default
config.production = getenv('ONC_ENV');
- if isempty(config.production)
+ if strcmp(config.production, 'prod')
+ config.production = true;
+ else
config.production = false;
end
% Set and save config
From 2a8c3dc982cfa7f93906743a6bfc1d478d8ae4d7 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Mon, 22 Apr 2024 11:33:42 -0700
Subject: [PATCH 21/35] issue-11: add back util function is_failed_response
---
onc/+util/is_failed_response.m | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
create mode 100644 onc/+util/is_failed_response.m
diff --git a/onc/+util/is_failed_response.m b/onc/+util/is_failed_response.m
new file mode 100644
index 0000000..9d417ac
--- /dev/null
+++ b/onc/+util/is_failed_response.m
@@ -0,0 +1,23 @@
+function isFailed = is_failed_response(response, status)
+ %% Checks if a server response describes a failure
+ %
+ % * response: (struct) Response as returned by do_request()
+ % - status: (double) http status code
+ %
+ % Returns: (Logical) true when the response is a failure
+ isFailed = false;
+
+ % Fail if HTTP status code is not a 2xx
+ if exist('status', 'var')
+ if status < 200 || status > 226
+ isFailed = true;
+ return
+ end
+ end
+
+ % Fail if the response is an error description
+ if isfield(response, "errors")
+ names = fieldnames(response.errors(1));
+ isFailed = ismember("errorCode", names) && ismember("errorMessage", names);
+ end
+end
\ No newline at end of file
From daf2ece0c2411ea365ded944b8b2df5dda7f8b72 Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Tue, 23 Apr 2024 15:20:54 -0700
Subject: [PATCH 22/35] issue-11: add success and failure messages
---
onc/+onc/OncDelivery.m | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/onc/+onc/OncDelivery.m b/onc/+onc/OncDelivery.m
index a24cf2a..ff97483 100644
--- a/onc/+onc/OncDelivery.m
+++ b/onc/+onc/OncDelivery.m
@@ -170,7 +170,9 @@
filters = struct('method', 'cancel', 'token', this.token, 'dpRequestId', dpRequestId);
[response, info] = this.doRequest(url, filters);
if isfield(response, 'status') && strcmp(response.status, 'cancelled') && info.status == 200
- fprintf("Data product with request id %d and run id %d was successfully cancelled\n", dpRequestId, response.dpRunId);
+ fprintf("The data product with request id %d and run id %d has been successfully cancelled\n", dpRequestId, response.dpRunId);
+ else
+ fprintf("Failed to cancel the data Product.");
end
end
@@ -220,7 +222,9 @@
filters = struct('method', 'restart', 'token', this.token, 'dpRequestId', dpRequestId);
[response, info] = this.doRequest(url, filters);
if isfield(response, 'status') && (strcmp(response.status, 'data product running') || strcmp(response.status, 'queued')) && info.status == 200
- fprintf("Restarted data product with request id %d and run id %d\n", dpRequestId, response.dpRunId);
+ fprintf("The data product with request id %d and run id %d has been successfully restarted\n", dpRequestId, response.dpRunId);
+ else
+ fprintf("Failed to restart the data product");
end
if waitComplete
[response, info] = this.runDataProduct(dpRequestId, true);
From 94c9dafbba15aa498f4b9ad3ce02170694587d44 Mon Sep 17 00:00:00 2001
From: IslaL <142735981+IslaL@users.noreply.github.com>
Date: Tue, 23 Apr 2024 15:23:03 -0700
Subject: [PATCH 23/35] add new line characters
---
onc/+onc/OncDelivery.m | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/onc/+onc/OncDelivery.m b/onc/+onc/OncDelivery.m
index ff97483..bc35aae 100644
--- a/onc/+onc/OncDelivery.m
+++ b/onc/+onc/OncDelivery.m
@@ -170,9 +170,9 @@
filters = struct('method', 'cancel', 'token', this.token, 'dpRequestId', dpRequestId);
[response, info] = this.doRequest(url, filters);
if isfield(response, 'status') && strcmp(response.status, 'cancelled') && info.status == 200
- fprintf("The data product with request id %d and run id %d has been successfully cancelled\n", dpRequestId, response.dpRunId);
+ fprintf("The data product with request id %d and run id %d has been successfully cancelled.\n", dpRequestId, response.dpRunId);
else
- fprintf("Failed to cancel the data Product.");
+ fprintf("Failed to cancel the data Product.\n");
end
end
@@ -222,9 +222,9 @@
filters = struct('method', 'restart', 'token', this.token, 'dpRequestId', dpRequestId);
[response, info] = this.doRequest(url, filters);
if isfield(response, 'status') && (strcmp(response.status, 'data product running') || strcmp(response.status, 'queued')) && info.status == 200
- fprintf("The data product with request id %d and run id %d has been successfully restarted\n", dpRequestId, response.dpRunId);
+ fprintf("The data product with request id %d and run id %d has been successfully restarted.\n", dpRequestId, response.dpRunId);
else
- fprintf("Failed to restart the data product");
+ fprintf("Failed to restart the data product.\n");
end
if waitComplete
[response, info] = this.runDataProduct(dpRequestId, true);
From 5b68f2742ffdf12b9a2d3db964a017634313867c Mon Sep 17 00:00:00 2001
From: Isla Li
Date: Wed, 8 May 2024 11:23:07 -0700
Subject: [PATCH 24/35] issue-20: documentation for each service
---
doc/Onc.html | 145 +++++++++++++
doc/OncArchive.html | 432 +++++++++++++++++++++++++++++++++++++++
doc/OncDelivery.html | 465 ++++++++++++++++++++++++++++++++++++++++++
doc/OncDiscovery.html | 458 +++++++++++++++++++++++++++++++++++++++++
doc/OncRealTime.html | 305 +++++++++++++++++++++++++++
doc/helptoc.xml | 41 ++++
info.xml | 26 +++
7 files changed, 1872 insertions(+)
create mode 100644 doc/Onc.html
create mode 100644 doc/OncArchive.html
create mode 100644 doc/OncDelivery.html
create mode 100644 doc/OncDiscovery.html
create mode 100644 doc/OncRealTime.html
create mode 100644 doc/helptoc.xml
create mode 100644 info.xml
diff --git a/doc/Onc.html b/doc/Onc.html
new file mode 100644
index 0000000..7ce17c8
--- /dev/null
+++ b/doc/Onc.html
@@ -0,0 +1,145 @@
+
+
+
+
+ Onc
+
+
+
The ONC class
+
The ONC class provides a wrapper for Oceans 3.0 API requests. All the client library’s functionality is provided as methods of this class. Create an ONC object to access this library’s functionalities.
+
Parameters:
+
* token ([char]) - The ONC API token, which could be retrieved at https://data.oceannetworks.ca/Profile once logged in.
+ * production (logical, optional, default = True) - Whether the ONC Production server URL is used for service requests.
+ True: Use the production server.
+ False: Use the internal ONC test server (reserved for ONC staff IP addresses).
+ * showInfo (logical, optional, default = false) - Whether verbose script messages are displayed, such as request url and processing time information.
+ True: Print all information and debug messages (intended for debugging).
+ False: Only print information messages.
+ * outPath ([char], optional, default = 'output') - Output path for downloaded files. The directory will be created if it does not exist during the download.
+ * timeout (int, optional, default = 60) - Number of seconds before a request to the API is canceled
+
For detailed information and usage examples, run doc command or visit MATLAB's help browser https://data.oceannetworks.ca/Profile then find Ocean Networks Canada API Client under supplemental software
+
function this = Onc(token, varargin)
+ p = inputParser;
+ addRequired(p, 'token', @ischar);
+ addOptional(p, 'production', true, @islogical);
+ addOptional(p, 'showInfo', false, @islogical);
+ addOptional(p, 'outPath', 'output', @ischar);
+ addOptional(p, 'timeout', 60, @isnumeric);
+ parse(p, token, varargin{:});
+
+ this.token = strtrim(p.Results.token);
+ this.production = p.Results.production;
+ this.showInfo = p.Results.showInfo;
+ this.timeout = p.Results.timeout;
+
+ % sanitize outPath
+ opath = strtrim(p.Results.outPath);
+ if strlength(opath) > 0
+ opath = strrep(opath, '\', '/');
+ if opath(end) == '/'
+ opath = opath(1:end-1);
+ end
+ end
+ this.outPath = opath;
+
+ if not(this.production)
+ this.baseUrl = 'https://qa.oceannetworks.ca/';
+ end
+
+ %If a search tree file exists, load it. If not, generate and save one
+ [source_path,~,~] = fileparts(which('Onc.m'));
+ tree_path = fullfile(source_path,'onc_tree.mat');
+ if ~exist(tree_path,'file')
+ fprintf('\n Loading ONC search tree. Accessible with onc.tree \n');
+ tree = util.extract_tree(this);
+ save(tree_path, 'tree')
+ elseif exist(tree_path,'file')
+ %Check if it's more than a week old. If so, update it:
+ dir_files = dir(source_path);
+ filenames = {dir_files(:).name};
+ [~,idx] = ismember('onc_tree.mat',filenames);
+ treeFileDate = dir_files(idx).datenum;
+ if now - treeFileDate > 7
+ fprintf('\n Updating ONC search tree. Accessible with onc.tree \n');
+ tree = util.extract_tree(this);
+ save(tree_path, 'tree')
+ end
+ end
+ temp = load(tree_path);
+ this.tree = temp.tree;
+ %These codes can then be used for input to onc.getDevices by
+ %providing the locationCodes
+end
+
Contains the functionality that wraps API archivefile services To be inherited by the Onc class These methods allow users to directly download previously generated data product files from our archive.
+
ONC systems auto-generate and archive files of different types at set time intervals. These archived files can be downloaded without waiting for a generation process to finish (potentially faster than Data product download methods).
+
+
Note
+
Archived files have a unique filename (e.g. “NAXYS_HYD_007_20091231T235919.476Z-spect.png”) that includes the device code (“NAXYS_HYD_007”) and the UTC date-time when the data in the file started being measured (“20091231T235919.476Z”). The filename might contain additional information.
+
+
+
+
Caution
+
Due to security regulations, some very recent files (e.g. hydrophone.wav files in the last hour) might not be made immediately available.
+
+
GetListByLocation(filters, allPages)
+
Get a list of files for a given location and device category filtered by other optional parameters.
+
Input:
+
+
filters(struct) - Describes the data origin
+
allPages(logical, optional, default = false) - When true, if the data requested is too large to fit a single API resquest, keep downloading data pages until we gather all data
+
+
Output:
+
+
array of structs - File list obtained
+
+
+
The API endpoint is /archivefile/location.
+
Parameters in filter: Query string parameters in the API request. Supported parameters are:
+
Get a list of files for a given device filtered by other optional parameters.
+
Input:
+
+
filters(struct) - Describes the data origin
+
allPages(logical, optional, default = false) - When true, if the data requested is too large to fit a single API resquest, keep downloading data pages until we gather all data
+
+
Output:
+
+
array of structs - File list obtaine
+
+
The API endpoint is /archivefile/location.
+
Parameters in filter: Query string parameters in the API request. Supported parameters are:
Downloads all archive files that match the filters Uses geListByDevice or getListByLocation to get a file list, then getFile's everything.
+
Input:
+
+
filters(struct) - Describes the data origin
+
allPages(logical, optional, default = false) - When true, if the data requested is too large to fit a single API resquest, keep downloading data pages until we gather all data
+
overwrite(logical, optional) - When true, downloaded files will overwrite any file with the same filename, otherwise(default) file will be skipped
+
+
Output:
+
+
struct - Information on the results of the operation, with 'downloadResults' for each file downloaded and general 'stats
+
+
Parameters in filter: Query string parameters in the API request. Supported parameters are:
Functionality that wraps the API data product delivery services. To be inherited by the Onc class Data product download methods allow you to request and download more than 120 different types of ONC data products, with granular control over what data to obtain, from where, and in what time frame. They are comparable to the download functionality from ONC’s Data Search tool. Examples of usage include:
+
+
+
Downloading PNG plots of sensor readings in a device
+
Downloading sensor readings as .mat files, text files, or in commercial manufacturer formats
+
Downloading compressed or raw audio files from hydrophones
+
+
+
+
Note
+
If the data product requested doesn’t exist in our archive, it will be generated by our servers before your download starts.
function [response, info] = cancelDataProduct(this, dpRequestId)
+ url = sprintf('%sapi/dataProductDelivery', this.baseUrl);
+ filters = struct('method', 'cancel', 'token', this.token, 'dpRequestId', dpRequestId);
+ [response, info] = this.doRequest(url, filters);
+ if isfield(response, 'status') && strcmp(response.status, 'cancelled') && info.status == 200
+ fprintf("The data product with request id %d and run id %d has been successfully cancelled.\n", dpRequestId, response.dpRunId);
+ else
+ fprintf("Failed to cancel the data Product.\n");
+ end
+end
+
Contains the functionality that wraps the API discovery services To be inherited by the Onc class Discovery methods can be used to search for available locations, deployments, device categories, devices, properties, and data products. They support numerous filters and might resemble an "advanced search” function for ONC data sources.
+
Use discovery methods to:
+
+
+
Obtain the identification codes required to use other API services.
+
Obtain the identification codes required to use other API services.
+
Explore what's available in a certain location or device.
+
Obtain the deployment dates for a device.
+
List available data products for download in a particular device or location.
+
+
+
+
+
Note
+
+
Locations can contain other locations.
+
"Cambridge bay" may contain separate children locations for its underwater network and shore station.
+
Locations can contain device categories, which contain devices, which contain properties.
+
Searches can be performed without considering the hierarchy mentioned above.
+
You can search for locations with data on a specific property or search for all properties in a specific location.
+
+
+
+
GetLocations(filters)
+
Obtain a filtered list of locations
+
Input: filters(cell array, optional) - Describes the data origin
Parameters: filters(cell array, optional) - Query string parameters in the API request. Return all locations available if None. Supported parameters are:
function r = getLocations(this, varargin)
+ filters = this.getFilters(varargin);
+ r = this.discoveryRequest(filters, 'locations');
+end
+
+
GetLocationHierarchy(filters)
+
Obtain a location tree
+
Input: filters(cell array, optional) - Describes the data origin
+
Output: struct - Location tree
+
The API endpoint is /locations/tree.
+
Return a hierarchical representation of the ONC Search Tree Nodes. The Search Tree is used in Oceans 3.0 to organize instruments and variables by location so that users can easily drill down by place name or mobile platform name to find the instruments or properties they are interested in.
Parameters: filters(cell array, optional) - Query string parameters in the API request. Return all locations available if None. Supported parameters are:
function r = getLocationHierarchy(this, varargin)
+ filters = this.getFilters(varargin);
+ r = this.discoveryRequest(filters, 'locations', 'method', 'getTree');
+end
+
+
+
GetDeployments(filters)
+
Obtain an array of device deployments.
+
Input: filters(cell array, optional) - Describes the data origin
+
Output: array of structs - Deployments found
+
The API endpoint is /deployments.
+
Return all deployments defined in Oceans 3.0 which meet the filter criteria, where a deployment is the installation of a device at a location. The deployments service assists in knowing when and where specific types of data are available.
+The primary purpose for the deployments service is to find the dates and locations of deployments and use the dateFrom and dateTo datetimes when requesting a data product using the dataProductDelivery web service.
Parameters: filters(cell array, optional) - Query string parameters in the API request. Return a tree of all available locations available if None. Supported parameters are:
function r = getDeployments(this, varargin)
+ filters = this.getFilters(varargin);
+ r = this.discoveryRequest(filters, 'deployments');
+end
+
+
GetDevices(filters)
+
Obtain a filtered list of devices
+
Input: filters(cell array, optional) - Describes the data origin
+
Output: array of structs - Devices found
+
The API endpoint is /devices.
+
Return all the devices defined in Oceans 3.0 that meet a set of filter criteria.
+ Devices are instruments that have one or more sensors that observe a property or phenomenon with a goal of producing an
+ estimate of the value of a property. Devices are uniquely identified by a device code and can be deployed at multiple
+ locations during their lifespan. The primary purpose of the devices service is to find devices that have the data you are interested in
+ and use the deviceCode when requesting a data product using the dataProductDelivery web service.
Parameters: filters(cell array, optional) - Query string parameters in the API request. Return all devices available if None. Supported parameters are:
function r = getDevices(this, varargin)
+ filters = this.getFilters(varargin);
+ r = this.discoveryRequest(filters, 'devices');
+end
+
+
+
GetDeviceCategories(filters)
+
Obtain a filtered list of device categories
+
Input: filters(cell array, optional) - Describes the data origin
+
Output: array of structs - Device categories found
+
The API endpoint is /deviceCategories.
+
Return all device categories defined in Oceans 3.0 that meet a filter criteria. A Device Category represents an instrument type
+classification such as CTD (Conductivity, Temperature & Depth Instrument) or BPR (Bottom Pressure Recorder).
+Devices from a category can record data for one or more properties (variables). The primary purpose of this service is to find
+device categories that have the data you want to access; the service provides the
+deviceCategoryCode you can use when requesting a data product via the dataProductDelivery web service.
Parameters: filters(cell array, optional) - Query string parameters in the API request. Return all device categories available if None. Supported parameters are:
function r = getDeviceCategories(this, varargin)
+ filters = this.getFilters(varargin);
+ r = this.discoveryRequest(filters, 'deviceCategories');
+end
+
+
GetProperties(filters)
+
Obtain a filtered list of properties
+
Input: filters(cell array, optional) - Describes the data origin
+
Output: array of structs - Properties found
+
The API endpoint is /properties.
+
Return all properties defined in Oceans 3.0 that meet a filter criteria. Properties are observable phenomena (aka, variables) are the
+ common names given to sensor types (i.e., oxygen, pressure, temperature, etc).
+ The primary purpose of this service is to find the available properties of the data you want to access;
+ the service provides the propertyCode that you can use to request a data product via the dataProductDelivery web service.
Parameters: filters(cell array, optional) - Query string parameters in the API request. Return all properties available if None. Supported parameters are:
function r = getProperties(this, varargin)
+ filters = this.getFilters(varargin);
+ r = this.discoveryRequest(filters, 'properties');
+end
+
+
GetDataProducts(filters)
+
Obtain a list of available data products for the filters
+
Input: filters(cell array, optional) - Describes the data origin
+
Output: array of structs - Data products found
+
The API endpoint is /dataProducts.
+
Return all data products defined in Oceans 3.0 that meet a filter criteria. Data Products are downloadable representations of ONC
+ observational data, provided in formats that can be easily ingested by analytical or visualization software. The primary purpose of this
+ service is to identify which data products and formats (file extensions) are available for the locations, devices, device categories or
+ properties of interest. Use the dataProductCode and extension when requesting a data product via the dataProductDelivery web service.
Parameters: filters(cell array, optional) - Query string parameters in the API request. Return all data products available if None. Supported parameters are:
Contains the functionality that wraps API real-time services. To be inherited by the Onc class Near real-time (as fast as they get into our database) data access methods allow the extraction of sensor data as time-series, either as processed scalar data with Quality Assurance and Control flags (QAQC) or directly as raw data obtained from the device in its specific output format. In contrast to the Data product download methods, this data can be downloaded directly without waiting for any kind of generation process.
+
Common use cases include:
+
+
+
Plotting time series from properties in a specific time frame or in “near real-time”
+
Quickly obtaining the latest reading from a particular sensor
+
Obtaining raw unprocessed data from our instruments (data might require processing to be readable)
+
+
+
+
Note
+
+
The methods getDirectByLocation() and getDirectRawByLocation() obtain data readings from a location no matter what device it came from (hence the need to specify a device category code instead of a single device code). You might want to obtain data by location instead of by device, as individual devices are often replaced/repositioned.
+
Each request to our API can return a maximum of 100,000 samples; larger data requests must be downloaded as a sequence of pages. Use the allPages parameter to automatically download all pages required for your requested time frame.
+
+
+
GetDirectByLocation(filters, allPages)
+
Obtains scalar data from a location, from the source described by the filters
+
Input:
+
+
filters(struct) - Describes the data origin
+
allPages(logical, optional) - When true, if the data requested is too large to fit a single API resquest, keep downloading data pages until we gather all data
+
+
+
Output:
array of structs - Scalar data obtained for all sensors found
+
The API endpoint is /scalardata/location.
+
Parameters in filter: Query string parameters in the API request. Supported parameters are:
function r = getDirectByLocation(this, filters, varargin)
+ [allPages] = util.param(varargin, 'allPages', false);
+ r = this.getDirectAllPages(filters, 'scalardata', 'getByLocation', allPages);
+end
+
+
GetDirectByDevice(filters, allPages)
+
Obtains scalar data from a device, as described by the filters
+
Input:
+
+
filters(struct) - Describes the data origin
+
allPages(logical, optional) - When true, if the data requested is too large to fit a single API resquest, keep downloading data pages until we gather all data
+
+
+
Output:
+
+
array of structs - Scalar data obtained for all sensors found
+
+
+
The API endpoint is /scalardata/device.
+
Parameters in filter: Query string parameters in the API request. Supported parameters are:
+
function r = getDirectByDevice(this, filters, varargin)
+ [allPages] = util.param(varargin, 'allPages', false);
+ r = this.getDirectAllPages(filters, 'scalardata', 'getByDevice', allPages);
+end
+
+
GetDirectRawByLocation(filters, allPages)
+
Obtains raw data from a location, from the source described by the filters
+
Input:
+
+
filters(struct) - Describes the data origin
+
allPages(logical, optional) - When true, if the data requested is too large to fit a single API resquest, keep downloading data pages until we gather all data
+
+
Output:
+
+
array of structs - Raw data obtained for all sensors found
+
+
The API endpoint is /rawdata/location.
+
Parameters in filter: Query string parameters in the API request. Supported parameters are:
+
function r = getDirectRawByLocation(this, filters, varargin)
+ [allPages] = util.param(varargin, 'allPages', false);
+ r = this.getDirectAllPages(filters, 'rawdata', 'getByLocation', allPages);
+end
+
+
GetDirectRawByDevice(filters, allPages)
+
Obtains raw data from a device, as described by the filters
+
Input:
+
+
filters(struct) - Describes the data origin
+
allPages(logical, optional) - When true, if the data requested is too large to fit a single API resquest, keep downloading data pages until we gather all data
+
+
+
Output:
+
+
array of structs - Raw data obtained for all sensors found
+
+
+
The API endpoint is /rawdata/device.
+
Parameters in filter: Query string parameters in the API request. Supported parameters are:
+
For more information about creating an Onc object, see ONC Class
+
+
+
Note
+
+
It's important to check that you are using the latest version. Outdated version may results in bugs.
+
When you run Onc() to create an Onc object, it will run a version check and prints out version outdated warning if fails. How to update to latest version.
+
+
+
+
General Tutorial - 1. Searching with discovery services
+
Check sections in the left panel for full documentation, source code and examples on each service.
+
To download data from Ocean Networks Canada's database (Oceans 3.0) , you need to specify the type of data you are interested in
+ and where in particular (i.e. location, from specific instrument (device)) it originates from.
+ In the Oceans 3.0 API, there are a unique codes that identify every location, device, property, data product type, etc.
+ You include these codes in a group of filters to retrieve the information you're interested in.
+ The Oceans 3.0 API Discovery services allow you to explore the hierarchy of ONC's database to obtain the codes required for your filters
+ to obtain the information/data you are interested in (they work like a "search" function).
+
+
The example below uses the getLocations method, which is querying the locations database tables that include "Bullseye" in
+ their name (i.e. "Clayoquot Slope Bullseye Vent"). It returns a list with all locations that match the search filters provided.
+
Using ONC library
+
% 1. Define your filter parameter
+params = {'locationName', 'Bullseye'};
+% 2. Call methods in the onc library
+onc.getLocations(params)
+
+ ans = struct with fields:
+ deployments: 38
+ locationName: 'Bullseye'
+ depth: 1.2569e+03
+ bbox: [1x1 struct]
+ description: ' Bullseye is a location at Clayoquot Slope, where gas hydrates, seafloor cold seeps, and hydrate dynamics are observed.'
+ hasDeviceData: 1
+ lon: -126.8480
+ locationCode: 'NC89'
+ hasPropertyData: 0
+ lat: 48.6706
+ dataSearchURL: 'https://data.oceannetworks.ca/DataSearch?location=NC89'
+
+
Each entry of this list contains more meta data information for that location, e.g. the locationName, the geographical coordinates and depth,
+ a description field and the URL for Oceans 3.0 Data Search tool. The parameter locationCode contains the string "NC89", which is needed for the next steps.
+
1.1 What device categories are available here at the location NC89?
+
Using ONC library
+
% 1. Define your filter parameter
+params = {'locationCode', 'NC89'};
+% 2. Call methods in the onc library
+onc.getDeviceCategories(params)
+ ' Acoustic Doppler Current Profilers are hydroacoustic instruments, similar to sonars. ADCPs measure current speed and direction at multiple predetermined depths simultaneously. ADCPs use the Doppler effect of sound waves that are scattered by particles in seawater over a depth range.'
+
+
+
+
+ 2
+
+
+ 1×1 struct
+
+
+ 'Acoustic Doppler Current Profiler 75 kHz '
+
+
+ 'ADCP75KHZ'
+
+
+ 'Acoustic Doppler Current Profiler 75 kHz'
+
+
+ 1
+
+
' Acoustic Doppler Current Profilers (ADCP) are hydroacoustic instruments, similar to sonars. ADCPs measure current speed and direction at multiple predetermined depths simultaneously. ADCPs use the Doppler effect of sound waves that are scattered by particles in seawater over a depth range.'
+
3
1×1 struct
+
'Broadband Seismometer'
+
'BBS'
+
'Broadband Seismometer'
+
1
+
' Broadband Seismometers measure seismic waves over a broad frequency range depending on the device (e.g. 0.00278 Hz - 250 Hz). A broadband seismometer facilitates measurement of seismic events in the widest frequency band possible.'
+
4
1×1 struct
+
'Bottom Pressure Recorder'
+
'BPR'
+
'Bottom Pressure Recorder'
+
1
+
' Bottom Pressure Recorders (BPR) are instruments that can detect small changes in pressure on the seafloor. '
+
5
1×1 struct
+
'Controlled Source Electromagnetic Method'
+
'CSEM'
+
'Controlled Source Electromagnetic Method'
+
1
+
' The Controlled Source Electromagnetic Method (CSEM) measures sub-surface resistivity structure through the measurement of the electromegnetic fields resulting from stimulation by a towed source.'
+
6
1×1 struct
+
'Conductivity Temperature (and Depth Sensor)'
+
'CTD'
+
'Conductivity Temperature Depth'
+
1
+
' Conductivity Temperature Depth (CTD) is an instrument package that contains sensors for measuring the conductivity, temperature, and pressure of seawater. Salinity, sound velocity, depth and density are variables that can be derived from sensor measurements. CTDs can carry additional instruments and sensors such as oxygen sensors, turbidity sensors and fluorometers.'
+
7
1×1 struct
+
'Current Meter'
+
'CURRENTMETER'
+
'Current Meter'
+
1
+
' Acoustic Current Meters (ACM) measure current speed and direction, using the Doppler Effect. Aquadopp current meters have a sensor head that contains 3 acoustic transducers, a tilt sensor, a temperature sensor and a pressure sensor. The instrument transmits a short pulse of sound, and then listens to its echo to measure the change in pitch or frequency. The change in pitch can determine the velocity of the current.'
+
8
1×1 struct
+
'Gravimeter'
+
'GRAVIMETER'
+
'Gravimeter'
+
1
+
' Gravimeters (or gravity meters) measure the gravity field of the Earth with such a resolution that they can detect very small changes in the underlying or surrounding structures.'
+
9
1×1 struct
+
'Junction Box'
+
'JB'
+
'Junction Box'
+
1
+
' Junction Boxes supply power and communications to deployed instruments. Junction boxes have a number of serial and ethernet ports, including 400V ethernet ports that enable connections to other junction boxes and high-voltage instruments. Junction boxes can convert high voltages to lower voltages (15V, 24V or 48V) required by many instruments.'
+
10
1×1 struct
+
'Oxygen Sensor'
+
'OXYSENSOR'
+
'Oxygen Sensor'
+
1
+
' Oxygen sensors measure dissolved oxygen concentration in seawater.'
+
11
1×1 struct
+
'Pan Tilt Lights'
+
'PTL'
+
'Pan Tilt Lights'
+
1
+
' Pan Tilt Lights are used for cameras and allow remotely controlled operations such as changing the camera's field of view and illuminating the subject matter.'
+
12
1×1 struct
+
'Tiltmeter'
+
'TILTMTR'
+
'Tiltmeter'
+
1
+
' A tiltmeter is a sensitive inclinometer designed to measure very small changes from the vertical level, either on the ground or in structures.'
+
13
1×1 struct
+
'Video Camera'
+
'VIDEOCAM'
+
'Video Camera'
+
1
+
' Video cameras record video of characteristics of the surrounding environments and can be deployed on fixed and mobile platforms.'
+
+
+
1.2 What properties are available for the device category CTD at this location NC89?
+
Using ONC library
+
% 1. Define your filter parameter
+params = {'locationCode', 'NC89', ...
+ 'deviceCategoryCode', 'CTD'};
+% 2. Call methods in the onc library
+onc.getProperties(params)
+
If the row of the data is above 100,000, not all the data will be returned. The rest of the data can be queried based on the next key in the response.
+
+
If you use onc library.
getDirectRawByLocation supports a boolean allPages parameter. When set to True, it will try to retrieve all the pages.
+
If you use MATLAB's HTTP library.
You have to manually query the next pages until the next key in the response json is None, and concatenate all the data together.
+
+
+
Using ONC library
+
% 1. Define your filter parameter with a longer date range (2 days of data)
+paramsLongerRange = {'locationCode', 'NC89', ...
+ 'deviceCategoryCode', 'CTD', ...
+ 'dateFrom', '2020-06-20T00:00:00.000Z', ...
+ 'dateTo', '2020-06-22T00:00:00.000Z', ...
+ };
+% 2. Call methods in the onc library
+result = onc.getDirectRawByLocation(paramsLongerRange, 'allPages', true)
+struct2table(result.data)
+
+
+
Data size is greater than the row limit and will be downloaded in multiple pages.
+ Estimated approx. 1 pages
+ Estimated approx. 5.79 seconds to complete
+
+ (100000 samples) Downloading page 2...
+ (172796 samples) Completed in 2.25 seconds.
A faster way to download data products and processed data files that are available in Oceans 3.0 (if it suits your needs) is to leverage how ONC scripts
+auto-generate and archive data products of different types at set time intervals. You can directly download these data product files from our files archive, as long as you know their unique filename.
+
In the following example, we get the list of archived files available for a camera (deviceCategoryCode: VIDEOCAM) at Ridley Island (locationCode: RISS) for 5-minute timerange.
Once we have the file names, you can use the method "getFile()" to download individual files:
+
Using ONC library
+
% 1. Call methods in the onc library with the filename. The file is downloaded in the output folder.
+onc.getFile('AXISQ6044PTZACCC8E334C53_20161201T000001.000Z.jpg', 'overwrite', true)
+
+
Downloading file "AXISQ6044PTZACCC8E334C53_20161201T000001.000Z.jpg"...
+ [==================================================] 100%
+ File was downloaded to "AXISQ6044PTZACCC8E334C53_20161201T000001.000Z.jpg"
% 1. Define your filter parameter with the filename
+params.filename = 'AXISQ6044PTZACCC8E334C53_20161201T000001.000Z.jpg';
+params.token = 'YOUR TOKEN HERE'; % or readToken
+% 2. Define your base url for this query
+url_location = 'https://data.oceannetworks.ca/api/archivefile/download';
+request = matlab.net.http.RequestMessage;
+uri = matlab.net.URI(url_location);
+uri.Query = matlab.net.QueryParameter(params);
+
+% 3. prepare MATLAB request options
+options = matlab.net.http.HTTPOptions();
+options.ConnectTimeout = 120;
+
+% 4. send request
+result = request.send(uri,options);
+% 5. Save the file
+% f = fopen(fullPath, 'w','n','ISO-8859-1');
+% if f ~= -1
+% fwrite(f, char(dataToWrite));
+% end
+
+
2.4 Downloading data products
+
Other than using Oceans 3.0 Data Search, we can request the ONC server to generate a data product. This is done through the data product delivery services methods.
+
+
Hint
+
This service should ONLY be used when the requested files are not already provided using the ArchiveFiles services (see 2.3 above).
+ The data product delivery services will re-generate files using ONC's web machines and this process can often take very long time to generate these results.
+ If you request data files for very long-time ranges and large file sizes, ONCs system will sometimes slow down and stall and requires some manual actions.
+
We therefore encourage you to check other services before requesting data through this service. If you are unsure what to use feel free to contact u.
+
+
This process will require three steps before you will be able to see the downloaded data product on your computer:
+
+
+
Request the data.
+
Run the Request.
+
Download the data.
+
+
+
The following example downloads two PNG files with plots for 30 minutes of data from a CTD (find them in the "output" folder beside this jupyter notebook).
+ The filter includes codes for location, deviceCategory, and dataProduct, as well as the file extension and a time interval.
+ They also include a couple of filters to configure this specific data product type (starting with the "dpo_" prefix) which can be obtained from the Data Product Options documentation. You can download more than 120 different types of data products including audio & video.
+
Using ONC library
+
ONCs library contains all three steps (methods) in one call. So this is the preferred library to use over the requests library.
citations: [1×1 struct]
+ disclaimer: 'Software Developers are implementing estimates of processing times and file sizes for data requests. These are extremely rough to begin with, but bear with us. We expect these estimates will gradually improve.'
+ dpRequestId: 18690550
+ estimatedFileSize: '185 kB'
+estimatedProcessingTime: '20 s'
+ messages: []
+ queryPids: 25848930
+ queryURL: 'https://data.oceannetworks.ca/api/dataProductDelivery/request?locationCode=NC89&deviceCategoryCode=CTD&dataProductCode=TSSP&extension=png&dateFrom=2017-01-19T00:00:00.000Z&dateTo=2017-01-19T00:30:00.000Z&token=YOUR_TOKEN&dpo_resample=none&dpo_resample=none&dpo_qualityControl=1'
+
+
+
+%% requests continued
+% Run the request
+% Note: you have to execute this cell multiple times until the return shows the "status": "complete"
+% Note: Depending on your request, you can have more than one file ('fileCount').
+% You will need to individually download these files by using the index parameter.
+url_run = 'https://data.oceannetworks.ca/api/dataProductDelivery/run';
+
+requestID = requestResponse.Body.Data.dpRequestId;
+params_run = struct();
+params_run.dpRequestId = requestID;
+params_run.token = readToken;
+
+request = matlab.net.http.RequestMessage;
+uri = matlab.net.URI(url_run);
+uri.Query = matlab.net.QueryParameter(params_run);
+runResponse = request.send(uri,options);
+result = runResponse.Body.Data
+
%% requests continued
+% Find the RunID for the next step
+runId = response(1).dpRunId
+
+
runId = 40531302
+
+
%% requests continued
+% 3. Download the data
+url_download = 'https://data.oceannetworks.ca/api/dataProductDelivery/download';
+params_download = struct();
+params_download.dpRunId = runId;
+params_download.token = readToken;
+params_download.index = '1';
+request = matlab.net.http.RequestMessage;
+uri = matlab.net.URI(url_download);
+uri.Query = matlab.net.QueryParameter(params_download);
+
+% Start - Rerun this part until the response code is 200.
+downloadResponse = request.send(uri,options);
+result = downloadResponse.Body.Data
+responseCode = double(downloadResponse.StatusCode)
+% End - Rerun this part until the response code is 200.
+
+% %downloadResponse.Headers has field Content-Disposition,
+% %and Content-Disposition has the format "attachement; filename=XXX.png"
+% contentDisposition = char(downloadResponse.Header.getFields('Content-Disposition').Value);
+% filename = contentDisposition(23:end);
+% imwrite(downloadResponse.Body.Data, filename);
+% %Use other download functions if content type is not png/jpg
+
+
result = struct with fields:
message: 'Running'
+ status: 'running'
+
+
responseCode = 202
+
+
+
Another option to get the data
+
Obtain your downloads from your user FTP directory (More -> User Directory) in Oceans 3.0. Navigate to the folder that contains the runId: You will see the files in this folder.
+
+
+
Copyright 2024, ONC Data Team.
+
+
diff --git a/doc/Index.html b/doc/Index.html
new file mode 100644
index 0000000..8a11bfd
--- /dev/null
+++ b/doc/Index.html
@@ -0,0 +1,158 @@
+
+
+
+ Overall Info
+
+
Ocean Networks Canada API Client Library
+
This library serves as a toolbox to access ONC Web Services which allows users to discover and retrieve Ocean Networks Canada's 12+ years of oceanographic data in raw, text, image, audio, video or any other format available. This codebase provides a class that wraps web service calls, complex workflows, and business logic so that users can download data with a single line of code.
+
Check left panel for more documentation and code examples. You can also find code examples under Examples section from here
+
Introduction to ONC Web Services/API
+
What are ONC Web Services?
+
A group of public web services that can be used to explore and download ONC data.
For detailed information and usage examples, run doc command or visit MATLAB's help browser https://data.oceannetworks.ca/Profile then find Ocean Networks Canada API Client under supplemental software
+
For detailed information and usage examples, run doc command or visit MATLAB's help browser https://www.mathworks.com/help/matlab/ then find Ocean Networks Canada API Client under supplemental software
+
Source code:
function this = Onc(token, varargin)
p = inputParser;
addRequired(p, 'token', @ischar);
diff --git a/doc/OncArchive.html b/doc/OncArchive.html
index 2a40b2d..e1edfb1 100644
--- a/doc/OncArchive.html
+++ b/doc/OncArchive.html
@@ -10,7 +10,7 @@
html,body,div,span,applet,object,iframe,h1,h2,h3,h4,h5,h6,p,blockquote,pre,a,abbr,acronym,address,big,cite,code,del,dfn,em,font,img,ins,kbd,q,s,samp,small,strike,strong,tt,var,b,u,i,center,dl,dt,dd,ol,ul,li,fieldset,form,label,legend,table,caption,tbody,tfoot,thead,tr,th,td{margin:0;padding:0;border:0;outline:0;font-size:100%;vertical-align:baseline;background:transparent}body{line-height:1}ol,ul{list-style:none}blockquote,q{quotes:none}blockquote:before,blockquote:after,q:before,q:after{content:'';content:none}:focus{outine:0}ins{text-decoration:none}del{text-decoration:line-through}table{border-collapse:collapse;border-spacing:0}
html { min-height:100%; margin-bottom:1px; }
-html body { height:100%; margin:0px; font-family:Arial, Helvetica, sans-serif; font-size:10px; color:#000; line-height:140%; background:#fff none; overflow-y:scroll; }
+html body { height:100%; margin:0px; font-family:Arial, Helvetica, sans-serif; font-size:14px; color:#000; line-height:140%; background:#fff none; overflow-y:scroll; }
html body td { vertical-align:top; text-align:left; }
h1 { padding:0px; margin:0px 0px 25px; font-family:Arial, Helvetica, sans-serif; font-size:2.0em; color:#d55000; line-height:100%; font-weight:bold; }
diff --git a/doc/OncDelivery.html b/doc/OncDelivery.html
index 70521da..77d8c04 100644
--- a/doc/OncDelivery.html
+++ b/doc/OncDelivery.html
@@ -128,7 +128,7 @@
response(struct) - restart process status and message
info(struct) - restart process http code and status
-
The API endpoint is /dataProductDelivery/restart.
+
The API endpoint is /dataProductDelivery/restart.
Returns: API response. Each response(struct) returned contains following fields:
* dpRunId: char array
* status: char array
diff --git a/doc/UpdateInstruction.html b/doc/UpdateInstruction.html
new file mode 100644
index 0000000..50f3601
--- /dev/null
+++ b/doc/UpdateInstruction.html
@@ -0,0 +1,151 @@
+
+
+
+ How to update to latest version
+
+
How to update to latest version
+
When using Ocean Networks Canada API Client Library, you may get a warning about outdated version. Follow the following instruction to update to latest version.
1. Select Manage Add-Ons under Add-Ons from the HOME menu.
+
+
2. Select updates at the top, new releases of installed libraries will show below. Click update to the right of Ocean Networks Canada API Client Library.