Merge branch 'release-staging-24.03' into 16-fix-download-issue-occur…

…red-in-data-product-delivery-service

.github/workflows/matlab.yml

@@ -0,0 +1,41 @@
+name: Run MATLAB Tests
+
+on:
+  pull_request:
+    branches: 
+      - 'main'
+      - 'release-staging-*'
+  push:
+    branches: 
+      - 'main'
+      - 'release-staging-*'
+  workflow_dispatch:
+
+jobs:
+  tests:
+    name: MATLAB Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        matlab: ["R2022b"]
+        os: [ubuntu-latest, windows-latest]
+    env:
+      MLM_LICENSE_FILE: ${{ secrets.MATLAB_LICENSE }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@main
+
+    - name: Set up MATLAB
+      uses: matlab-actions/setup-matlab@v2
+      with:
+        release: ${{ matrix.matlab }}
+        cache: true
+
+    - name: Run tests
+      uses: matlab-actions/run-tests@v2
+      with:
+         source-folder: 'onc'
+      env:
+        TOKEN: ${{ secrets.TOKEN }}
+        ONC_ENV: true
+

onc/+onc/DataProductFile.m

@@ -64,7 +64,9 @@
             uri = matlab.net.URI(this.baseUrl);
             uri.Query = matlab.net.QueryParameter(this.filters);
             fullUrl = char(uri);
+
             options = matlab.net.http.HTTPOptions('ConnectTimeout', timeout);
+
             while this.status == 202
                 % run and time request
                 if this.showInfo, log.printLine(sprintf('Requesting URL:\n   %s', fullUrl)); end
@@ -115,6 +117,7 @@
                             rethrow(ME);
                         end
                     end
+
                     this.downloadingTime = round(duration, 3);
 
                     % log status
@@ -132,6 +135,7 @@
                     % API Error
                     util.print_error(response, fullUrl);
                     throw(util.prepare_exception(s, double(response.Body.Data.errors.errorCode)));
+
                 elseif s == 404
                     % Index too high, no more files to download
                 else

onc/+onc/MultiPage.m

@@ -208,13 +208,13 @@
                 end    
             elseif isArchive
                 row0 = response.files(1);
+                rowEnd = response.files(end);
 
-
-                if isa(row0, 'char')
+                if isa(row0, 'char') || iscell(row0)
                     % extract the date from the filename
                     regExp  = '\\d{8}T\\d{6}d\\.\\d{3}Z';
-                    nameFirst = response.files(1);
-                    nameLast  = response.files(end);
+                    nameFirst = char(row0);
+                    nameLast  = char(rowEnd);
                     mFirst = regexp(nameFirst, regExp, 'once', 'match');
                     mLast  = regexp(nameLast,  regExp, 'once', 'match');
                     if isempty(mFirst) || isempty(mLast)

onc/+onc/OncArchive.m

@@ -51,7 +51,9 @@
             % Returns: (struct) Information on the download result
             %
             % Documentation: https://wiki.oceannetworks.ca/display/CLmatlab/Archive+file+download+methods
-
+            if ~exist('filename', 'var')
+                filename = '';
+            end
             [overwrite, showMsg] = util.param(varargin, 'overwrite', false, 'showMsg', true);
 
             url = this.serviceUrl('archivefiles');
@@ -61,10 +63,11 @@
 
             [response, info] = ...
                 util.do_request(url, filters, 'timeout', this.timeout, 'showInfo', this.showInfo, ...
-                                'rawResponse', true, 'showProgress', true);
+                                'showProgress', true);
 
             if not(info.status == 200)
-                fileInfo = jsondecode(response);
+                %fileInfo = jsondecode(response);
+                fileInfo = response;
                 return;
             end
 
@@ -76,9 +79,6 @@
                 if saveStatus == 0
                     txtStatus = 'completed';
                     if showMsg, fprintf('   File was downloaded to "%s"\n', filename); end
-                elseif saveStatus == -2
-                    if showMsg, fprintf('   File was skipped (already exists).\n'); end
-                    txtStatus = 'skipped';
                 end
 
                 fullUrl = this.getDownloadUrl(filename);

onc/+onc/OncDelivery.m

@@ -33,18 +33,11 @@
             fileList = [];
 
             % Request the product
-            [rqData, status] = this.requestDataProduct(filters);
-            if util.is_failed_response(rqData, status)
-                r = rqData;
-                return
-            end
+            [rqData, ~] = this.requestDataProduct(filters);
 
             % Run the product request
             [runData, status] = this.runDataProduct(rqData.dpRequestId);
-            if util.is_failed_response(runData, status)
-                r = runData;
-                return;
-            end
+
 
             if downloadResultsOnly
                 % Only run and return links
@@ -98,13 +91,13 @@
             % runDataProduct (dpRequestId)
             %
             % * dpRequestId (int) Request id obtained by requestDataProduct()
-            % - waitComplete @TODO
+            % - waitComplete: wait until dp finish when set to true (default)
             %
             % Returns: (struct) information of the run process
             %
             % Documentation: https://wiki.oceannetworks.ca/display/CLIBS/Data+product+download+methods
 
-            if ~exist('waitComplete','var'), waitComplete = false; end
+            if ~exist('waitComplete','var'), waitComplete = true; end
             url = sprintf('%sapi/dataProductDelivery', this.baseUrl);
             log = onc.DPLogger();
 
@@ -113,23 +106,26 @@
 
             % run timed run request
             tic
-            status = 202;
-            while status == 202
+            flag = 'queued';
+            while ~strcmp(flag,'complete') && ~strcmp(flag,'cancelled')
                 [response, info] = this.doRequest(url, filters);
                 status = info.status;
                 r.requestCount = r.requestCount + 1;
 
                 % guard against failed request
                 if util.is_failed_response(response, status)
-                    r = response;
                     throw(util.prepare_exception(status));
                 end
-
                 % repeat only if waitComplete
                 if waitComplete
                     log.printResponse(response);
-                    if status == 202, pause(this.pollPeriod); end
+                    if status == 202
+                       pause(this.pollPeriod); 
+                    end
+                else
+                    break;
                 end
+                flag = response.status;
             end
             duration = toc;
             fprintf('\n')

onc/+util/do_request.m

@@ -25,7 +25,8 @@
     % run and time request
     if showInfo, fprintf('\nRequesting URL:\n   %s\n', fullUrl); end
     tic
-    response = send(request, uri, options);
+    response = request.send(uri,options);
+
     duration = toc;
 
     % print duration
@@ -47,22 +48,14 @@
         case 202
             % Accepted, no need to print error, handle manually
             result = response.Body.Data;
-        case 400
-            % Bad request
-            result = response.Body.Data;
-            util.print_error(response, fullUrl);
-        case 401
-            % Unauthorized
-            fprintf('\nERROR 401: Unauthorized. Please verify your user token.\n')
-            result = response.Body.Data;
-        case 503
-            % Down for maintenance
-            fprintf('\nERROR 503: Service unavailable.\nWe could be down for maintenance; ');
-            fprintf('visit https://data.oceannetworks.ca for more information.\n')
-            result = struct('errors', [struct('errorCode', 503, 'message', 'Service Unavailable')]);
         otherwise
-            result = response.Body.Data;
-            fprintf('\nERROR: The request failed with HTTP error %d\n', status)
+            util.print_error(response, fullUrl);
+            if status == 400 || status == 401
+                errorStruct = response.Body.Data;
+                throw(util.prepare_exception(status, double(errorStruct.errors.errorCode)));
+            else
+                throw(util.prepare_exception(status));
+            end
     end
 
     % prepare info.size only if the response is a file, otherwise 0

onc/+util/param.m

@@ -15,13 +15,14 @@
         addOptional(p, name, defValue);
     end
 
-    if n == 2
-        % Handle single element separately to avoid problem with struct param
+    if n == 2 && isstruct(varargin{2})
+        % Handle struct param seperately
         name = names(1);
         if length(args) == 1
+            % one struct given
             varargout{1} = args{1};
         else
-            varargout{1} = varargin{2};
+            varargout{1} = varargin{2}; 
         end
     else
         % parse using inputParser

onc/+util/prepare_exception.m

@@ -1,15 +1,17 @@
-function ex = prepare_exception(status)
+function ex = prepare_exception(status, errorCode)
     %% Prepares a throwable exception object for the response
     %
     % * response: (struct) Response as returned by do_request()
-    % - status:   @TODO
+    % - status:   http response code
+    % - errorCode: optional
+    %              specific error code returned in field errors (only for 400 and 401)
     %
     % Returns: (MException) @TODO
     switch status
         case 400
-            ex = MException('onc:http400', 'HTTP 400: Invalid request parameters');
+            ex = MException(sprintf('onc:http400:error%d', errorCode), 'HTTP 400: Invalid request parameters');
         case 401
-            ex = MException('onc:http401', 'HTTP 401: Invalid token');
+            ex = MException(sprintf('onc:http401:error%d', errorCode), 'HTTP 401: Invalid token');
         case 404
             ex = MException('onc:http404', 'HTTP 404: Not found');
         case 410

onc/+util/print_error.m

@@ -4,32 +4,47 @@ function print_error(response, url)
     status = double(response.StatusCode);
 
     if status == 400
+        % Bad request
         fprintf('\nERROR 400 - Bad Request:\n  %s\n\n', url)
-        payload = response.Body.Data;
+        print_error_message(response.Body.Data);
 
-        % Make sure the payload was parsed automatically
-        if ~isstruct(payload)
-            payload = jsondecode(payload);
-        end
+    elseif status == 401
+        % Unauthorized
+        fprintf('\nERROR 401: Unauthorized. Please verify your user token.\n')
+        print_error_message(response.Body.Data)
 
-        if isfield(payload, 'errors')
-            for i = 1 : numel(payload.errors)
-                e = payload.errors(i);
-                msg = e.errorMessage;
-                parameters = e.parameter;
-                fprintf('  Parameter "%s" -> %s\n', string(parameters), msg)
-            end
-            fprintf('\n');
+    elseif status == 404
+        %Not Found
+        fprintf('\nERROR 404: Not Found \n')
 
-        elseif status == 401
-            fprintf('\nERROR 401: Unauthorized - %s\n', url)
-            fprintf('Please check that your Web Services API token is valid. Find your token in your registered profile at https://data.oceannetworks.ca.\n')
+    elseif status == 500
+        % Internal Server Error
+        fprintf('\nERROR 500: Internal Server Error - %s\n', url)
+        fprintf('The API failed to process your request. You might want to retry later in case this is a temporary issue (i.e. Maintenance).\n')
+
+    elseif status == 503
+        % Down for maintenance
+        fprintf('\nERROR 503: Service unavailable.\nWe could be down for maintenance; ');
+        fprintf('visit https://data.oceannetworks.ca for more information.\n')
+
+    else
+        fprintf('\nERROR: The request failed with HTTP error %d\n', status);
+    end
+end
 
-        elseif status == 500
-            fprintf('\nERROR 500: Internal Server Error - %s\n', url)
-            fprintf('The API failed to process your request. You might want to retry later in case this is a temporary issue (i.e. Maintenance).\n')
-        else
-            fprintf('\nERROR %d: The request failed.\n', status)
-        end
+% helper function
+function print_error_message(payload)
+    % Make sure the payload was parsed automatically
+    if ~isstruct(payload)
+        payload = jsondecode(payload);
     end
+
+    for i = 1 : numel(payload.errors)
+        e = payload.errors(i);
+        msg = e.errorMessage;
+        parameters = e.parameter;
+        fprintf('  Parameter "%s" -> %s\n', string(parameters), msg)
+    end
+    fprintf('\n');
 end
+

onc/+util/save_as_file.m

@@ -25,6 +25,7 @@
     try
         matlabVersion = version('-release');
         year = str2double(matlabVersion(1:end-1));
+
         ext = util.extractFileExtension(fileName);
 
         % if result is an image file or .xml file, use other save methods instead of fwrite. 

onc/tests/README.md

@@ -24,20 +24,23 @@ Although each test suite inherits from matlab.unittest.TestCase, each is written
 2. Use MATLAB's command window to run all tests, using the commands described below:
 
 *Running all tests:*
+    in folder api-matlab-client, run command:
+    runtests('onc/tests/suites', 'IncludeSubfolders', true);
 
-    runAllTests
-
+(if it doesn't display test results table at the end, all tests pass. You can also remove semicolon to see test results)
+    
 *Running a test suite:*
 
-    runTestSuite <NUMBER_OF_TEST_SUITE>
+    runtests(<NUMBER_OF_TEST_SUITE>)
     i.e.:
-    runTestSuite 1
+    runtests("Test01_Locations"); 
 
 *Running a test case:*
+
+    runtests(<NAME_OF_TEST_SUITE_CLASS>/<NAME_OF_CASE_METHOD>)
 
-    runTestCase <NAME_OF_TEST_SUITE_CLASS> <NAME_OF_CASE_METHOD>
     i.e.:
-    runTestCase Test01_Locations testGetAllLocations
+    runtests("Test01_Locations/testInvalidTimeRangeGreaterStartTime")
 
 
 **DEVELOPING TESTS**