User defined archive access url (local branch PR)

.github/workflows/live-preview.yaml

@@ -26,7 +26,7 @@ jobs:
     steps:
     - uses: actions/setup-python@v4
       with:
-        python-version: "3.7"
+        python-version: "3.8"
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
@@ -63,7 +63,7 @@ jobs:
       run: |
         cd deploy-preview
 
-        dir_list=`for tl in $TOOL_LIST ; do echo $tl | awk -F '/' '{print $2}'; done | uniq`
+        dir_list=`for tl in $TOOL_LIST ; do echo $tl | awk -F '/' '{for(i=2;i<=(NF-1);i++) printf("%s%s",$i,i==(NF-1)?" ":FS)}'; done | uniq`
 
         for dr in $dir_list; do
           tools_list=`for tl in $TOOL_LIST; do if [[ "$tl" == *"$dr"*  ]] ; then basename $tl ; fi ; done`

deploy-preview/deploy.yml

@@ -12,6 +12,12 @@
 
 
   tasks:
+    - name: Create tool dir
+      ansible.builtin.file:
+        path: "{{ dest_tools_dir }}/{{ tool_dir }}_pr{{ pr_num }}"
+        state: directory
+        mode: 0755
+
     - name: Sync tool dir
       ansible.posix.synchronize:
         src: "{{ src_tools_dir }}/{{ tool_dir }}/"
@@ -29,8 +35,8 @@
     - name: Make name unique
       replace:
         path: "{{ dest_tools_dir }}/{{ tool_dir }}_pr{{ pr_num }}/{{ item }}"
-        regexp: '<tool(.*)name="([a-z0-9_]+)"'
-        replace: '<tool\1name="\2_pr{{ pr_num }}"'
+        regexp: '<tool(.*)name="([^"]+)"'
+        replace: '<tool\1name="\2 (PR {{ pr_num }})"'
       loop: "{{ tools_list }}"
 
     - name: Clean previous references

environment.yml

@@ -4,4 +4,4 @@ channels:
   - defaults
 dependencies:
   - pip:
-      - planemo==0.75.7
+      - planemo==0.75.20

tools/archives/pyvo_integration/astronomical_archives.py

@@ -2,6 +2,7 @@
 import functools
 import json
 import os
+import re
 import signal
 import sys
 import urllib
@@ -17,6 +18,10 @@
 MAX_ALLOWED_ENTRIES = 100
 MAX_REGISTRIES_TO_SEARCH = 100
 
+ARCHIVES_TIMEOUT_BYPASS = [
+    "https://datalab.noirlab.edu/tap"
+]
+
 
 class TimeoutException(Exception):
     pass
@@ -217,28 +222,34 @@ def _set_archive_tables(self):
 
         self.tables = []
 
-        for table in self.archive_service.tables:
-            archive_table = {
-                'name': table.name,
-                'type': table.type,
-                'fields': None
-            }
-
-            fields = []
-
-            for table_field in table.columns:
-                field = {
-                    'name': table_field.name,
-                    'description': table_field.description,
-                    'unit': table_field.unit,
-                    'datatype': table_field.datatype.content
+        try:
+            for table in self.archive_service.tables:
+                archive_table = {
+                    'name': table.name,
+                    'type': table.type,
+                    'fields': None
                 }
 
-                fields.append(field)
+                fields = []
+
+                for table_field in table.columns:
+                    field = {
+                        'name': table_field.name,
+                        'description': table_field.description,
+                        'unit': table_field.unit,
+                        'datatype': table_field.datatype.content
+                    }
+
+                    fields.append(field)
+
+                archive_table['fields'] = fields
 
-            archive_table['fields'] = fields
+                self.tables.append(archive_table)
 
-            self.tables.append(archive_table)
+        # Exception is raised when a table schema is missing
+        # Missing table will be omitted so no action needed
+        except DALServiceError:
+            pass
 
     def _is_query_valid(self, query) -> bool:
         is_valid = True
@@ -507,6 +518,20 @@ def _set_archive(self):
             self._archives.append(
                 TapArchive(access_url=self._service_access_url))
 
+        elif self._archive_type == 'custom':
+            self._service_access_url = \
+                self._json_parameters['archive_selection']['access_url']
+
+            if Utils.is_valid_url(self._service_access_url):
+                self._archives.append(
+                    TapArchive(access_url=self._service_access_url))
+            else:
+                error_message = "archive access url is not a valid url"
+                Logger.create_action_log(
+                    Logger.ACTION_ERROR,
+                    Logger.ACTION_TYPE_ARCHIVE_CONNECTION,
+                    error_message)
+
         else:
             keyword = \
                 self._json_parameters['archive_selection']['keyword']
@@ -752,6 +777,11 @@ def run(self):
 
             for archive in self._archives:
                 try:
+
+                    if archive.access_url in ARCHIVES_TIMEOUT_BYPASS:
+                        archive.get_resources = \
+                            timeout(40)(TapArchive.get_resources.__get__(archive))
+
                     _file_url, error_message = archive.get_resources(
                         self._adql_query,
                         self._number_of_files,
@@ -1250,9 +1280,9 @@ def write_urls_to_output(urls: [], output, access_url="access_url"):
         with open(output, "w") as file_output:
             for url in urls:
                 try:
-                    file_output.write(url[access_url] + ',')
+                    file_output.write(str(url[access_url]) + ',')
                 except Exception:
-                    error_message = "url field not found for url"
+                    error_message = "url field "+access_url+" not found for url"
                     Logger.create_action_log(
                         Logger.ACTION_ERROR,
                         Logger.ACTION_TYPE_WRITE_URL,
@@ -1305,6 +1335,12 @@ def collect_resource_keys(urls_data: list) -> list:
                     resource_keys.append(key)
         return resource_keys
 
+    @staticmethod
+    def is_valid_url(url: str) -> bool:
+        regex_url = re.compile('^https?://(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,6}(?::\d+)?(?:/[^\s]*)?$')
+
+        return re.match(regex_url, url) is not None
+
 
 class Logger:
     _logs = []

tools/archives/pyvo_integration/astronomical_archives.xml

@@ -1,4 +1,4 @@
-<tool id="astronomical_archives" name="Astronomical Archives (IVOA)" version="0.9.1">
+<tool id="astronomical_archives" name="Astronomical Archives (IVOA)" version="0.9.2">
     <description>queries astronomical archives through Virtual Observatory protocols</description>
     <edam_operations>
         <edam_operation>operation_0224</edam_operation>
@@ -24,6 +24,7 @@
             <param name="archive_type" type="select" label="Archive Selection">
               <option value="archive">Query specific IVOA archive</option>
               <option value="registry">Query all matching IVOA archives</option>
+              <option value="custom">Query custom TAP archive</option>
             </param>
             <when value="registry">
               <param name="keyword" type="text" label="Keyword" />
@@ -47,6 +48,11 @@
                 <options from_data_table="astronomical_archives" />
               </param>
             </when>
+            <when value="custom">
+              <param name="access_url" type="text" label="TAP archive access url">
+                <validator type="regex" message="URL seems invalid">^https?://[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.[A-Za-z]{2,6}(:\d+)?(/[^\s]*)?$</validator>
+              </param>
+            </when>
         </conditional>
         <section name="query_section" title="Query selection" expanded="true">
           <conditional name="query_selection">
@@ -255,6 +261,40 @@
             </assert_contents>
           </output>
         </test>
+        <test expect_num_outputs="2">
+          <param name="output_selection" value="c"/>
+          <param name="number_of_files" value="1"/>
+          <conditional name="archive_selection">
+              <param name="archive_type" value="custom"/>
+              <param name="access_url" value="https://datalab.noirlab.edu/tap"/>
+          </conditional>
+          <conditional name="query_selection">
+              <param name="query_type" value="raw_query" />
+              <param name="table" value="allwise.source" />
+              <param name="url_field" value="random_id" />
+          </conditional>
+          <output name="output_csv" count="1">
+            <assert_contents>
+                <has_text_matching expression=".*\S+.*" />
+            </assert_contents>
+          </output>
+        </test>
+        <test expect_num_outputs="2">
+          <param name="output_selection" value="c"/>
+          <param name="number_of_files" value="1"/>
+          <conditional name="archive_selection">
+              <param name="archive_type" value="custom"/>
+              <param name="access_url" value="http://voparis-tap-he.obspm.fr/tap"/>
+          </conditional>
+          <conditional name="query_selection">
+              <param name="query_type" value="obscore_query" />
+          </conditional>
+          <output name="output_csv" count="1">
+            <assert_contents>
+                <has_text_matching expression=".*\S+.*" />
+            </assert_contents>
+          </output>
+        </test>
     </tests>
     <help>
 
@@ -543,7 +583,7 @@ Tables exposed through this endpoint include: epn_core from the gem_mars schema,
 
 -----
 
-**ArVO Byu TAP** http://arvo-registry.sci.am/tap ArVO Byurakan TAP service
+**ArVO Byu TAP** arvo-registry.sci.am/tap ArVO Byurakan TAP service
 
 -----
 
@@ -612,7 +652,7 @@ Illumination by the Sun of each face of the comet 67P/Churyumov-Gerasimenko base
                 CSHP_DV_130_01_LORES_OBJ.OBJ. The service provides the cosine between the normal of each face (in the same order as the faces defined in the shape model) and the Sun direction; both
                 numerical values and images of the illumination are available. Each map is defined for a given position of the Sun
                 in the frame of 67P (67P/C-G_CK). Longitude 0 is at the center of each map. The code is developed by A. Beth,
-                Imperial College London, UK and the service is provided by CDPP (http://cdpp.eu). Acknowlegment: The illumination models
+                Imperial College London, UK and the service is provided by CDPP (cdpp.eu). Acknowlegment: The illumination models
                 have been developed at the Department of Physics at Imperial College London (UK) under the financial support of STFC
                 grant of UK ST/N000692/1 and ESA contract 4000119035/16/ES/JD (Rosetta RPC-PIU). We would also like to warmly
                 thank Bernhard Geiger (ESA) for his support in validating the 2D-illumination maps.