From b08780474238fea7ef6ce3b0b3ba4796c1f11467 Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Wed, 31 Jan 2024 01:56:49 +0100 Subject: [PATCH 01/21] added custom access url input and conditional --- tools/archives/pyvo_integration/astronomical_archives.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml index 8f9a6433..bfee40fc 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.xml +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -24,6 +24,7 @@ + @@ -47,6 +48,9 @@ + + +
From fb57bb9d1a28d752936f93e2aa777b1e2c5f6cc6 Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Wed, 31 Jan 2024 02:16:40 +0100 Subject: [PATCH 02/21] Custom access url + url validation + regex fix --- .../pyvo_integration/astronomical_archives.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.py b/tools/archives/pyvo_integration/astronomical_archives.py index 17897f1c..b0826e77 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.py +++ b/tools/archives/pyvo_integration/astronomical_archives.py @@ -2,6 +2,7 @@ import functools import json import os +import re import signal import sys import urllib @@ -506,7 +507,21 @@ def _set_archive(self): self._archives.append( TapArchive(access_url=self._service_access_url)) - + + elif self._archive_type == 'custom': + self._service_access_url = \ + self._json_parameters['archive_selection']['access_url'] + + if Utils.is_valid_url(self._service_access_url): + self._archives.append( + TapArchive(access_url=self._service_access_url)) + else: + error_message = "archive access url is not a valid url" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_ARCHIVE_CONNECTION, + error_message) + else: keyword = \ self._json_parameters['archive_selection']['keyword'] @@ -1305,6 +1320,17 @@ def collect_resource_keys(urls_data: list) -> list: resource_keys.append(key) return resource_keys + @staticmethod + def is_valid_url(url: str) -> bool: + regex_url = re.compile( + r'^(?:http)s?://' + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+' + r'(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?))' + r'(?::\d+)?' + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + + return re.match(regex_url, url) is not None + class Logger: _logs = [] From c44ddc8bda2f507d3d189e800c65e3c955f0eadc Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Wed, 31 Jan 2024 11:21:28 +0100 Subject: [PATCH 03/21] enhance preview deployment --- .github/workflows/live-preview.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/live-preview.yaml b/.github/workflows/live-preview.yaml index a1fb307e..03c7ea8c 100644 --- a/.github/workflows/live-preview.yaml +++ b/.github/workflows/live-preview.yaml @@ -26,7 +26,7 @@ jobs: steps: - uses: actions/setup-python@v4 with: - python-version: "3.7" + python-version: "3.8" - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -63,7 +63,7 @@ jobs: run: | cd deploy-preview - dir_list=`for tl in $TOOL_LIST ; do echo $tl | awk -F '/' '{print $2}'; done | uniq` + dir_list=`for tl in $TOOL_LIST ; do echo $tl | awk -F '/' '{for(i=2;i<=(NF-1);i++) printf("%s%s",$i,i==(NF-1)?" ":FS)}'; done | uniq` for dr in $dir_list; do tools_list=`for tl in $TOOL_LIST; do if [[ "$tl" == *"$dr"* ]] ; then basename $tl ; fi ; done` From 5cd5a8dab90b2906b63287b9178eb800bb6a49ff Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Wed, 31 Jan 2024 11:27:12 +0100 Subject: [PATCH 04/21] deploy preview create missing dirs --- deploy-preview/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy-preview/deploy.yml b/deploy-preview/deploy.yml index 23019223..7e6ebc32 100644 --- a/deploy-preview/deploy.yml +++ b/deploy-preview/deploy.yml @@ -18,6 +18,7 @@ dest: "{{ dest_tools_dir }}/{{ tool_dir }}_pr{{ pr_num }}" rsync_opts: - "--chown=galaxy:galaxy" + - "--mkpath" - name: Make id unique replace: From af19767af2ee2270b0c40a6c5005a62a35524142 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Wed, 31 Jan 2024 11:35:42 +0100 Subject: [PATCH 05/21] explicitly create preview tool dir --- deploy-preview/deploy.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deploy-preview/deploy.yml b/deploy-preview/deploy.yml index 7e6ebc32..4937f5c1 100644 --- a/deploy-preview/deploy.yml +++ b/deploy-preview/deploy.yml @@ -12,13 +12,18 @@ tasks: + - name: Create tool dir + ansible.builtin.file: + path: "{{ dest_tools_dir }}/{{ tool_dir }}_pr{{ pr_num }}" + state: directory + mode: 0755 + - name: Sync tool dir ansible.posix.synchronize: src: "{{ src_tools_dir }}/{{ tool_dir }}/" dest: "{{ dest_tools_dir }}/{{ tool_dir }}_pr{{ pr_num }}" rsync_opts: - "--chown=galaxy:galaxy" - - "--mkpath" - name: Make id unique replace: From 82bdf72b912c08ca44dd45d12b8d3e98d4763d0e Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Wed, 31 Jan 2024 11:47:10 +0100 Subject: [PATCH 06/21] renaming cosmetics --- deploy-preview/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy-preview/deploy.yml b/deploy-preview/deploy.yml index 4937f5c1..46cae926 100644 --- a/deploy-preview/deploy.yml +++ b/deploy-preview/deploy.yml @@ -35,8 +35,8 @@ - name: Make name unique replace: path: "{{ dest_tools_dir }}/{{ tool_dir }}_pr{{ pr_num }}/{{ item }}" - regexp: ' Date: Thu, 1 Feb 2024 17:49:59 +0100 Subject: [PATCH 07/21] linting --- tools/archives/pyvo_integration/astronomical_archives.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.py b/tools/archives/pyvo_integration/astronomical_archives.py index b0826e77..69681509 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.py +++ b/tools/archives/pyvo_integration/astronomical_archives.py @@ -507,7 +507,7 @@ def _set_archive(self): self._archives.append( TapArchive(access_url=self._service_access_url)) - + elif self._archive_type == 'custom': self._service_access_url = \ self._json_parameters['archive_selection']['access_url'] @@ -521,7 +521,7 @@ def _set_archive(self): Logger.ACTION_ERROR, Logger.ACTION_TYPE_ARCHIVE_CONNECTION, error_message) - + else: keyword = \ self._json_parameters['archive_selection']['keyword'] From 255262f9e5bc4d6630a13098e46f9eb36d85b73d Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Thu, 1 Feb 2024 18:22:23 +0100 Subject: [PATCH 08/21] fixed linting (unreachable url) --- tools/archives/pyvo_integration/astronomical_archives.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml index bfee40fc..87de4c88 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.xml +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -616,7 +616,7 @@ Illumination by the Sun of each face of the comet 67P/Churyumov-Gerasimenko base CSHP_DV_130_01_LORES_OBJ.OBJ. The service provides the cosine between the normal of each face (in the same order as the faces defined in the shape model) and the Sun direction; both numerical values and images of the illumination are available. Each map is defined for a given position of the Sun in the frame of 67P (67P/C-G_CK). Longitude 0 is at the center of each map. The code is developed by A. Beth, - Imperial College London, UK and the service is provided by CDPP (http://cdpp.eu). Acknowlegment: The illumination models + Imperial College London, UK and the service is provided by CDPP (cdpp.eu). Acknowlegment: The illumination models have been developed at the Department of Physics at Imperial College London (UK) under the financial support of STFC grant of UK ST/N000692/1 and ESA contract 4000119035/16/ES/JD (Rosetta RPC-PIU). We would also like to warmly thank Bernhard Geiger (ESA) for his support in validating the 2D-illumination maps. From 405266f08a429ff483df72c1c39edeaea6b81ada Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Thu, 1 Feb 2024 21:02:39 +0100 Subject: [PATCH 09/21] update planemo version --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 80de5102..aa84baa7 100644 --- a/environment.yml +++ b/environment.yml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - pip: - - planemo==0.75.7 + - planemo==0.75.20 From 1d28059375b7b4b9e50e80a7e85d6bd7cb483e91 Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Tue, 6 Feb 2024 06:47:15 +0100 Subject: [PATCH 10/21] Updated tool version number --- tools/archives/pyvo_integration/astronomical_archives.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml index 87de4c88..080ed292 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.xml +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -1,4 +1,4 @@ - + queries astronomical archives through Virtual Observatory protocols operation_0224 From 89beafde6cc0c03078c2a9ba75b982effab27823 Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Mon, 12 Feb 2024 10:42:01 +0100 Subject: [PATCH 11/21] WIP added validator --- tools/archives/pyvo_integration/astronomical_archives.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml index 080ed292..2bd49ace 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.xml +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -49,7 +49,9 @@ - + + ^(?:http)s?://(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|)(?::\d+)?(?:/?|[/?]\S+)$ +
From e10d624be9365b610311bd9ef5bfc72357f22866 Mon Sep 17 00:00:00 2001 From: Volodymyr Date: Fri, 16 Feb 2024 16:58:45 +0100 Subject: [PATCH 12/21] Update tools/archives/pyvo_integration/astronomical_archives.xml --- tools/archives/pyvo_integration/astronomical_archives.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml index 2bd49ace..33b74417 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.xml +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -50,7 +50,7 @@ - ^(?:http)s?://(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|)(?::\d+)?(?:/?|[/?]\S+)$ + ^(?:http)s?://(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+(?:[a-zA-Z]{2,6}\.?|[a-zA-Z0-9-]{2,}\.?))(?::\d+)?(?:/?|[/?]\S+)$ From 5ea6f6aa347d4e7d6a75546e641ea27f6df477fe Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Sun, 10 Mar 2024 17:40:16 +0100 Subject: [PATCH 13/21] tests + regex --- .../astronomical_archives.xml | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml index 33b74417..03fcc084 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.xml +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -50,7 +50,7 @@ - ^(?:http)s?://(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+(?:[a-zA-Z]{2,6}\.?|[a-zA-Z0-9-]{2,}\.?))(?::\d+)?(?:/?|[/?]\S+)$ + ^https?://[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.[A-Za-z]{2,6}(:\d+)?(/[^\s]*)?$ @@ -261,6 +261,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From a70d01b1d699ff7fcdd04bc3d530e802e81a61b4 Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Sun, 10 Mar 2024 17:57:50 +0100 Subject: [PATCH 14/21] WIP regex + timeout bypass + check on archive tables initialization --- .../pyvo_integration/astronomical_archives.py | 64 +++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.py b/tools/archives/pyvo_integration/astronomical_archives.py index 69681509..e6df9c1d 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.py +++ b/tools/archives/pyvo_integration/astronomical_archives.py @@ -18,6 +18,10 @@ MAX_ALLOWED_ENTRIES = 100 MAX_REGISTRIES_TO_SEARCH = 100 +ARCHIVES_TIMEOUT_BYPASS = [ + "https://datalab.noirlab.edu/tap" +] + class TimeoutException(Exception): pass @@ -218,28 +222,34 @@ def _set_archive_tables(self): self.tables = [] - for table in self.archive_service.tables: - archive_table = { - 'name': table.name, - 'type': table.type, - 'fields': None - } - - fields = [] - - for table_field in table.columns: - field = { - 'name': table_field.name, - 'description': table_field.description, - 'unit': table_field.unit, - 'datatype': table_field.datatype.content + try: + for table in self.archive_service.tables: + archive_table = { + 'name': table.name, + 'type': table.type, + 'fields': None } - fields.append(field) + fields = [] + + for table_field in table.columns: + field = { + 'name': table_field.name, + 'description': table_field.description, + 'unit': table_field.unit, + 'datatype': table_field.datatype.content + } + + fields.append(field) + + archive_table['fields'] = fields - archive_table['fields'] = fields + self.tables.append(archive_table) - self.tables.append(archive_table) + # Exception is raised when a table schema is missing + # Missing table will be omitted so no action needed + except DALServiceError: + pass def _is_query_valid(self, query) -> bool: is_valid = True @@ -767,6 +777,11 @@ def run(self): for archive in self._archives: try: + + if archive.access_url in ARCHIVES_TIMEOUT_BYPASS: + archive.get_resources = \ + timeout(40)(TapArchive.get_resources.__get__(archive)) + _file_url, error_message = archive.get_resources( self._adql_query, self._number_of_files, @@ -1267,7 +1282,7 @@ def write_urls_to_output(urls: [], output, access_url="access_url"): try: file_output.write(url[access_url] + ',') except Exception: - error_message = "url field not found for url" + error_message = "url field "+access_url+" not found for url" Logger.create_action_log( Logger.ACTION_ERROR, Logger.ACTION_TYPE_WRITE_URL, @@ -1323,12 +1338,11 @@ def collect_resource_keys(urls_data: list) -> list: @staticmethod def is_valid_url(url: str) -> bool: regex_url = re.compile( - r'^(?:http)s?://' - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+' - r'(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?))' - r'(?::\d+)?' - r'(?:/?|[/?]\S+)$', re.IGNORECASE) - + r'^https?: // [A - Za - z0 - 9]([A - Za - z0 - 9 -]' + r'{0, 61}[A - Za - z0 - 9])?\.[A - Za - z]' + r'{2, 6}(:\d +)?(/[^ \s] *)?$' + ) + return re.match(regex_url, url) is not None From 2c464f4f8e8a74dbce2fe16bf2d79f0d1742d638 Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Sun, 10 Mar 2024 19:02:58 +0100 Subject: [PATCH 15/21] fix for file writing bug --- tools/archives/pyvo_integration/astronomical_archives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.py b/tools/archives/pyvo_integration/astronomical_archives.py index e6df9c1d..df849b9e 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.py +++ b/tools/archives/pyvo_integration/astronomical_archives.py @@ -1280,7 +1280,7 @@ def write_urls_to_output(urls: [], output, access_url="access_url"): with open(output, "w") as file_output: for url in urls: try: - file_output.write(url[access_url] + ',') + file_output.write(str(url[access_url]) + ',') except Exception: error_message = "url field "+access_url+" not found for url" Logger.create_action_log( From 5c283be887e30157f50b58eb24de24d7f9c696f0 Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Sun, 10 Mar 2024 19:30:29 +0100 Subject: [PATCH 16/21] fix for regex bug in python --- tools/archives/pyvo_integration/astronomical_archives.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.py b/tools/archives/pyvo_integration/astronomical_archives.py index df849b9e..be16dc13 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.py +++ b/tools/archives/pyvo_integration/astronomical_archives.py @@ -1337,11 +1337,7 @@ def collect_resource_keys(urls_data: list) -> list: @staticmethod def is_valid_url(url: str) -> bool: - regex_url = re.compile( - r'^https?: // [A - Za - z0 - 9]([A - Za - z0 - 9 -]' - r'{0, 61}[A - Za - z0 - 9])?\.[A - Za - z]' - r'{2, 6}(:\d +)?(/[^ \s] *)?$' - ) + regex_url = re.compile('^https?://(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,6}(?::\d+)?(?:/[^\s]*)?$') return re.match(regex_url, url) is not None From 14e4d3e8debbff54061031e7e23fe847a8f8a232 Mon Sep 17 00:00:00 2001 From: Denys Savchenko <56398430+dsavchenko@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:34:46 +0100 Subject: [PATCH 17/21] Delete .github/workflows/lint-and-test.yml --- .github/workflows/lint-and-test.yml | 37 ----------------------------- 1 file changed, 37 deletions(-) delete mode 100644 .github/workflows/lint-and-test.yml diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml deleted file mode 100644 index af128fb6..00000000 --- a/.github/workflows/lint-and-test.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Lint and Test Galaxy Tools with Planemo - -on: [push] - -jobs: - lint-and-test: - runs-on: ubuntu-latest - strategy: - max-parallel: 5 - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - name: Add conda to system path - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - - name: Install dependencies - run: | - conda env update --file environment.yml --name base - pip install planemo - - name: Lint with Planemo - run: | - for tool in tools/astropytools/*xml; do - echo "linting $tool" - planemo l $tool - done - - name: Test with Planemo - run: | - for tool in tools/astropytools/*xml; do - echo "testing $tool" - planemo t $tool - done - From 0fc01228ad987d8680582479a5c1e5cf503a6609 Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:53:51 +0100 Subject: [PATCH 18/21] remove unreachable url for linting --- tools/archives/pyvo_integration/astronomical_archives.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml index 03fcc084..aab7e4c3 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.xml +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -583,7 +583,7 @@ Tables exposed through this endpoint include: epn_core from the gem_mars schema, ----- -**ArVO Byu TAP** http://arvo-registry.sci.am/tap ArVO Byurakan TAP service +**ArVO Byu TAP** arvo-registry.sci.am/tap ArVO Byurakan TAP service ----- From 5f5ab38e35fba9d92161314370df6ec1963dd601 Mon Sep 17 00:00:00 2001 From: francoismg <46478217+francoismg@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:59:21 +0100 Subject: [PATCH 19/21] Update tools/archives/pyvo_integration/astronomical_archives.py f string instead of concat Co-authored-by: Denys Savchenko <56398430+dsavchenko@users.noreply.github.com> --- tools/archives/pyvo_integration/astronomical_archives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.py b/tools/archives/pyvo_integration/astronomical_archives.py index be16dc13..1a2fe9a2 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.py +++ b/tools/archives/pyvo_integration/astronomical_archives.py @@ -1282,7 +1282,7 @@ def write_urls_to_output(urls: [], output, access_url="access_url"): try: file_output.write(str(url[access_url]) + ',') except Exception: - error_message = "url field "+access_url+" not found for url" + error_message = f"url field {access_url} not found for url" Logger.create_action_log( Logger.ACTION_ERROR, Logger.ACTION_TYPE_WRITE_URL, From ea3736bb093615719e622777a56ee3a2317f43e7 Mon Sep 17 00:00:00 2001 From: Denys Savchenko <56398430+dsavchenko@users.noreply.github.com> Date: Mon, 25 Mar 2024 18:47:00 +0100 Subject: [PATCH 20/21] Apply suggestions from code review --- tools/archives/pyvo_integration/astronomical_archives.py | 7 +++---- tools/archives/pyvo_integration/astronomical_archives.xml | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.py b/tools/archives/pyvo_integration/astronomical_archives.py index 1a2fe9a2..7ac4f91c 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.py +++ b/tools/archives/pyvo_integration/astronomical_archives.py @@ -780,8 +780,8 @@ def run(self): if archive.access_url in ARCHIVES_TIMEOUT_BYPASS: archive.get_resources = \ - timeout(40)(TapArchive.get_resources.__get__(archive)) - + timeout(40)(TapArchive.get_resources.__get__(archive)) # noqa: E501 + _file_url, error_message = archive.get_resources( self._adql_query, self._number_of_files, @@ -1337,8 +1337,7 @@ def collect_resource_keys(urls_data: list) -> list: @staticmethod def is_valid_url(url: str) -> bool: - regex_url = re.compile('^https?://(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,6}(?::\d+)?(?:/[^\s]*)?$') - + regex_url = re.compile(r'^https?://(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,6}(?::\d+)?(?:/[^\s]*)?$') # noqa: E501 return re.match(regex_url, url) is not None diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml index aab7e4c3..2200b9e1 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.xml +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -1,4 +1,4 @@ - + queries astronomical archives through Virtual Observatory protocols operation_0224 From b6676c71fbf1838297edcd57c7093e8af5d479f5 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Mon, 25 Mar 2024 19:21:00 +0100 Subject: [PATCH 21/21] fix regex --- tools/archives/pyvo_integration/astronomical_archives.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/archives/pyvo_integration/astronomical_archives.xml b/tools/archives/pyvo_integration/astronomical_archives.xml index 2200b9e1..7e944008 100644 --- a/tools/archives/pyvo_integration/astronomical_archives.xml +++ b/tools/archives/pyvo_integration/astronomical_archives.xml @@ -50,7 +50,7 @@ - ^https?://[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.[A-Za-z]{2,6}(:\d+)?(/[^\s]*)?$ + ^https?://[A-Za-z0-9]([A-Za-z0-9-\.]{0,61}[A-Za-z0-9])?\.[A-Za-z]{2,6}(:\d+)?(/[^\s]*)?$ @@ -500,7 +500,7 @@ The Table Access Protocol (TAP) lets you execute queries against our database ta ----- -The MAST Archive at STScI TAP end point for the TESS Input Catalog.

The TIC is used to help identify two-minute cadence target selection for the TESS mission, and to calculate physical and observational properties of planet candidates. It is for use by both the TESS science team and the public, and it is periodically updated – the current version is TIC-8. TIC-8 uses the GAIA DR2 catalog as a base and merges a large number of other photometric catalogs, including 2MASS, UCAC4, APASS, SDSS, WISE, etc. There are roughly 1.5 billion stellar and extended sources in TIC-8, containing compiled magnitudes including B, V, u, g, r, i, z, J, H, K, W1-W4, and G. +The MAST Archive at STScI TAP end point for the TESS Input Catalog.The TIC is used to help identify two-minute cadence target selection for the TESS mission, and to calculate physical and observational properties of planet candidates. It is for use by both the TESS science team and the public, and it is periodically updated – the current version is TIC-8. TIC-8 uses the GAIA DR2 catalog as a base and merges a large number of other photometric catalogs, including 2MASS, UCAC4, APASS, SDSS, WISE, etc. There are roughly 1.5 billion stellar and extended sources in TIC-8, containing compiled magnitudes including B, V, u, g, r, i, z, J, H, K, W1-W4, and G. The TIC can be directly accessed through the Mikulski Archive for Space Telescopes (MAST), using either queries or bulk download. The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported.