From 9b1080bf505d8329ce1ecbc100af793a23a8bc3d Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 24 Apr 2024 16:55:33 +0200 Subject: [PATCH 01/74] test --- internships/ETN-Elias-howest/test | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 internships/ETN-Elias-howest/test diff --git a/internships/ETN-Elias-howest/test b/internships/ETN-Elias-howest/test new file mode 100644 index 00000000..e69de29b From 3c156d48c8e5c43b4ca10ccdbdd2fb676fc9fab4 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Fri, 26 Apr 2024 12:04:08 +0200 Subject: [PATCH 02/74] ETN-EliasSteyaert-stage.Rmd Adding the ETN file into my github --- .../ETN-EliasSteyaert-stage.Rmd | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd new file mode 100644 index 00000000..0bdfef54 --- /dev/null +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -0,0 +1,71 @@ +--- +title: "ETN_EliasSteyaert-stage" +author: "Elias Steyaert" +date: "2024-04-23" +output: pdf_document +--- + +# Electronic traineeship notebook (ETN) - Elias Steyaert +### A logbook that describes what happened on a daily basis. +### Link to my github: +https://github.com/EliasSteyaert/stage-VIB-irefindex + +## Week 1 +#### Monday 22/04/2023 + +##### Exploring the main project for the following weeks: +- Explore the github repository of the project (https://github.com/vibbits/irefindex) +- Read article about the iRefIndex (Razick, S., Magklaras, G., & Donaldson, I. M. (2008). iRefIndex: A consolidated protein interaction database with provenance. BMC Bioinformatics, 9(1), 405. https://doi.org/10.1186/1471-2105-9-405) +- Read documentation on the iRefIndex (https://irefindex.vib.be/wiki/index.php/iRefIndex) +- Explore the scripts to be used + +##### Preparation for the project: +- Making an account on csv +- Downloading and installing the needed programs and having a look at how to use these (ansible, terraform and nextflow). + + +#### Tuesday 23/04/2023 + +##### Exploring the main project: +- Explore the github repository of the project even further (https://github.com/vibbits/irefindex) + +##### Preparation for the project: +- Getting the csv account fully setup after getting it approved (with a ssh key) +- Trying to get past the “13000 gate” with VPN’s. (Setting up a KULeuven- and a UGent-VPN) +- Learning more about ansible and terraform through tutorials and "how-to's" +- Exploring the ansible code with the newly requiered knowledge from earlier. + + +#### Wednesday 24/04/2023 + +##### Trying to setup the needed resources through terraform: +- Setting up the vsc connection (solving troubleshooting for errors for “terraform init” in the VM +- Setting up the identifcation credentials on the VSC_2021_102 cloud +- Running into troubleshooting while deleting the previous instance and volume + +##### Setting up a github repository +- Forking and cloning the VIB github repository into my own and setting up the needed folders/files into it (https://github.com/EliasSteyaert/stage-VIB-irefindex) + + + +#### Thursday 25/04/2023 + +##### Performing the "terraform-step" of the procedure: +- Running into troubleshooting for the whole day when trying to perform the command "terraform apply -auto-approve" +- Been getting a lot of different errors and finding different solutions which don't work + +##### side-tasks to do while waiting for the : +- Making the ETN Markdown file and complement it with the things that are performed till this day +- Making a troubleshooting file and filling in all the issueas that were encountered so far +- + + + +#### Friday 26/04/2023 +###### Exploration of the Complex Portal +- Follow a online tutorial concerning a quick tour in the Complex Portal (https://www.ebi.ac.uk/training/online/courses/complex-portal-quick-tour/) +- Look at a webinar about the Complex Portal (https://www.ebi.ac.uk/training/events/introduction-complex-portal/) +###### Visualize the number of unique proteins per complex: +- Make sure to only work with the non-redundant list of subunits in each complex (Human_NonRedundant.py) +- Make a plot visualizing the data in both Python and R (HumanComplexes.R, onlyHumanComplexes.py) + From df9ecab9550222a946568f8b675c9358d4a365da Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Mon, 29 Apr 2024 14:53:39 +0200 Subject: [PATCH 03/74] adding a file with the google cloud IP adress in --- vsc/inventory | 1 + 1 file changed, 1 insertion(+) create mode 100644 vsc/inventory diff --git a/vsc/inventory b/vsc/inventory new file mode 100644 index 00000000..f53d39cd --- /dev/null +++ b/vsc/inventory @@ -0,0 +1 @@ +35.240.84.94 From 6b708cdd1a3101854605a34678881104052e90a7 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 30 Apr 2024 09:10:44 +0200 Subject: [PATCH 04/74] changing some directories --- vsc/ansible/irinit.yml | 4 ++-- vsc/ansible/irunpack.yml | 2 +- vsc/ansible/vars/general.yml | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/vsc/ansible/irinit.yml b/vsc/ansible/irinit.yml index 0b818f95..1896eecf 100644 --- a/vsc/ansible/irinit.yml +++ b/vsc/ansible/irinit.yml @@ -109,8 +109,8 @@ state: directory mode: "0755" with_items: - - /home/irefindex/var/lib/irdata/data - - /home/irefindex/var/log/irdata + - "{{ download_data }}var/lib/irdata/data" + - "{{ download_data }}var/log/irdata" - name: Run irinit ansible.builtin.command: ./irinit --init diff --git a/vsc/ansible/irunpack.yml b/vsc/ansible/irunpack.yml index 12bc9bf1..5309586d 100644 --- a/vsc/ansible/irunpack.yml +++ b/vsc/ansible/irunpack.yml @@ -9,7 +9,7 @@ vars: log_location: "{{ logs_dir }}/irunpack" - data_dir: "/home/irefindex/var/lib/irdata/data" + data_dir: " {{download_data}}data/var/lib/irdata/data" tasks: - name: Install unzip diff --git a/vsc/ansible/vars/general.yml b/vsc/ansible/vars/general.yml index b8d1fd09..a130719a 100644 --- a/vsc/ansible/vars/general.yml +++ b/vsc/ansible/vars/general.yml @@ -2,3 +2,4 @@ release_major: "19" release_minor: "0" release: "{{ release_major }}.{{ release_minor }}" data: "/dataext/irdata{{ release_major }}" +download_data: "/mnt/disks/data/" From 70a1a10e2dda4463a1a3cf6b48cb0763b22fc302 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 30 Apr 2024 14:30:12 +0200 Subject: [PATCH 05/74] correcting the usage of a variable --- vsc/ansible/irunpack.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vsc/ansible/irunpack.yml b/vsc/ansible/irunpack.yml index 5309586d..68dd27bd 100644 --- a/vsc/ansible/irunpack.yml +++ b/vsc/ansible/irunpack.yml @@ -6,10 +6,11 @@ vars_files: - ./vars/logs.yml - ./vars/sources.yml + - ./vars/general.yml vars: log_location: "{{ logs_dir }}/irunpack" - data_dir: " {{download_data}}data/var/lib/irdata/data" + data_dir: "{{ download_data }}var/lib/irdata/data" tasks: - name: Install unzip From aaca91b32fa2e00ab45936984261efe89347c582 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 30 Apr 2024 14:31:12 +0200 Subject: [PATCH 06/74] adding a new variable DATAPREFIX --- usr/bin/irdata-config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usr/bin/irdata-config b/usr/bin/irdata-config index 731059af..7d0bf1d0 100755 --- a/usr/bin/irdata-config +++ b/usr/bin/irdata-config @@ -12,10 +12,11 @@ RELEASE="${RELEASE_MAJOR}.${RELEASE_MINOR}" # iRefIndex release identifier # General configuration settings for irdata. # (SYSPREFIX and EMAIL are most likely to need changing.) -SYSPREFIX="/home/irefindex" # system-wide installation root +DATAPREFIX="/mnt/disks/data/" # start of the user defined data directory location +SYSPREFIX="/mnt/disks/data" # system-wide installation root EMAIL="alexander.botzki@vib.be" # e-mail address for anonymous FTP logins DATABASE="irdata${RELEASE_MAJOR}" # database name -DATA="/dataext/irdata${RELEASE_MAJOR}" # user defined data directory location +DATA="${DATAPREFIX}dataext/irdata${RELEASE_MAJOR}" # user defined data directory location WIKIFORMAT="MediaWiki" # format for generated Wiki pages CURL_OPT="-L" # Follow HTTP redirects WGET_WAIT="5" # time to wait between wget requests From cecf7e11d1e4dd467d08574c23ca12167fd72025 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 30 Apr 2024 14:51:37 +0200 Subject: [PATCH 07/74] correcting a directory --- usr/bin/irdata-config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/bin/irdata-config b/usr/bin/irdata-config index 7d0bf1d0..f2ff2776 100755 --- a/usr/bin/irdata-config +++ b/usr/bin/irdata-config @@ -13,7 +13,7 @@ RELEASE="${RELEASE_MAJOR}.${RELEASE_MINOR}" # iRefIndex release identifier # (SYSPREFIX and EMAIL are most likely to need changing.) DATAPREFIX="/mnt/disks/data/" # start of the user defined data directory location -SYSPREFIX="/mnt/disks/data" # system-wide installation root +SYSPREFIX="/home/irefindex" # system-wide installation root EMAIL="alexander.botzki@vib.be" # e-mail address for anonymous FTP logins DATABASE="irdata${RELEASE_MAJOR}" # database name DATA="${DATAPREFIX}dataext/irdata${RELEASE_MAJOR}" # user defined data directory location @@ -761,7 +761,7 @@ fi # Data directory. -SYSDATA="$SYSPREFIX/var/lib/irdata/data" +SYSDATA="$SYSPREFIX/var/lib/irdata/" if [ "$DATA" ] && [ -e "$DATA" ]; then : From b5f8e4ff9e4ac63f2cda95175b18f430843cd68d Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Thu, 2 May 2024 16:51:47 +0200 Subject: [PATCH 08/74] adding 2 markers, adding and updating (still needs automatisation for the making of the DATA folder) --- usr/bin/irdata-config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/usr/bin/irdata-config b/usr/bin/irdata-config index f2ff2776..479ecbd9 100755 --- a/usr/bin/irdata-config +++ b/usr/bin/irdata-config @@ -761,8 +761,8 @@ fi # Data directory. -SYSDATA="$SYSPREFIX/var/lib/irdata/" - +SYSDATA="$SYSPREFIX/var/lib/irdata/data" +echo "marker1 $DATA" if [ "$DATA" ] && [ -e "$DATA" ]; then : elif [ -e "$SCRIPTS/../data" ]; then @@ -773,6 +773,8 @@ else DATA="$SYSDATA" fi +echo "marker $DATA" + if [ ! -e "$DATA" ] && [ ! "$INSPECTONLY" ]; then echo "Cannot locate the irdata data directory." 1>&2 exit 1 From c7fa1b4e6d4b7b8b778f6312912c7af2113b7558 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Thu, 2 May 2024 16:52:59 +0200 Subject: [PATCH 09/74] updating the data_dir variable and creating an empty existing_dirs array --- vsc/ansible/irunpack.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/vsc/ansible/irunpack.yml b/vsc/ansible/irunpack.yml index 68dd27bd..0b1f1051 100644 --- a/vsc/ansible/irunpack.yml +++ b/vsc/ansible/irunpack.yml @@ -10,7 +10,7 @@ vars: log_location: "{{ logs_dir }}/irunpack" - data_dir: "{{ download_data }}var/lib/irdata/data" + data_dir: "{{ download_data }}dataext/irdata{{ release_major }}" tasks: - name: Install unzip @@ -45,12 +45,20 @@ recurse: true register: all_directories + - name: Initialise existing_dirs + ansible.builtin.set_fact: + existing_dirs: [] + - name: Convert directories to uniform format ansible.builtin.set_fact: existing_dirs: "{{ existing_dirs | default([]) + [item.path | regex_replace('_', '-') | lower | basename] }}" no_log: true loop: "{{ all_directories.files }}" + - name: wtf as in existing dirs + ansible.builtin.debug: + msg: "existing_dirs {{ existing_dirs is defined }}" + - name: Gather all directory contents ansible.builtin.find: paths: "{{ item.path }}" @@ -69,7 +77,6 @@ - name: Store all missing directories ansible.builtin.set_fact: failed_dirs: "{{ failed_dirs | default([]) + [item.name | lower] }}" - no_log: true when: item.name | regex_replace('_', '-') | lower not in existing_dirs loop: "{{ jobs }}" From 22fa4adb49cc6ba6df5dcac9d70c8d1078541a82 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Sat, 4 May 2024 13:50:27 +0200 Subject: [PATCH 10/74] a new variable data2 --- vsc/ansible/vars/general.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/vsc/ansible/vars/general.yml b/vsc/ansible/vars/general.yml index a130719a..3cf4734d 100644 --- a/vsc/ansible/vars/general.yml +++ b/vsc/ansible/vars/general.yml @@ -3,3 +3,4 @@ release_minor: "0" release: "{{ release_major }}.{{ release_minor }}" data: "/dataext/irdata{{ release_major }}" download_data: "/mnt/disks/data/" +data2: "dataext/irdata{{ release_major }}" From 85aa31f7a2ef75c2311d0e37c767540a1f681bd0 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Sat, 4 May 2024 13:51:07 +0200 Subject: [PATCH 11/74] commenting the jobs that can't be run --- vsc/ansible/vars/sources.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vsc/ansible/vars/sources.yml b/vsc/ansible/vars/sources.yml index 9255095f..cd7d5f56 100644 --- a/vsc/ansible/vars/sources.yml +++ b/vsc/ansible/vars/sources.yml @@ -41,12 +41,12 @@ # import_delay[default=5]: Delay in seconds between the checks jobs: - { name: "athaliana" } - - { name: "bar" } + # - { name: "bar" } - { name: "bind" } - { name: "dig", import_async: 300, import_delay: 30, import_retries: 10 } - { name: "fly" } - { name: "innatedb" } - - { name: "ipi" } + # - { name: "ipi" } - { name: "mppi", import_async: 160, import_delay: 10, import_retries: 16 } - { name: "psi_mi", import_async: 160, import_delay: 10, import_retries: 16 } - { name: "quickgo", parse_async: 120, parse_delay: 10, import_async: 160, import_delay: 10, import_retries: 16 } @@ -67,8 +67,8 @@ jobs: - { name: "intcomplex", unpack_async: 60, unpack_delay: 10, parse_async: 350, parse_delay: 30, parse_retries: 12, import_async: 400, import_delay: 30, import_retries: 14 } - { name: "huri", download_runner_async: 60, download_retries: 36, parse_async: 240, parse_retries: 16, parse_delay: 15, import_async: 400, import_delay: 30, import_retries: 14 } - { name: "biogrid", unpack_async: 120, unpack_retries: 8, unpack_delay: 15, download_runner_async: 60, download_retries: 12, download_delay: 15, parse_async: 4400, parse_retries: 15, parse_delay: 300, import_async: 5000, import_retries: 17, import_delay: 300 } - - { name: "corum", download_runner_async: 60, download_retries: 12, download_delay: 15 } - - { name: "matrixdb", download_runner_async: 300, download_retries: 30, download_delay: 30, parse_async: 200, parse_retries: 8, parse_delay: 25, import_async: 160, import_delay: 10, import_retries: 16 } + # - { name: "corum", download_runner_async: 60, download_retries: 12, download_delay: 15 } + # - { name: "matrixdb", download_runner_async: 300, download_retries: 30, download_delay: 30, parse_async: 200, parse_retries: 8, parse_delay: 25, import_async: 160, import_delay: 10, import_retries: 16 } - { name: "uniprotpp", download_runner_async: 300, download_retries: 30, download_delay: 30, parse_async: 120, import_async: 160, import_delay: 10, import_retries: 16 } # - { name: "uniprot", download_runner_async: 2000, download_retries: 60, download_delay: 100, parse_async: 36000, parse_retries: 60, parse_delay: 600 } - { name: "mint", download_runner_async: 600, download_retries: 60, download_delay: 30, parse_async: 120, import_async: 400, import_delay: 30, import_retries: 14 } From 26fbb9e06539d17aa20ab56fe1b91abb41d924f2 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Sat, 4 May 2024 13:51:59 +0200 Subject: [PATCH 12/74] adding code to create a venv --- vsc/ansible/irparse.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 02ff6ccd..10da9e2e 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -6,20 +6,27 @@ vars_files: - ./vars/logs.yml - ./vars/sources.yml + - ./vars/general.yml vars: log_location: "{{ logs_dir }}/irparse" - tasks: - - name: Install irdata module - ansible.builtin.pip: - name: irdata + tasks: + - name: Install python3-venv package + ansible.builtin.apt: + name: python3-venv state: present + - name: Create virtual environment + ansible.builtin.command: + cmd: python3 -m venv "{{ download_data }}{{ data2 }}/virtualenv" + - name: Convert the downloaded data to files for each source ansible.builtin.command: "./irparse {{ item.name | upper }}" args: chdir: "/home/irefindex/usr/bin" + environment: + python_interpreter: "{{ download_data }}{{ data2 }}/virtualenv/bin/python" changed_when: false poll: 0 async: "{{ item.parse_async | default(60) }}" From f1002912d15c8b2a0c6f694b9b97be155659519f Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Mon, 6 May 2024 10:22:25 +0200 Subject: [PATCH 13/74] adding the USE_PYTHON_INTERPRETER variable --- usr/bin/irparse-source | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usr/bin/irparse-source b/usr/bin/irparse-source index 99a2e28e..3567a6ed 100755 --- a/usr/bin/irparse-source +++ b/usr/bin/irparse-source @@ -23,6 +23,8 @@ else . 'irdata-config' fi +USE_PYTHON_INTERPRETER=${python_interpreter:/usr/bin/python3} + if [ "$1" = '--help' ]; then cat 1>&2 < @@ -173,7 +175,7 @@ for FILEPATTERN in $FILEPATTERNS; do echo "$TOOLS/$PARSER" "$OUTPUTDIR" $FILES if [ ! "$NO_PARSE" ]; then if [ "$XMLFILEPATTERN" ]; then - if ! "$TOOLS/irdata_xml2tab.py" "$OUTPUTDIR" "$SOURCE" $FILES ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_xml2tab.py" "$OUTPUTDIR" "$SOURCE" $FILES ; then echo "$PROGNAME: XML processing for source $SOURCE failed." 1>&2 exit 1 fi From 794f69c90b898d3d02fca36060bb0e6a97add0a0 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Mon, 6 May 2024 14:22:37 +0200 Subject: [PATCH 14/74] adjusting the variable USE_PYTHON_INTERPRETER --- usr/bin/irparse-source | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/irparse-source b/usr/bin/irparse-source index 3567a6ed..e46a3bd4 100755 --- a/usr/bin/irparse-source +++ b/usr/bin/irparse-source @@ -23,7 +23,7 @@ else . 'irdata-config' fi -USE_PYTHON_INTERPRETER=${python_interpreter:/usr/bin/python3} +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} if [ "$1" = '--help' ]; then cat 1>&2 < Date: Tue, 7 May 2024 10:09:38 +0200 Subject: [PATCH 15/74] Update ETN-EliasSteyaert-stage.Rmd --- .../ETN-EliasSteyaert-stage.Rmd | 74 +++++++++++++++++-- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index 0bdfef54..b81bf7c5 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -54,18 +54,76 @@ https://github.com/EliasSteyaert/stage-VIB-irefindex - Running into troubleshooting for the whole day when trying to perform the command "terraform apply -auto-approve" - Been getting a lot of different errors and finding different solutions which don't work -##### side-tasks to do while waiting for the : +##### side-tasks to do while waiting for the task to be executed: - Making the ETN Markdown file and complement it with the things that are performed till this day - Making a troubleshooting file and filling in all the issueas that were encountered so far -- #### Friday 26/04/2023 -###### Exploration of the Complex Portal -- Follow a online tutorial concerning a quick tour in the Complex Portal (https://www.ebi.ac.uk/training/online/courses/complex-portal-quick-tour/) -- Look at a webinar about the Complex Portal (https://www.ebi.ac.uk/training/events/introduction-complex-portal/) -###### Visualize the number of unique proteins per complex: -- Make sure to only work with the non-redundant list of subunits in each complex (Human_NonRedundant.py) -- Make a plot visualizing the data in both Python and R (HumanComplexes.R, onlyHumanComplexes.py) + +##### Trying to perform "terraform apply" +- Trying to solve the problems that occur when performing “terraform apply”, trying to get the instance and its volume working. + +##### Learning more about the Vlaamse Super Computer +- Learning more about instances and their volumes, what they are, how thet are used. +Making an instance and a volume manually and connecting them with eachother. +Connecting to the instance. + +##### Playing the first playbook + + +#### Monday 29/04/2024 + +##### Running the playbook +- Run the playbook untill I get an error that needs to be solved. (main1 and after_main1 could be ran, main2 gave problems). + +##### Solving minor problems +- Deleting all the data in google cloud. +- Making a new directory to store everything in. + + +#### Tuesday 30/04/2024 + +##### Problemsoloving the Irdownload playbook +- Irdownload doesn’t work anymore so we will be adjusting the irdata-config so it can work again. + + +#### Wednesday 01/05/2024 + +##### Labours day + + +#### Thursday 02/05/2024 + +##### Problemsolving the Irunpack playbook +- Running into problems when performing the Irunpack playbook, existing_dirs wasn’t defined and the location of the files were wrong + +##### Problemsolving the Irparse playbook +- Running into problems when performing the Irparse playbook, pip3 has to be installed with apt, not dnf + + +#### Friday 03/05/2024 + +##### Problemsolving new problems the Irparse playbook brings with it +- Running into new problems while running the irparse playbook: creating a venv and trying to get the irdata package downloaded + + +#### Monday 06/05/2024 + +##### Problemsolving new problems the Irparse playbook brings with it +- Running into different kinds of problems when running the irparse playbook. The error says that the space is full but that doesn’t seem to be the problem. + + + + + + + + + + + + + From 897116455728b7508f36755879a728a5a53b247a Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 7 May 2024 15:48:29 +0200 Subject: [PATCH 16/74] changed the cloning the github part to copying my own files to use on the server --- vsc/ansible/irinit.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vsc/ansible/irinit.yml b/vsc/ansible/irinit.yml index 1896eecf..12d3c118 100644 --- a/vsc/ansible/irinit.yml +++ b/vsc/ansible/irinit.yml @@ -7,6 +7,10 @@ - ./vars/general.yml - ./vars/logs.yml + vars: + local_source_directory: "/home/guest/internship/stage-VIB-irefindex" + remote_destination_directory: "/home/irefindex" + tasks: - name: Install dependencies ansible.builtin.apt: @@ -98,10 +102,10 @@ name: git state: present - - name: Clone iRefIndex repository - ansible.builtin.git: - repo: https://github.com/vibbits/irefindex.git - dest: /home/irefindex + - name: Copy the irefindex files from my local machine to the remote machine + ansible.builtin.copy: + src: "{{ local_source_directory }}" + dest: "{{ remote_destination_directory }}" - name: Create directories for irdata data ansible.builtin.file: From c6a29833950f06eda4fcb29f21f2edff7a43357d Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 7 May 2024 15:49:16 +0200 Subject: [PATCH 17/74] adding an extra backslash (since it doesn't matter) --- vsc/ansible/vars/general.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vsc/ansible/vars/general.yml b/vsc/ansible/vars/general.yml index 3cf4734d..6ef3a23d 100644 --- a/vsc/ansible/vars/general.yml +++ b/vsc/ansible/vars/general.yml @@ -3,4 +3,5 @@ release_minor: "0" release: "{{ release_major }}.{{ release_minor }}" data: "/dataext/irdata{{ release_major }}" download_data: "/mnt/disks/data/" -data2: "dataext/irdata{{ release_major }}" +data2: "/dataext/irdata{{ release_major }}" +home_dir: "/home/elias.steyaert" From 648b8b3d824df40b59b7c3344523f13e8e6ef745 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 7 May 2024 15:50:46 +0200 Subject: [PATCH 18/74] added an echo message to see where the USE_PYTHON_INTERPRETER sourced from --- usr/bin/irparse-source | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/bin/irparse-source b/usr/bin/irparse-source index e46a3bd4..d9aebb94 100755 --- a/usr/bin/irparse-source +++ b/usr/bin/irparse-source @@ -24,6 +24,7 @@ else fi USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} +echo "$USE_PYTHON_INTERPRETER" if [ "$1" = '--help' ]; then cat 1>&2 < Date: Tue, 7 May 2024 15:51:44 +0200 Subject: [PATCH 19/74] changing a backslash (minor unimportant detail) --- vsc/ansible/irunpack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vsc/ansible/irunpack.yml b/vsc/ansible/irunpack.yml index 0b1f1051..b363ca43 100644 --- a/vsc/ansible/irunpack.yml +++ b/vsc/ansible/irunpack.yml @@ -10,7 +10,7 @@ vars: log_location: "{{ logs_dir }}/irunpack" - data_dir: "{{ download_data }}dataext/irdata{{ release_major }}" + data_dir: "{{ download_data }}{{ data2 }}" tasks: - name: Install unzip From ad00f23844c61d1272f972bd83b50d458db26d1d Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 7 May 2024 16:02:04 +0200 Subject: [PATCH 20/74] added the python_interpreter as an environment --- vsc/ansible/irparse.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 10da9e2e..3b960e1b 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -12,6 +12,14 @@ log_location: "{{ logs_dir }}/irparse" tasks: + - name: Set permissions on Ansible temporary directory + ansible.builtin.file: + path: "{{ home_dir }}/.ansible/tmp/" + state: directory + mode: "0755" + recurse: yes + become: yes + - name: Install python3-venv package ansible.builtin.apt: name: python3-venv From d84ca62c34af7b2d85dee723934e939ac6d59436 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 8 May 2024 14:42:50 +0200 Subject: [PATCH 21/74] trying to find a solution for the error msgs --- vsc/ansible/irparse.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 3b960e1b..b8bee28d 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -19,7 +19,7 @@ mode: "0755" recurse: yes become: yes - + - name: Install python3-venv package ansible.builtin.apt: name: python3-venv @@ -27,14 +27,14 @@ - name: Create virtual environment ansible.builtin.command: - cmd: python3 -m venv "{{ download_data }}{{ data2 }}/virtualenv" + cmd: python3 -m venv "{{ download_data }}/{{ data2 }}/virtualenv" - name: Convert the downloaded data to files for each source ansible.builtin.command: "./irparse {{ item.name | upper }}" args: chdir: "/home/irefindex/usr/bin" environment: - python_interpreter: "{{ download_data }}{{ data2 }}/virtualenv/bin/python" + python_interpreter: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python" changed_when: false poll: 0 async: "{{ item.parse_async | default(60) }}" @@ -44,13 +44,24 @@ - name: Wait for all jobs to complete ansible.builtin.async_status: jid: "{{ item.ansible_job_id }}" + environment: + python_interpreter: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python" register: job_result until: job_result.finished ignore_errors: true retries: "{{ item.item.parse_retries | default(12) }}" delay: "{{ item.item.parse_delay | default(5) }}" loop: "{{ job_result.results }}" + become: yes + + - name: Debug job_result + ansible.builtin.debug: + var: job_result + - name: df -h + ansible.builtin.command: "df -h" + register: disk_space + - name: "Ensure that `log_location` exists" ansible.builtin.file: path: "{{ log_location }}" From 7080461629814e9bdb570fc2717809066662d3d1 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 10 May 2024 09:19:56 +0200 Subject: [PATCH 22/74] Added code to put the pyhton interpreter before in the scripts --- vsc/ansible/irparse.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index b8bee28d..553ee229 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -28,6 +28,22 @@ - name: Create virtual environment ansible.builtin.command: cmd: python3 -m venv "{{ download_data }}/{{ data2 }}/virtualenv" + + - name: Find files containing "TOOLS" + find: + paths: ./usr/bin/ + patterns: "*" + recurse: yes + register: found_files + + - name: Iterate over found files + vars: + use_python_interpreter: "{{ USE_PYTHON_INTERPRETER }}" + ansible.builtin.shell: | + for filename in {{ found_files.files | map(attribute='path') | join(' ') }}; do sed "s|\(\$TOOLS/.*.py\)|{{ use_python_interpreter }}\" \"\1|g" "{{ filename }}" > "{{ filename }}.tmp" && mv "{{ filename }}.tmp" "{{ filename }}"; done + args: + executable: /bin/bash + when: found_files.matched > 0 - name: Convert the downloaded data to files for each source ansible.builtin.command: "./irparse {{ item.name | upper }}" From a6c7c99f19b784bf5e8c93a344590074353d5e77 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 10 May 2024 16:01:18 +0200 Subject: [PATCH 23/74] adding the python interpreter before the --- usr/bin/irbuild | 62 +++++++++++++------------- usr/bin/irbuild-output-resources | 8 ++-- usr/bin/irbuild2 | 42 ++++++++--------- usr/bin/irbuild3 | 8 ++-- usr/bin/irbuild4 | 56 +++++++++++------------ usr/bin/ircanonical | 8 ++-- usr/bin/irdata-config | 2 +- usr/bin/irdata-show-settings | 0 usr/bin/irdata_archive_irefscape.sh | 2 +- usr/bin/irdata_fetch_refseq.sh | 4 +- usr/bin/irdata_index_irefscape_data.sh | 2 +- usr/bin/irdata_parse_A_thaliana.sh | 2 +- usr/bin/irdata_parse_apid.sh | 2 +- usr/bin/irdata_parse_bar.sh | 2 +- usr/bin/irdata_parse_bhfucl.sh | 2 +- usr/bin/irdata_parse_bind.sh | 8 ++-- usr/bin/irdata_parse_dig.sh | 2 +- usr/bin/irdata_parse_dip.sh | 2 +- usr/bin/irdata_parse_fly.sh | 2 +- usr/bin/irdata_parse_gene.sh | 0 usr/bin/irdata_parse_gene_info.sh | 4 +- usr/bin/irdata_parse_genpept.sh | 2 +- usr/bin/irdata_parse_hpidb.sh | 2 +- usr/bin/irdata_parse_huri.sh | 2 +- usr/bin/irdata_parse_imex.sh | 2 +- usr/bin/irdata_parse_innatedb.sh | 2 +- usr/bin/irdata_parse_ipi.sh | 6 +-- usr/bin/irdata_parse_matrixdb.sh | 2 +- usr/bin/irdata_parse_mbinfo.sh | 2 +- usr/bin/irdata_parse_mint.sh | 2 +- usr/bin/irdata_parse_molcon.sh | 2 +- usr/bin/irdata_parse_mpidb.sh | 2 +- usr/bin/irdata_parse_pdb.sh | 2 +- usr/bin/irdata_parse_quickgo.sh | 2 +- usr/bin/irdata_parse_reactome.sh | 2 +- usr/bin/irdata_parse_refseq.sh | 2 +- usr/bin/irdata_parse_spike.sh | 2 +- usr/bin/irdata_parse_uniprot.sh | 4 +- usr/bin/irdata_parse_uniprotpp.sh | 2 +- usr/bin/irdata_parse_virushost.sh | 2 +- usr/bin/irdata_process_signatures.sh | 2 +- usr/bin/irdata_split_irefscape_data.sh | 0 usr/bin/irdrop | 36 +++++++-------- usr/bin/irefpro.sh | 2 +- usr/bin/irimport | 2 +- usr/bin/irimport-source | 4 +- usr/bin/irinit | 34 +++++++------- usr/bin/irmanifest | 2 +- usr/bin/irmissing | 6 +-- usr/bin/iroutput | 8 ++-- usr/bin/irparse-source | 4 +- usr/bin/irprevious | 8 ++-- usr/bin/irprevious-irefweb | 2 +- vsc/ansible/irinit.yml | 9 ++-- vsc/ansible/irparse.yml | 2 +- vsc/ansible/main2.yml | 2 +- vsc/inventory | 2 +- vsc/tryout_replacingTOOLS.yml | 22 +++++++++ 58 files changed, 217 insertions(+), 194 deletions(-) mode change 100755 => 100644 usr/bin/irbuild mode change 100755 => 100644 usr/bin/irbuild-output-resources mode change 100755 => 100644 usr/bin/irbuild2 mode change 100755 => 100644 usr/bin/irbuild3 mode change 100755 => 100644 usr/bin/irbuild4 mode change 100755 => 100644 usr/bin/ircanonical mode change 100755 => 100644 usr/bin/irdata-config mode change 100755 => 100644 usr/bin/irdata-show-settings mode change 100755 => 100644 usr/bin/irdata_archive_irefscape.sh mode change 100755 => 100644 usr/bin/irdata_fetch_refseq.sh mode change 100755 => 100644 usr/bin/irdata_index_irefscape_data.sh mode change 100755 => 100644 usr/bin/irdata_parse_A_thaliana.sh mode change 100755 => 100644 usr/bin/irdata_parse_apid.sh mode change 100755 => 100644 usr/bin/irdata_parse_bar.sh mode change 100755 => 100644 usr/bin/irdata_parse_bhfucl.sh mode change 100755 => 100644 usr/bin/irdata_parse_bind.sh mode change 100755 => 100644 usr/bin/irdata_parse_dig.sh mode change 100755 => 100644 usr/bin/irdata_parse_dip.sh mode change 100755 => 100644 usr/bin/irdata_parse_fly.sh mode change 100755 => 100644 usr/bin/irdata_parse_gene.sh mode change 100755 => 100644 usr/bin/irdata_parse_gene_info.sh mode change 100755 => 100644 usr/bin/irdata_parse_genpept.sh mode change 100755 => 100644 usr/bin/irdata_parse_hpidb.sh mode change 100755 => 100644 usr/bin/irdata_parse_huri.sh mode change 100755 => 100644 usr/bin/irdata_parse_imex.sh mode change 100755 => 100644 usr/bin/irdata_parse_innatedb.sh mode change 100755 => 100644 usr/bin/irdata_parse_ipi.sh mode change 100755 => 100644 usr/bin/irdata_parse_matrixdb.sh mode change 100755 => 100644 usr/bin/irdata_parse_mbinfo.sh mode change 100755 => 100644 usr/bin/irdata_parse_mint.sh mode change 100755 => 100644 usr/bin/irdata_parse_molcon.sh mode change 100755 => 100644 usr/bin/irdata_parse_mpidb.sh mode change 100755 => 100644 usr/bin/irdata_parse_pdb.sh mode change 100755 => 100644 usr/bin/irdata_parse_quickgo.sh mode change 100755 => 100644 usr/bin/irdata_parse_reactome.sh mode change 100755 => 100644 usr/bin/irdata_parse_refseq.sh mode change 100755 => 100644 usr/bin/irdata_parse_spike.sh mode change 100755 => 100644 usr/bin/irdata_parse_uniprot.sh mode change 100755 => 100644 usr/bin/irdata_parse_uniprotpp.sh mode change 100755 => 100644 usr/bin/irdata_parse_virushost.sh mode change 100755 => 100644 usr/bin/irdata_process_signatures.sh mode change 100755 => 100644 usr/bin/irdata_split_irefscape_data.sh mode change 100755 => 100644 usr/bin/irdrop mode change 100755 => 100644 usr/bin/irefpro.sh mode change 100755 => 100644 usr/bin/irimport mode change 100755 => 100644 usr/bin/irimport-source mode change 100755 => 100644 usr/bin/irinit mode change 100755 => 100644 usr/bin/irmanifest mode change 100755 => 100644 usr/bin/irmissing mode change 100755 => 100644 usr/bin/iroutput mode change 100755 => 100644 usr/bin/irparse-source mode change 100755 => 100644 usr/bin/irprevious mode change 100755 => 100644 usr/bin/irprevious-irefweb create mode 100644 vsc/tryout_replacingTOOLS.yml diff --git a/usr/bin/irbuild b/usr/bin/irbuild old mode 100755 new mode 100644 index 9fbca55c..a376b5fc --- a/usr/bin/irbuild +++ b/usr/bin/irbuild @@ -66,70 +66,70 @@ if [ "$BUILD" ]; then # Import any manifest information. - if [ -e "$DATA/irefindex_manifest" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_manifest.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/irefindex_manifest" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_manifest.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Manifest information could not be imported." 1>&2 exit 1 fi # Convert MITAB-based data to an XML-compatible representation. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/mitab_to_xml.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/mitab_to_xml.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not convert MITAB data to the common data representation." 1>&2 exit 1 fi # Finalise the XML-based data. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_constraints.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_constraints.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Constraints could not be applied to the common data representation." 1>&2 exit 1 fi # Import archived identifiers from a previous release. - if [ -e "$DATA/sequences_archived" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/sequences_archived" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import archived identifiers from the previous release." 1>&2 exit 1 fi # Build a sequence archive for the release. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_sequences.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build a sequence archive from the imported data." 1>&2 exit 1 fi # Identify the interactors. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactors.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactors.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interactor information from the imported data." 1>&2 exit 1 fi # Establish gene-to-protein mappings. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not create gene-to-protein mappings." 1>&2 exit 1 fi # Map identifiers to sequences in current databases. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences linkprefix '' ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences linkprefix '' ; then echo "$PROGNAME: Could not build sequence information from the imported data." 1>&2 exit 1 fi # Map identifiers to sequences in archived databases. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences_archived linkprefix 'archived/' ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences_archived linkprefix 'archived/' ; then echo "$PROGNAME: Could not build sequence information from archived data." 1>&2 exit 1 fi # Map interactors to sequences. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactor_sequences.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactor_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not map interactors to sequences." 1>&2 exit 1 fi @@ -146,14 +146,14 @@ if [ "$MAKE_REPORTS" ]; then # Make a tidied up manifest report. echo "$REPORTS $DATA $PSQL_OPTIONS" 1>&2 - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report manifest information." 1>&2 exit 1 fi # Make an interactor report. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report source interactor information." 1>&2 exit 1 fi @@ -172,7 +172,7 @@ if [ "$BUILD" ]; then # Assign definitive sequence information to interactors. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build assignment information from the imported data." 1>&2 exit 1 fi @@ -182,17 +182,17 @@ if [ "$MAKE_REPORTS" ]; then # Make assigned interactor reports, if requested. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report sequence assignment information." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier assignment information." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier scoring information." 1>&2 exit 1 fi @@ -204,34 +204,34 @@ if [ "$BUILD" ]; then # Export ROG information for RIG identifier calculation, make and import RIG identifiers. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export ROG identifiers for assigned sequences." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then echo "$PROGNAME: Could not generate RIG identifiers for the exported ROG identifiers." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated RIG identifiers." 1>&2 exit 1 fi # Make canonical interaction information. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export canonical ROG identifiers." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then echo "$PROGNAME: Could not generate canonical RIG identifiers for the exported canonical ROG identifiers." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated canonical RIG identifiers." 1>&2 exit 1 fi @@ -241,22 +241,22 @@ if [ "$MAKE_REPORTS" ]; then # Make an interactions report, if requested. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier assignment information." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier taxonomy statistics." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report interaction statistics." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then echo "$PROGNAME: Could not generate statistics Wiki page." 1>&2 exit 1 fi @@ -268,29 +268,29 @@ if [ "$BUILD" ]; then # done in separate script # Make convenient representations for other entity types. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interaction information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate confidence score information." 1>&2 exit 1 fi # Import integer identifiers from a previous release. - if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import integer identifiers from the previous release." 1>&2 exit 1 fi diff --git a/usr/bin/irbuild-output-resources b/usr/bin/irbuild-output-resources old mode 100755 new mode 100644 index 6b8b64d8..5597497d --- a/usr/bin/irbuild-output-resources +++ b/usr/bin/irbuild-output-resources @@ -66,22 +66,22 @@ if [ "$BUILD" ]; then # Make convenient representations for other entity types. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interaction information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate confidence score information." 1>&2 exit 1 fi diff --git a/usr/bin/irbuild2 b/usr/bin/irbuild2 old mode 100755 new mode 100644 index 3df46762..0377f8c3 --- a/usr/bin/irbuild2 +++ b/usr/bin/irbuild2 @@ -60,14 +60,14 @@ if [ "$MAKE_REPORTS" ]; then # Make a tidied up manifest report. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report manifest information." 1>&2 exit 1 fi # Make an interactor report. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report source interactor information." 1>&2 exit 1 fi @@ -86,7 +86,7 @@ if [ "$BUILD" ]; then # Assign definitive sequence information to interactors. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build assignment information from the imported data." 1>&2 exit 1 fi @@ -96,17 +96,17 @@ if [ "$MAKE_REPORTS" ]; then # Make assigned interactor reports, if requested. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report sequence assignment information." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier assignment information." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier scoring information." 1>&2 exit 1 fi @@ -118,34 +118,34 @@ if [ "$BUILD" ]; then # Export ROG information for RIG identifier calculation, make and import RIG identifiers. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export ROG identifiers for assigned sequences." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then echo "$PROGNAME: Could not generate RIG identifiers for the exported ROG identifiers." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated RIG identifiers." 1>&2 exit 1 fi # Make canonical interaction information. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export canonical ROG identifiers." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then echo "$PROGNAME: Could not generate canonical RIG identifiers for the exported canonical ROG identifiers." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated canonical RIG identifiers." 1>&2 exit 1 fi @@ -155,22 +155,22 @@ if [ "$MAKE_REPORTS" ]; then # Make an interactions report, if requested. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier assignment information." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier taxonomy statistics." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report interaction statistics." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then echo "$PROGNAME: Could not generate statistics Wiki page." 1>&2 exit 1 fi @@ -182,29 +182,29 @@ if [ "$BUILD" ]; then # Make convenient representations for other entity types. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interaction information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate confidence score information." 1>&2 exit 1 fi # Import integer identifiers from a previous release. - if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import integer identifiers from the previous release." 1>&2 exit 1 fi diff --git a/usr/bin/irbuild3 b/usr/bin/irbuild3 old mode 100755 new mode 100644 index f7aa8d29..44e00f81 --- a/usr/bin/irbuild3 +++ b/usr/bin/irbuild3 @@ -59,24 +59,24 @@ if [ "$BUILD" ]; then # Make convenient representations for other entity types. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate confidence score information." 1>&2 exit 1 fi # Import integer identifiers from a previous release. - if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import integer identifiers from the previous release." 1>&2 exit 1 fi diff --git a/usr/bin/irbuild4 b/usr/bin/irbuild4 old mode 100755 new mode 100644 index 4c590605..52f111c5 --- a/usr/bin/irbuild4 +++ b/usr/bin/irbuild4 @@ -66,49 +66,49 @@ if [ "$BUILD" ]; then # Import archived identifiers from a previous release. - if [ -e "$DATA/sequences_archived" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/sequences_archived" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import archived identifiers from the previous release." 1>&2 exit 1 fi # Build a sequence archive for the release. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_sequences.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build a sequence archive from the imported data." 1>&2 exit 1 fi # Identify the interactors. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactors.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactors.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interactor information from the imported data." 1>&2 exit 1 fi # Establish gene-to-protein mappings. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not create gene-to-protein mappings." 1>&2 exit 1 fi # Map identifiers to sequences in current databases. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences linkprefix '' ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences linkprefix '' ; then echo "$PROGNAME: Could not build sequence information from the imported data." 1>&2 exit 1 fi # Map identifiers to sequences in archived databases. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences_archived linkprefix 'archived/' ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences_archived linkprefix 'archived/' ; then echo "$PROGNAME: Could not build sequence information from archived data." 1>&2 exit 1 fi # Map interactors to sequences. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactor_sequences.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactor_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not map interactors to sequences." 1>&2 exit 1 fi @@ -125,14 +125,14 @@ if [ "$MAKE_REPORTS" ]; then # Make a tidied up manifest report. echo "$REPORTS $DATA $PSQL_OPTIONS" 1>&2 - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report manifest information." 1>&2 exit 1 fi # Make an interactor report. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report source interactor information." 1>&2 exit 1 fi @@ -151,7 +151,7 @@ if [ "$BUILD" ]; then # Assign definitive sequence information to interactors. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build assignment information from the imported data." 1>&2 exit 1 fi @@ -161,17 +161,17 @@ if [ "$MAKE_REPORTS" ]; then # Make assigned interactor reports, if requested. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report sequence assignment information." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier assignment information." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier scoring information." 1>&2 exit 1 fi @@ -183,34 +183,34 @@ if [ "$BUILD" ]; then # Export ROG information for RIG identifier calculation, make and import RIG identifiers. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export ROG identifiers for assigned sequences." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then echo "$PROGNAME: Could not generate RIG identifiers for the exported ROG identifiers." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated RIG identifiers." 1>&2 exit 1 fi # Make canonical interaction information. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export canonical ROG identifiers." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then echo "$PROGNAME: Could not generate canonical RIG identifiers for the exported canonical ROG identifiers." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated canonical RIG identifiers." 1>&2 exit 1 fi @@ -220,22 +220,22 @@ if [ "$MAKE_REPORTS" ]; then # Make an interactions report, if requested. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier assignment information." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier taxonomy statistics." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report interaction statistics." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then echo "$PROGNAME: Could not generate statistics Wiki page." 1>&2 exit 1 fi @@ -247,29 +247,29 @@ if [ "$BUILD" ]; then # done in separate script # Make convenient representations for other entity types. - # if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then + # if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not build interaction information from the imported data." 1>&2 # exit 1 # fi - # if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + # if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 # exit 1 # fi - # if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + # if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 # exit 1 # fi - # if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + # if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not generate confidence score information." 1>&2 # exit 1 # fi # Import integer identifiers from a previous release. - if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import integer identifiers from the previous release." 1>&2 exit 1 fi diff --git a/usr/bin/ircanonical b/usr/bin/ircanonical old mode 100755 new mode 100644 index 354d1290..144f5755 --- a/usr/bin/ircanonical +++ b/usr/bin/ircanonical @@ -33,14 +33,14 @@ EOF exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the canonical schema." 1>&2 exit 1 fi # Combine groups of genes until no more updates can be made. -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_iteration.sql" "$DATA" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_iteration.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not update the canonical schema." 1>&2 exit 1 fi @@ -48,14 +48,14 @@ fi while [ `cat "$DATA/canonical_updates"` != '0' ]; do echo "$PROGNAME: Updating the canonical schema..." 1>&2 - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_iteration.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_iteration.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not update the canonical schema." 1>&2 exit 1 fi done -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_final.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_final.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not define the canonical representative interactors." 1>&2 exit 1 fi diff --git a/usr/bin/irdata-config b/usr/bin/irdata-config old mode 100755 new mode 100644 index 479ecbd9..13d5f3b5 --- a/usr/bin/irdata-config +++ b/usr/bin/irdata-config @@ -139,7 +139,7 @@ BHF_UCL_DOWNLOAD_FILES="bhfucl_psicquic.txt" BHF_UCL_DIR_NAME="BHF-UCL" BHF_UCL_OTHER_FILE_PATTERN='*.txt' BHF_UCL_PARSER='irdata_parse_bhfucl.sh' -BHF_UCL_IMPORT_TEMPLATE='import_mitab.sql' +BHF_UCL_IMPORT_TEMPLATE='import_mitab.sql' ############################################################ # BIND diff --git a/usr/bin/irdata-show-settings b/usr/bin/irdata-show-settings old mode 100755 new mode 100644 diff --git a/usr/bin/irdata_archive_irefscape.sh b/usr/bin/irdata_archive_irefscape.sh old mode 100755 new mode 100644 index 781f1e4e..317b9cc0 --- a/usr/bin/irdata_archive_irefscape.sh +++ b/usr/bin/irdata_archive_irefscape.sh @@ -62,7 +62,7 @@ done # Convert the graph to the Java serialisation format. -"$TOOLS/irdata_convert_graph.py" "$DATADIR/graph.txt" "$DATADIR/graph" +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_convert_graph.py" "$DATADIR/graph.txt" "$DATADIR/graph" diff --git a/usr/bin/irdata_fetch_refseq.sh b/usr/bin/irdata_fetch_refseq.sh old mode 100755 new mode 100644 index 7a76830d..c955b1d6 --- a/usr/bin/irdata_fetch_refseq.sh +++ b/usr/bin/irdata_fetch_refseq.sh @@ -71,7 +71,7 @@ for ATTEMPT in `seq 1 $WGET_ATTEMPTS` ; do # Split the input file into manageable pieces and process each one in turn. # Note that this is done serially due to Entrez usage restrictions. - "$TOOLS/irdata_split.py" -1 10000 "$WORKFILE" \ + "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_split.py" -1 10000 "$WORKFILE" \ | xargs $XARGS_I'{}' sh -c "echo {} | \"$SCRIPTS/irslice\" \"$WORKFILE\" - | \"$TOOLS/irdata_fetch_eutils.sh\" \"$FILENAME\"" \ >> "$RESULTFILE" @@ -87,7 +87,7 @@ done # Parse the feature table output, producing files similar to those normally # available for RefSeq. -if ! "$TOOLS/irdata_parse_refseq.py" "$DATADIR" "$RESULTFILE" ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_refseq.py" "$DATADIR" "$RESULTFILE" ; then echo "$PROGNAME: Could not parse the retrieved data." 1>&2 exit 1 fi diff --git a/usr/bin/irdata_index_irefscape_data.sh b/usr/bin/irdata_index_irefscape_data.sh old mode 100755 new mode 100644 index 3a05c718..592d6181 --- a/usr/bin/irdata_index_irefscape_data.sh +++ b/usr/bin/irdata_index_irefscape_data.sh @@ -48,5 +48,5 @@ fi INTERVAL=100 - "$TOOLS/irdata_text_index.py" $INTERVAL "$FILENAME" -d '|' \ + "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_index.py" $INTERVAL "$FILENAME" -d '|' \ > "$OUTFILE" diff --git a/usr/bin/irdata_parse_A_thaliana.sh b/usr/bin/irdata_parse_A_thaliana.sh old mode 100755 new mode 100644 index 49fca643..2c6c180f --- a/usr/bin/irdata_parse_A_thaliana.sh +++ b/usr/bin/irdata_parse_A_thaliana.sh @@ -57,5 +57,5 @@ echo "$FILETYPE" # cat the file and present it to the parser. echo "$DATADIR/$OUTFILE" 1>&2 zcat "$FILENAME" \ -| "$TOOLS/irdata_parse_A_thaliana2.py" "$FILETYPE" --discard-ill-formed \ +| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_A_thaliana2.py" "$FILETYPE" --discard-ill-formed \ > "$DATADIR/$OUTFILE" diff --git a/usr/bin/irdata_parse_apid.sh b/usr/bin/irdata_parse_apid.sh old mode 100755 new mode 100644 index d637bf22..98a65c7b --- a/usr/bin/irdata_parse_apid.sh +++ b/usr/bin/irdata_parse_apid.sh @@ -43,4 +43,4 @@ if [ ! "$DATADIR" ] || [ ! "$FILENAMES" ]; then fi -"$TOOLS/irdata_parse_mitab.py" 'APID' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'APID' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_bar.sh b/usr/bin/irdata_parse_bar.sh old mode 100755 new mode 100644 index 5515b67a..099b86b7 --- a/usr/bin/irdata_parse_bar.sh +++ b/usr/bin/irdata_parse_bar.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'BAR' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'BAR' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_bhfucl.sh b/usr/bin/irdata_parse_bhfucl.sh old mode 100755 new mode 100644 index 67cd8d66..2106c8a1 --- a/usr/bin/irdata_parse_bhfucl.sh +++ b/usr/bin/irdata_parse_bhfucl.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'BHF_UCL' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'BHF_UCL' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_bind.sh b/usr/bin/irdata_parse_bind.sh old mode 100755 new mode 100644 index c8e9f263..2eb9a10d --- a/usr/bin/irdata_parse_bind.sh +++ b/usr/bin/irdata_parse_bind.sh @@ -91,7 +91,7 @@ for FILENAME in $FILENAMES; do # Pad the file in order to add missing methods. - "$TOOLS/irdata_text_pad.py" -n 4 -p \\N "$FILENAME" \ + "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_pad.py" -n 4 -p \\N "$FILENAME" \ > "$DATADIR/references.txt" elif [ "$FILETYPE" = 'complex2refs.txt' ]; then @@ -107,7 +107,7 @@ for FILENAME in $FILENAMES; do sed -n -e 'p;=' "$FILENAME" \ | sed -e 'N;s/\n/\t/' \ - | "$TOOLS/irdata_text_transpose.py" -f 8 -t 8 -w '|' -s 0 - \ + | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 8 -t 8 -w '|' -s 0 - \ | sed -e "s/^/$SED_FILENAME\t/" \ > "$DATADIR/complexes.txt" @@ -116,14 +116,14 @@ for FILENAME in $FILENAMES; do # First interactor (add position 0). cut -f 1,2,4 "$FILENAME" \ - | "$TOOLS/irdata_text_transpose.py" -f 3 -w '|' -s 0 - \ + | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 3 -w '|' -s 0 - \ | sed -e "s/^/0\t/" \ > "$DATADIR/labels.txt" # Second interactor (add position 1). cut -f 1,3,5 "$FILENAME" \ - | "$TOOLS/irdata_text_transpose.py" -f 3 -w '|' -s 0 - \ + | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 3 -w '|' -s 0 - \ | sed -e "s/^/1\t/" \ >> "$DATADIR/labels.txt" diff --git a/usr/bin/irdata_parse_dig.sh b/usr/bin/irdata_parse_dig.sh old mode 100755 new mode 100644 index 99d96251..9473c380 --- a/usr/bin/irdata_parse_dig.sh +++ b/usr/bin/irdata_parse_dig.sh @@ -51,6 +51,6 @@ fi tail -n +2 "$FILENAME" \ | cut -f 2,7 \ -| "$TOOLS/irdata_text_transpose.py" -f 1 -t 1 -w ', ' - \ +| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 1 -t 1 -w ', ' - \ | sort -u \ > "$DATADIR/dig_genes.txt" diff --git a/usr/bin/irdata_parse_dip.sh b/usr/bin/irdata_parse_dip.sh old mode 100755 new mode 100644 index c89a84f0..10df6258 --- a/usr/bin/irdata_parse_dip.sh +++ b/usr/bin/irdata_parse_dip.sh @@ -57,4 +57,4 @@ fi # mv tmp $THISFILE #done -"$TOOLS/irdata_parse_mitab.py" 'DIP' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'DIP' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_fly.sh b/usr/bin/irdata_parse_fly.sh old mode 100755 new mode 100644 index f2749dcd..ea114c8d --- a/usr/bin/irdata_parse_fly.sh +++ b/usr/bin/irdata_parse_fly.sh @@ -57,5 +57,5 @@ END=`grep -ne '^----' "$FILENAME" | tail -n 2 | head -n 1 | cut -d ':' -f 1` head -n $((END - 1)) "$FILENAME" \ | tail -n "+$START" \ -| "$TOOLS/irdata_parse_fly.py" "$FILETYPE" --discard-ill-formed \ +| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fly.py" "$FILETYPE" --discard-ill-formed \ > "$DATADIR/$OUTFILE" diff --git a/usr/bin/irdata_parse_gene.sh b/usr/bin/irdata_parse_gene.sh old mode 100755 new mode 100644 diff --git a/usr/bin/irdata_parse_gene_info.sh b/usr/bin/irdata_parse_gene_info.sh old mode 100755 new mode 100644 index 80bb3938..84f780de --- a/usr/bin/irdata_parse_gene_info.sh +++ b/usr/bin/irdata_parse_gene_info.sh @@ -76,7 +76,7 @@ fi | tail -n +2 \ | cut -f 2,8 \ | grep -v -e 'NEWENTRY' \ -| "$TOOLS/irdata_text_transpose.py" -f 2 -w '|' -s 0 - \ +| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 2 -w '|' -s 0 - \ > "$DATADIR/$OUTFILE_MAPLOCATIONS" # For synonyms, remove the header, extract the geneid and synonyms. @@ -87,5 +87,5 @@ fi | tail -n +2 \ | cut -f 2,5 \ | grep -v -e 'NEWENTRY' \ -| "$TOOLS/irdata_text_transpose.py" -f 2 -w '|' -s 0 - \ +| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 2 -w '|' -s 0 - \ > "$DATADIR/$OUTFILE_SYNONYMS" diff --git a/usr/bin/irdata_parse_genpept.sh b/usr/bin/irdata_parse_genpept.sh old mode 100755 new mode 100644 index fe4c6d9c..836263dc --- a/usr/bin/irdata_parse_genpept.sh +++ b/usr/bin/irdata_parse_genpept.sh @@ -44,7 +44,7 @@ if [ ! "$DATADIR" ] || [ ! "$FILENAMES" ]; then fi "$SCRIPTS/argument-per-line" $FILENAMES \ -| "$SCRIPTS/irparallel" "\"$TOOLS/irdata_parse_fasta.py\" 'GENPEPT' \"$DATADIR\" 'acc,name,organism' 'acc,name,organism' {}" +| "$SCRIPTS/irparallel" "\"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fasta.py\" 'GENPEPT' \"$DATADIR\" 'acc,name,organism' 'acc,name,organism' {}" # Concatenate the output data # Tab character used in the sed command. diff --git a/usr/bin/irdata_parse_hpidb.sh b/usr/bin/irdata_parse_hpidb.sh old mode 100755 new mode 100644 index 281786a6..f03d9bac --- a/usr/bin/irdata_parse_hpidb.sh +++ b/usr/bin/irdata_parse_hpidb.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'HPIDB' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'HPIDB' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_huri.sh b/usr/bin/irdata_parse_huri.sh old mode 100755 new mode 100644 index e94718a6..16aa5723 --- a/usr/bin/irdata_parse_huri.sh +++ b/usr/bin/irdata_parse_huri.sh @@ -58,4 +58,4 @@ for THISFILE in $FILENAMES; do done -"$TOOLS/irdata_parse_mitab.py" 'HURI' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'HURI' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_imex.sh b/usr/bin/irdata_parse_imex.sh old mode 100755 new mode 100644 index 4c1ffdd7..1425912c --- a/usr/bin/irdata_parse_imex.sh +++ b/usr/bin/irdata_parse_imex.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'IMEX' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'IMEX' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_innatedb.sh b/usr/bin/irdata_parse_innatedb.sh old mode 100755 new mode 100644 index 961dbf79..d57723e1 --- a/usr/bin/irdata_parse_innatedb.sh +++ b/usr/bin/irdata_parse_innatedb.sh @@ -52,4 +52,4 @@ if [ $? != 0 ]; then exit 1 fi -"$TOOLS/irdata_parse_mitab.py" 'INNATEDB' "$DATADIR" $DST +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'INNATEDB' "$DATADIR" $DST diff --git a/usr/bin/irdata_parse_ipi.sh b/usr/bin/irdata_parse_ipi.sh old mode 100755 new mode 100644 index ad386383..80d08382 --- a/usr/bin/irdata_parse_ipi.sh +++ b/usr/bin/irdata_parse_ipi.sh @@ -47,7 +47,7 @@ for FILENAME in $FILENAMES; do BASENAME=`basename "$FILENAME"` FILETYPE=${BASENAME##*.} if [ "$FILETYPE" = 'fasta' ] || [ "$FILETYPE" = 'gz' ]; then - if ! "$TOOLS/irdata_parse_fasta.py" 'IPI' "$DATADIR" 'acc' 'acc' "$FILENAME" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fasta.py" 'IPI' "$DATADIR" 'acc' 'acc' "$FILENAME" ; then echo "$PROGNAME: FASTA parsing of $FILENAME failed." 1>&2 exit 1 fi @@ -74,10 +74,10 @@ for FILENAME in $FILENAMES; do | cut -d '>' -f 2- \ | cut -d ' ' -f 1,2 \ | sed -e 's/ /|/g' \ - | "$TOOLS/irdata_text_transpose.py" -f 2 -d '|' - \ + | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 2 -d '|' - \ | sed -e 's/[:=]/\t/g' \ | cut -f 2- \ - | "$TOOLS/irdata_text_transpose.py" -f 3 -w ';' - \ + | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 3 -w ';' - \ > "$DATADIR/${BASENAME}_identifiers.txt" else diff --git a/usr/bin/irdata_parse_matrixdb.sh b/usr/bin/irdata_parse_matrixdb.sh old mode 100755 new mode 100644 index ac39173c..7f96f7d3 --- a/usr/bin/irdata_parse_matrixdb.sh +++ b/usr/bin/irdata_parse_matrixdb.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'MATRIXDB' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'MATRIXDB' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_mbinfo.sh b/usr/bin/irdata_parse_mbinfo.sh old mode 100755 new mode 100644 index a8bef66b..f3fd9f6c --- a/usr/bin/irdata_parse_mbinfo.sh +++ b/usr/bin/irdata_parse_mbinfo.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'MBINFO' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'MBINFO' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_mint.sh b/usr/bin/irdata_parse_mint.sh old mode 100755 new mode 100644 index b04e5cec..aa2246aa --- a/usr/bin/irdata_parse_mint.sh +++ b/usr/bin/irdata_parse_mint.sh @@ -43,4 +43,4 @@ if [ ! "$DATADIR" ] || [ ! "$FILENAMES" ]; then fi -"$TOOLS/irdata_parse_mpidb.py" 'MINT' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mpidb.py" 'MINT' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_molcon.sh b/usr/bin/irdata_parse_molcon.sh old mode 100755 new mode 100644 index b8b272d8..df399989 --- a/usr/bin/irdata_parse_molcon.sh +++ b/usr/bin/irdata_parse_molcon.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'MOLCON' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'MOLCON' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_mpidb.sh b/usr/bin/irdata_parse_mpidb.sh old mode 100755 new mode 100644 index 77dbd44c..06159107 --- a/usr/bin/irdata_parse_mpidb.sh +++ b/usr/bin/irdata_parse_mpidb.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'MPIDB' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'MPIDB' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_pdb.sh b/usr/bin/irdata_parse_pdb.sh old mode 100755 new mode 100644 index 680943e3..f100aba3 --- a/usr/bin/irdata_parse_pdb.sh +++ b/usr/bin/irdata_parse_pdb.sh @@ -47,7 +47,7 @@ for FILENAME in $FILENAMES; do BASENAME=`basename "$FILENAME"` FILETYPE=${BASENAME#*.} if [ "$FILETYPE" = 'fasta' ] || [ "$FILETYPE" = 'gz' ]; then - if ! "$TOOLS/irdata_parse_fasta.py" 'PDB' "$DATADIR" 'acc,chain' 'acc,chain' "$FILENAME" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fasta.py" 'PDB' "$DATADIR" 'acc,chain' 'acc,chain' "$FILENAME" ; then echo "$PROGNAME: FASTA parsing of $FILENAME failed." 1>&2 exit 1 fi diff --git a/usr/bin/irdata_parse_quickgo.sh b/usr/bin/irdata_parse_quickgo.sh old mode 100755 new mode 100644 index 43fef988..6ec30804 --- a/usr/bin/irdata_parse_quickgo.sh +++ b/usr/bin/irdata_parse_quickgo.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'QUICKGO' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'QUICKGO' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_reactome.sh b/usr/bin/irdata_parse_reactome.sh old mode 100755 new mode 100644 index 51327cb3..1498bdba --- a/usr/bin/irdata_parse_reactome.sh +++ b/usr/bin/irdata_parse_reactome.sh @@ -61,4 +61,4 @@ fi # mv tmp $THISFILE #done -"$TOOLS/irdata_parse_mitab.py" 'REACTOME' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'REACTOME' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_refseq.sh b/usr/bin/irdata_parse_refseq.sh old mode 100755 new mode 100644 index 13864798..119b982f --- a/usr/bin/irdata_parse_refseq.sh +++ b/usr/bin/irdata_parse_refseq.sh @@ -52,7 +52,7 @@ fi # Parse the data files. "$SCRIPTS/argument-per-line" $FILENAMES \ -| "$SCRIPTS/irparallel" "\"$TOOLS/irdata_parse_refseq.py\" \"$DATADIR\" {}" +| "$SCRIPTS/irparallel" "\"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_refseq.py\" \"$DATADIR\" {}" # Concatenate the output data. echo "$DATADIR cat step" 1>&2 diff --git a/usr/bin/irdata_parse_spike.sh b/usr/bin/irdata_parse_spike.sh old mode 100755 new mode 100644 index e2bb20e8..44847712 --- a/usr/bin/irdata_parse_spike.sh +++ b/usr/bin/irdata_parse_spike.sh @@ -62,4 +62,4 @@ fi # mv tmp $THISFILE #done -"$TOOLS/irdata_parse_mitab.py" 'SPIKE' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'SPIKE' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_uniprot.sh b/usr/bin/irdata_parse_uniprot.sh old mode 100755 new mode 100644 index 886bd62f..298b1808 --- a/usr/bin/irdata_parse_uniprot.sh +++ b/usr/bin/irdata_parse_uniprot.sh @@ -50,7 +50,7 @@ for FILENAME in $FILENAMES; do # Parse the FASTA files. if [ "$FILETYPE" = 'fasta' ] || [ "$FILETYPE" = 'fasta.gz' ]; then - if ! "$TOOLS/irdata_parse_fasta.py" 'UNIPROT' "$DATADIR" 'sp,acc,id' 'id,acc,date,taxid,mw' "$FILENAME" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fasta.py" 'UNIPROT' "$DATADIR" 'sp,acc,id' 'id,acc,date,taxid,mw' "$FILENAME" ; then echo "$PROGNAME: FASTA parsing of $FILENAME failed." 1>&2 exit 1 fi @@ -66,7 +66,7 @@ for FILENAME in $FILENAMES; do # Split the data file into $PROCESS pieces. # Pass each process its rank (0-based) and the total number of processes. seq 0 $((PROCESSES - 1)) \ - | "$SCRIPTS/irparallel" "$TOOLS/irdata_altsplit.py $FILENAME {} $PROCESSES | $TOOLS/irdata_parse_uniprot.py $DATADIR - ${LEAFNAME}_%s-{}.txt" + | "$SCRIPTS/irparallel" "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_altsplit.py $FILENAME {} $PROCESSES | $TOOLS/irdata_parse_uniprot.py $DATADIR - ${LEAFNAME}_%s-{}.txt" # Merge the pieces. echo "$PROGNAME: Merging data files for $LEAFNAME..." 1>&2 diff --git a/usr/bin/irdata_parse_uniprotpp.sh b/usr/bin/irdata_parse_uniprotpp.sh old mode 100755 new mode 100644 index 47cb264a..c13341c8 --- a/usr/bin/irdata_parse_uniprotpp.sh +++ b/usr/bin/irdata_parse_uniprotpp.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"$TOOLS/irdata_parse_mitab.py" 'UNIPROTPP' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'UNIPROTPP' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_virushost.sh b/usr/bin/irdata_parse_virushost.sh old mode 100755 new mode 100644 index 2cb3266c..502d2483 --- a/usr/bin/irdata_parse_virushost.sh +++ b/usr/bin/irdata_parse_virushost.sh @@ -62,4 +62,4 @@ fi # mv tmp $THISFILE #done -"$TOOLS/irdata_parse_mitab.py" 'VIRUSHOST' "$DATADIR" $FILENAMES +"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'VIRUSHOST' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_process_signatures.sh b/usr/bin/irdata_process_signatures.sh old mode 100755 new mode 100644 index cccc27d1..a9dc38f7 --- a/usr/bin/irdata_process_signatures.sh +++ b/usr/bin/irdata_process_signatures.sh @@ -48,7 +48,7 @@ for FILENAME in "$DATADIR/"*_proteins.txt ; do exit 1 fi - if ! "$TOOLS/irdata_process_signatures.py" "$FILENAME" "$FILENAME.seq" $* ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$FILENAME" "$FILENAME.seq" $* ; then echo "$PROGNAME: Sequence digest processing of $FILENAME failed." 1>&2 exit 1 fi diff --git a/usr/bin/irdata_split_irefscape_data.sh b/usr/bin/irdata_split_irefscape_data.sh old mode 100755 new mode 100644 diff --git a/usr/bin/irdrop b/usr/bin/irdrop old mode 100755 new mode 100644 index ff1166bf..a5238cb0 --- a/usr/bin/irdrop +++ b/usr/bin/irdrop @@ -53,9 +53,9 @@ fi if [ "$DROPALL" ]; then - if ! ( "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop.sql" $PSQL_OPTIONS \ - && "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_psixml.sql" $PSQL_OPTIONS \ - && "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_mitab.sql" $PSQL_OPTIONS \ + if ! ( "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop.sql" $PSQL_OPTIONS \ + && "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_psixml.sql" $PSQL_OPTIONS \ + && "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_mitab.sql" $PSQL_OPTIONS \ ) ; then echo "$PROGNAME: Could not remove the database schema." 1>&2 exit 1 @@ -67,7 +67,7 @@ if [ "$DROPALL" ]; then eval SCHEMA='$'"$SOURCE"_SCHEMA if [ "$SCHEMA" ]; then - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_${SCHEMA}.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_${SCHEMA}.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove schema for $SOURCE." 1>&2 [ ! "$FORCE" ] && exit 1 fi @@ -78,72 +78,72 @@ fi # Remove the different build tables. # Continue without exiting if the force option is given. -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_manifest.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_manifest.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the manifest schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if [ ! "$DROPALL" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_constraints.sql" $PSQL_OPTIONS ; then +if [ ! "$DROPALL" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_constraints.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the common representation's constraints." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_sequences.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the consolidated sequence information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_archived.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_archived.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the archived sequence information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_interactors.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_interactors.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the interactor information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the gene-to-protein mappings schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_canonical.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_canonical.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the canonical interactor information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_assignments.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the assignment information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_rigids.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_rigids.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the interaction information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_integer_identifiers.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_integer_identifiers.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the integer identifier mappings schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_interactions.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the additional interaction information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_experiments.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the additional experiment information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_participants.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the additional participant information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_confidence.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the confidence score schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi diff --git a/usr/bin/irefpro.sh b/usr/bin/irefpro.sh old mode 100755 new mode 100644 index 2f8116c1..170421a3 --- a/usr/bin/irefpro.sh +++ b/usr/bin/irefpro.sh @@ -31,7 +31,7 @@ echo $SQL echo $PSQL_OPTIONS echo `whoami` -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefpro.sql" "$DATA" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefpro.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate iRefPro data." 1>&2 exit 1 fi diff --git a/usr/bin/irimport b/usr/bin/irimport old mode 100755 new mode 100644 index 144bc68b..2ba19982 --- a/usr/bin/irimport +++ b/usr/bin/irimport @@ -72,7 +72,7 @@ elif [ ! "$SOURCES" ]; then fi if [ "$CHECK" ]; then - "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_sources.sql" "$DATA" $PSQL_OPTIONS + "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_sources.sql" "$DATA" $PSQL_OPTIONS IMPORTEDSOURCES=`cat "$DATA/imported_sources"` fi diff --git a/usr/bin/irimport-source b/usr/bin/irimport-source old mode 100755 new mode 100644 index 93e7a0ba..12cfc3eb --- a/usr/bin/irimport-source +++ b/usr/bin/irimport-source @@ -72,7 +72,7 @@ fi # Import XML sources using a common template. if [ "$XMLFILEPATTERN" ]; then - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_psixml.sql" "$IMPORTDATADIR" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_psixml.sql" "$IMPORTDATADIR" $PSQL_OPTIONS ; then echo "$PROGNAME: Import failed for XML files from $SOURCE." 1>&2 exit 1 fi @@ -86,7 +86,7 @@ else eval DEFS='$'"$SOURCE"_DEFS [[ -n ${DEFS} ]] && DEFS="--defs ${DEFS}" - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/$TEMPLATE" "$IMPORTDATADIR" $PSQL_OPTIONS $DEFS; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/$TEMPLATE" "$IMPORTDATADIR" $PSQL_OPTIONS $DEFS; then echo "$PROGNAME: Import failed for data files from $SOURCE." 1>&2 exit 1 fi diff --git a/usr/bin/irinit b/usr/bin/irinit old mode 100755 new mode 100644 index 147d93c0..9cec478f --- a/usr/bin/irinit +++ b/usr/bin/irinit @@ -44,9 +44,9 @@ fi if [ ! "$INITBUILD" ]; then - if ! ( "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init.sql" $PSQL_OPTIONS \ - && "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_psixml.sql" $PSQL_OPTIONS \ - && "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_mitab.sql" $PSQL_OPTIONS \ + if ! ( "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init.sql" $PSQL_OPTIONS \ + && "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_psixml.sql" $PSQL_OPTIONS \ + && "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_mitab.sql" $PSQL_OPTIONS \ ) ; then echo "$PROGNAME: Could not initialise the database schema." 1>&2 exit 1 @@ -58,7 +58,7 @@ if [ ! "$INITBUILD" ]; then eval SCHEMA='$'"$SOURCE"_SCHEMA if [ "$SCHEMA" ]; then - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_${SCHEMA}.sql" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_${SCHEMA}.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise schema for $SOURCE." 1>&2 exit 1 fi @@ -68,67 +68,67 @@ fi # Initialise the different build tables. -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_manifest.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_manifest.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the manifest schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_sequences.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the consolidated sequence information schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_archived.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_archived.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the archived sequence information schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_interactors.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_interactors.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the interactor information schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the gene-to-protein mappings schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_canonical.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_canonical.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the canonical interactor information schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_assignments.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the assignment information schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_rigids.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_rigids.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the interaction information schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_integer_identifiers.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_integer_identifiers.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the integer identifier mappings schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_interactions.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the additional interaction information schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_experiments.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the additional experiment information schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_participants.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the additional participant information schema." 1>&2 exit 1 fi -if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_confidence.sql" $PSQL_OPTIONS ; then +if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the confidence score schema." 1>&2 exit 1 fi diff --git a/usr/bin/irmanifest b/usr/bin/irmanifest old mode 100755 new mode 100644 index d7636d01..0a5e3f06 --- a/usr/bin/irmanifest +++ b/usr/bin/irmanifest @@ -99,7 +99,7 @@ function irfiletime { source=$(echo $SOURCE | tr '[:upper:]' '[:lower:]') DETAILS="$MANIFESTS/irdata_manifest_$source.txt" - "$TOOLS/irdata_manifest.py" "$URL" < "$DETAILS" \ + "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_manifest.py" "$URL" < "$DETAILS" \ | sed -e "s/\(.*\)/$SOURCE\t\\1/" # If a source uses a manifest parser, use that to obtain information. diff --git a/usr/bin/irmissing b/usr/bin/irmissing old mode 100755 new mode 100644 index 6c9ca798..53a998cf --- a/usr/bin/irmissing +++ b/usr/bin/irmissing @@ -69,14 +69,14 @@ if [ ! -e "$DATADIR" ] ; then fi if [ "$CHECK" ] ; then - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_unknown_refseq.sql" "$DATADIR" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_unknown_refseq.sql" "$DATADIR" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export RefSeq identifiers not providing sequences." 1>&2 exit 1 fi # commented since gi no longer in use -# if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_unknown_gi.sql" "$DATADIR" $PSQL_OPTIONS ; then +# if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_unknown_gi.sql" "$DATADIR" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not export GenBank identifiers not providing sequences." 1>&2 # exit 1 # fi @@ -102,7 +102,7 @@ if [ "$IMPORT" ] ; then exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_unknown_refseq.sql" "$DATADIR" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_unknown_refseq.sql" "$DATADIR" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import missing sequences." 1>&2 exit 1 fi diff --git a/usr/bin/iroutput b/usr/bin/iroutput old mode 100755 new mode 100644 index 42859bbf..5a5d4caa --- a/usr/bin/iroutput +++ b/usr/bin/iroutput @@ -76,7 +76,7 @@ if [ "$MITAB" ]; then cp "$SQL/organisms.txt" "$DATA" - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_mitab.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_mitab.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate MITAB data." 1>&2 exit 1 fi @@ -107,7 +107,7 @@ fi if [ "$MAPPING" ]; then - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_protein_identifier_mapping.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_protein_identifier_mapping.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate mapping file." 1>&2 exit 1 fi @@ -125,7 +125,7 @@ if [ "$IREFSCAPE" ]; then rm "$DATA/iRefScape/"* fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefscape.sql" "$DATA/iRefScape" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefscape.sql" "$DATA/iRefScape" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate iRefScape data." 1>&2 exit 1 fi @@ -147,7 +147,7 @@ if [ "$IREFWEB" ]; then rm "$DATA/iRefWeb/"* fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefweb.sql" "$DATA/iRefWeb" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefweb.sql" "$DATA/iRefWeb" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate iRefWeb data." 1>&2 exit 1 fi diff --git a/usr/bin/irparse-source b/usr/bin/irparse-source old mode 100755 new mode 100644 index d9aebb94..5129c17a --- a/usr/bin/irparse-source +++ b/usr/bin/irparse-source @@ -146,7 +146,7 @@ for FILEPATTERN in $FILEPATTERNS; do # Where the check fails, run the fixing tool if requested. - if ! "$TOOLS/irdata_xmlcheck.py" "$FILE" > /dev/null 2> "$CHECKLOG" ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_xmlcheck.py" "$FILE" > /dev/null 2> "$CHECKLOG" ; then if [ "$FIX" ]; then "$TOOLS/irdata_fixencoding.sh" "$FILE" @@ -176,7 +176,7 @@ for FILEPATTERN in $FILEPATTERNS; do echo "$TOOLS/$PARSER" "$OUTPUTDIR" $FILES if [ ! "$NO_PARSE" ]; then if [ "$XMLFILEPATTERN" ]; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_xml2tab.py" "$OUTPUTDIR" "$SOURCE" $FILES ; then + if ! "${USE_PYTHON_INTERPRETER}" "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_xml2tab.py" "$OUTPUTDIR" "$SOURCE" $FILES ; then echo "$PROGNAME: XML processing for source $SOURCE failed." 1>&2 exit 1 fi diff --git a/usr/bin/irprevious b/usr/bin/irprevious old mode 100755 new mode 100644 index c1449096..f1f8884e --- a/usr/bin/irprevious +++ b/usr/bin/irprevious @@ -55,24 +55,24 @@ if [ "$PGSQL" ]; then # Export identifiers from a PostgreSQL database. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_integer_rigids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_integer_rigids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export integer RIG identifiers from the previous database." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_integer_rogids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_integer_rogids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export integer ROG identifiers from the previous database." 1>&2 exit 1 fi # Export archived identifiers from a PostgreSQL database. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export archived identifiers from the previous database." 1>&2 exit 1 fi - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_archived_sequences.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_archived_sequences.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export archived sequences from the previous database." 1>&2 exit 1 fi diff --git a/usr/bin/irprevious-irefweb b/usr/bin/irprevious-irefweb old mode 100755 new mode 100644 index 37c40fe0..b8af5cec --- a/usr/bin/irprevious-irefweb +++ b/usr/bin/irprevious-irefweb @@ -55,7 +55,7 @@ if [ "$PGSQL" ]; then # Export data from a PostgreSQL database. - if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefweb.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefweb.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export iRefWeb data from the previous database." 1>&2 exit 1 fi diff --git a/vsc/ansible/irinit.yml b/vsc/ansible/irinit.yml index 12d3c118..c27817f9 100644 --- a/vsc/ansible/irinit.yml +++ b/vsc/ansible/irinit.yml @@ -102,10 +102,10 @@ name: git state: present - - name: Copy the irefindex files from my local machine to the remote machine - ansible.builtin.copy: - src: "{{ local_source_directory }}" - dest: "{{ remote_destination_directory }}" + # - name: Copy the irefindex files from my local machine to the remote machine + # ansible.builtin.copy: + # src: "{{ local_source_directory }}" + # dest: "{{ remote_destination_directory }}" - name: Create directories for irdata data ansible.builtin.file: @@ -115,6 +115,7 @@ with_items: - "{{ download_data }}var/lib/irdata/data" - "{{ download_data }}var/log/irdata" + - "{{ download_data }}/{{ data }}" - name: Run irinit ansible.builtin.command: ./irinit --init diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 553ee229..553f787a 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -31,7 +31,7 @@ - name: Find files containing "TOOLS" find: - paths: ./usr/bin/ + paths: ./../usr/bin/ patterns: "*" recurse: yes register: found_files diff --git a/vsc/ansible/main2.yml b/vsc/ansible/main2.yml index 0f9540c3..3327703f 100644 --- a/vsc/ansible/main2.yml +++ b/vsc/ansible/main2.yml @@ -5,5 +5,5 @@ - name: Run irmanifest playbook import_playbook: irmanifest.yml -- name: Run irparse playbook +- name: Run irparse playbook import_playbook: irparse.yml \ No newline at end of file diff --git a/vsc/inventory b/vsc/inventory index f53d39cd..c40ba7e1 100644 --- a/vsc/inventory +++ b/vsc/inventory @@ -1 +1 @@ -35.240.84.94 +130.211.85.16 diff --git a/vsc/tryout_replacingTOOLS.yml b/vsc/tryout_replacingTOOLS.yml new file mode 100644 index 00000000..faa214bf --- /dev/null +++ b/vsc/tryout_replacingTOOLS.yml @@ -0,0 +1,22 @@ +--- +- name: Replace TOOLS in files + hosts: all + gather_facts: false + tasks: + - name: Find files containing "TOOLS" + find: + paths: ./usr/test/ + patterns: "*" + recurse: yes + register: found_files + + - name: Iterate over found files + vars: + use_python_interpreter: "{{ USE_PYTHON_INTERPRETER }}" + ansible.builtin.shell: | + for filename in {{ found_files.files | map(attribute='path') | join(' ') }}; do + sed "s|\(\$TOOLS/.*.py\)|{{ use_python_interpreter }}\" \"\1|g" "{{ filename }}" > "{{ filename }}.tmp" && mv "{{ filename }}.tmp" "{{ filename }}" + done + args: + executable: /bin/bash + when: found_files.matched > 0 \ No newline at end of file From f94320f3b74bc07644a252c36f012ac208be5cef Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Mon, 13 May 2024 17:23:34 +0200 Subject: [PATCH 24/74] adding a sudo_command and markers --- usr/bin/irparse-source | 17 ++++++++++++++--- vsc/ansible/irinit.yml | 8 ++++---- 2 files changed, 18 insertions(+), 7 deletions(-) mode change 100644 => 100755 usr/bin/irparse-source diff --git a/usr/bin/irparse-source b/usr/bin/irparse-source old mode 100644 new mode 100755 index 5129c17a..ece2693f --- a/usr/bin/irparse-source +++ b/usr/bin/irparse-source @@ -15,6 +15,15 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# Add sudo before commands that require elevated privileges +sudo_command() { + if [ "$(id -u)" != "0" ]; then + sudo "$@" + else + "$@" + fi +} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -24,7 +33,9 @@ else fi USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} -echo "$USE_PYTHON_INTERPRETER" +echo "Marker 2: $USE_PYTHON_INTERPRETER" +echo "Marker 3: $python_interpreter" + if [ "$1" = '--help' ]; then cat 1>&2 <&2 exit 1 fi @@ -184,7 +195,7 @@ for FILEPATTERN in $FILEPATTERNS; do echo "$PROGNAME: No parser defined for source $SOURCE." 1>&2 exit 1 else - if ! "$TOOLS/$PARSER" "$OUTPUTDIR" $FILES ; then + if ! sudo_command "$TOOLS/$PARSER" "$OUTPUTDIR" $FILES ; then echo "$PROGNAME: Data processing for source $SOURCE failed." 1>&2 exit 1 fi diff --git a/vsc/ansible/irinit.yml b/vsc/ansible/irinit.yml index c27817f9..31099cb8 100644 --- a/vsc/ansible/irinit.yml +++ b/vsc/ansible/irinit.yml @@ -102,10 +102,10 @@ name: git state: present - # - name: Copy the irefindex files from my local machine to the remote machine - # ansible.builtin.copy: - # src: "{{ local_source_directory }}" - # dest: "{{ remote_destination_directory }}" + - name: Copy the irefindex files from my local machine to the remote machine + ansible.builtin.copy: + src: "{{ local_source_directory }}" + dest: "{{ remote_destination_directory }}" - name: Create directories for irdata data ansible.builtin.file: From 4e04620302ee931b26f557e182acf90c74a9eeb5 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Mon, 13 May 2024 17:24:49 +0200 Subject: [PATCH 25/74] putting things in commands because they don't work anymore and use_python_interpreter variable --- vsc/ansible/irparse.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 553f787a..5d0ef176 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -10,6 +10,7 @@ vars: log_location: "{{ logs_dir }}/irparse" + USE_PYTHON_INTERPRETER: "{{ python_interpreter | default('/usr/bin/python3') }}" tasks: - name: Set permissions on Ansible temporary directory @@ -31,19 +32,19 @@ - name: Find files containing "TOOLS" find: - paths: ./../usr/bin/ + paths: /home/irefindex/usr/bin/ patterns: "*" recurse: yes register: found_files - - name: Iterate over found files - vars: - use_python_interpreter: "{{ USE_PYTHON_INTERPRETER }}" - ansible.builtin.shell: | - for filename in {{ found_files.files | map(attribute='path') | join(' ') }}; do sed "s|\(\$TOOLS/.*.py\)|{{ use_python_interpreter }}\" \"\1|g" "{{ filename }}" > "{{ filename }}.tmp" && mv "{{ filename }}.tmp" "{{ filename }}"; done - args: - executable: /bin/bash - when: found_files.matched > 0 + # - name: Iterate over found files + # vars: + # use_python_interpreter: "{{ USE_PYTHON_INTERPRETER }}" + # ansible.builtin.shell: | + # for filename in {{ found_files.files | map(attribute='path') | join(' ') }}; do sed "s|\(\$TOOLS/.*.py\)|{{ use_python_interpreter }}\" \"\1|g" "{{ filename }}" > "{{ filename }}.tmp" && mv "{{ filename }}.tmp" "{{ filename }}"; done + # args: + # executable: /bin/bash + # when: found_files.matched > 0 - name: Convert the downloaded data to files for each source ansible.builtin.command: "./irparse {{ item.name | upper }}" From 5557e0e8a4345ca5db7fecff6a1c3f991a345a41 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 14 May 2024 11:29:51 +0200 Subject: [PATCH 26/74] New block of code to give the scripts the execute privileges --- vsc/ansible/irparse.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 5d0ef176..2c08e198 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -46,6 +46,18 @@ # executable: /bin/bash # when: found_files.matched > 0 + - name: Find .sh and .py files + find: + paths: /home/irefindex/usr/bin + patterns: "*.sh,*.py" + register: files_to_update + + - name: Give execute permissions to .sh and .py files + file: + path: "{{ item.path }}" + mode: u+x,g+x,o+x + loop: "{{ files_to_update.files }}" + - name: Convert the downloaded data to files for each source ansible.builtin.command: "./irparse {{ item.name | upper }}" args: From fe7eb3854a934a39afe795e79500ae4c154fa9a1 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Tue, 14 May 2024 11:31:31 +0200 Subject: [PATCH 27/74] making it more user friendly --- vsc/ansible/vars/general.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vsc/ansible/vars/general.yml b/vsc/ansible/vars/general.yml index 6ef3a23d..688975ff 100644 --- a/vsc/ansible/vars/general.yml +++ b/vsc/ansible/vars/general.yml @@ -2,6 +2,7 @@ release_major: "19" release_minor: "0" release: "{{ release_major }}.{{ release_minor }}" data: "/dataext/irdata{{ release_major }}" -download_data: "/mnt/disks/data/" +download_data: "/mnt/disks/data/" # The directory linked to /dev/sda data2: "/dataext/irdata{{ release_major }}" -home_dir: "/home/elias.steyaert" +home_dir: "/home/elias.steyaert" # personal home folder on the google cloud server +home_folder: "/home/irefindex" # Personal folder on local machine where the GitHub is stored of the project \ No newline at end of file From 31514ce4a556e3823a70e9301ee4f24da7507223 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Thu, 16 May 2024 10:19:11 +0200 Subject: [PATCH 28/74] adding a block of code to install pip and changing the python environment to python3 --- vsc/ansible/irparse.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 2c08e198..689ccbfb 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -29,6 +29,12 @@ - name: Create virtual environment ansible.builtin.command: cmd: python3 -m venv "{{ download_data }}/{{ data2 }}/virtualenv" + + - name: Install irdata module + pip: + name: irdata + executable: "{{ download_data }}/{{ data2 }}/virtualenv/bin/pip" + become: true - name: Find files containing "TOOLS" find: @@ -63,7 +69,7 @@ args: chdir: "/home/irefindex/usr/bin" environment: - python_interpreter: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python" + python_interpreter: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python3" changed_when: false poll: 0 async: "{{ item.parse_async | default(60) }}" @@ -74,7 +80,7 @@ ansible.builtin.async_status: jid: "{{ item.ansible_job_id }}" environment: - python_interpreter: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python" + python_interpreter: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python3" register: job_result until: job_result.finished ignore_errors: true From 5d4e55cb67efb2709e1c5e72565e97192dad54f2 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Thu, 16 May 2024 10:20:23 +0200 Subject: [PATCH 29/74] deleting the sudo command and the markers that weren't needed --- usr/bin/irparse-source | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/usr/bin/irparse-source b/usr/bin/irparse-source index ece2693f..2d81cbae 100755 --- a/usr/bin/irparse-source +++ b/usr/bin/irparse-source @@ -16,14 +16,6 @@ # with this program. If not, see . # Add sudo before commands that require elevated privileges -sudo_command() { - if [ "$(id -u)" != "0" ]; then - sudo "$@" - else - "$@" - fi -} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -33,8 +25,6 @@ else fi USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} -echo "Marker 2: $USE_PYTHON_INTERPRETER" -echo "Marker 3: $python_interpreter" if [ "$1" = '--help' ]; then @@ -187,7 +177,7 @@ for FILEPATTERN in $FILEPATTERNS; do echo "$TOOLS/$PARSER" "$OUTPUTDIR" $FILES if [ ! "$NO_PARSE" ]; then if [ "$XMLFILEPATTERN" ]; then - if ! sudo_command "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_xml2tab.py" "$OUTPUTDIR" "$SOURCE" $FILES ; then + if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_xml2tab.py" "$OUTPUTDIR" "$SOURCE" $FILES ; then echo "$PROGNAME: XML processing for source $SOURCE failed." 1>&2 exit 1 fi @@ -195,7 +185,7 @@ for FILEPATTERN in $FILEPATTERNS; do echo "$PROGNAME: No parser defined for source $SOURCE." 1>&2 exit 1 else - if ! sudo_command "$TOOLS/$PARSER" "$OUTPUTDIR" $FILES ; then + if ! "$TOOLS/$PARSER" "$OUTPUTDIR" $FILES ; then echo "$PROGNAME: Data processing for source $SOURCE failed." 1>&2 exit 1 fi From 47905b5c747a7031721a15b8a766c61a172a0db4 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Thu, 16 May 2024 10:41:26 +0200 Subject: [PATCH 30/74] Update ETN-EliasSteyaert-stage.Rmd Updating the ETN 'till wednesday 15/05/2024 --- .../ETN-EliasSteyaert-stage.Rmd | 62 +++++++++++++++++-- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index b81bf7c5..41bac607 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -85,7 +85,7 @@ Connecting to the instance. #### Tuesday 30/04/2024 -##### Problemsoloving the Irdownload playbook +##### Problem-soloving the Irdownload playbook - Irdownload doesn’t work anymore so we will be adjusting the irdata-config so it can work again. @@ -96,23 +96,75 @@ Connecting to the instance. #### Thursday 02/05/2024 -##### Problemsolving the Irunpack playbook +##### Problem-solving the Irunpack playbook - Running into problems when performing the Irunpack playbook, existing_dirs wasn’t defined and the location of the files were wrong -##### Problemsolving the Irparse playbook +##### Problem-solving the Irparse playbook - Running into problems when performing the Irparse playbook, pip3 has to be installed with apt, not dnf #### Friday 03/05/2024 -##### Problemsolving new problems the Irparse playbook brings with it +##### Problem-solving new problems the Irparse playbook brings with it - Running into new problems while running the irparse playbook: creating a venv and trying to get the irdata package downloaded #### Monday 06/05/2024 -##### Problemsolving new problems the Irparse playbook brings with it +##### Problem-solving new problems the Irparse playbook brings with it - Running into different kinds of problems when running the irparse playbook. The error says that the space is full but that doesn’t seem to be the problem. +- Adding a python interpreter as an environment for the .py scripts + + +#### Thursday 07/05/2024 + +##### Problem-solving new problems the Irparse playbook brings with it +- Being stuck on the irparse.yml playbook. +- Trying to figure out how to solve the “need more space” error. + + +#### Wednesday 08/05/2024 + +##### Problem-solving new problems the Irparse playbook brings with it +- Creating a "for loop" to get the python interpreter into the scripts which need the environment to solve “from irdata import data ModuleNotFoundError: No module named 'irdata'” +- for filename in $(grep -rl "TOOLS" ./usr/bin/* | sort | uniq); do cat "$filename" | sed "s|\(\$TOOLS/.*.py\)|\${USE_PYTHON_INTERPRETER}\" \"\1|g" > "$filename.tmp" && mv "$filename.tmp" "$filename"; done + + +#### Thursday 09/05/204 + +##### Ascension Day + + +#### Friday 10/05/2024 + +##### Problem-solving new problems the Irparse playbook brings with it +- Trying to expand the disk space to solve the problem of “need more space”. Doing so I messed up the server by trying to make a new partition. +- I made the server crash. Alex setted up a new server. The IP-adress in the file "inventory" must be updated to the new one. To get the /mnt/disks/data back (linked to the /dev/sda), I followed the following site of google itself: https://cloud.google.com/compute/docs/disks/format-mount-disk-linux +After the /mnt/disks/data was set-up, the playbook could be played from my VM onto the google cloud instance. + + +#### Monday 13/05/2024 + +##### Playing the playbook all over again +- Starting over again with the playbook: changing the inventory, playing main1.yml, after_main1.yml and main2.yml till it got stuck at irparse.yml + +##### Problem-solving new problems the Irparse playbook brings with it +- Trying to find a solution for the "from irdata import data as irdt ModuleNotFoundError: No module named 'irdata'" + + +#### Tuesday 14/05/2024 + +##### Problem-solving new problems the Irparse playbook brings with it +- Trying to get the Irparse.yml running +- Learning more about binairy and how a kernel works. Learning more about the hashbang, how it works and why it is there. + + +#### Wednesday 15/05/2024 + +##### Problem-solving new problems the Irparse playbook brings with it +- Looking for a solution to get Irparse.yml running with the knowledge of the day before. +- Running into a new error: "/home/irefindex/usr/bin/irdata_parse_bind.sh: line 94: : command not found" where the line it references is a line with the python interpreter. +- Trying little things like installing pip into the environment, even though it should be already there From eed975b4662cd5d8ce17ddd511a7b7ca4528c33f Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Thu, 16 May 2024 10:45:15 +0200 Subject: [PATCH 31/74] Update ETN-EliasSteyaert-stage.Rmd Adding the learning outcome and the TDP to the ETN --- .../ETN-EliasSteyaert-stage.Rmd | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index 41bac607..4cac1120 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -173,6 +173,63 @@ After the /mnt/disks/data was set-up, the playbook could be played from my VM on +### Trainee documentation plan (TDP) + +#####Tentative planning: +Which tasks will be performed during the traineeship (briefly)? +The main task of this internship is to go look at all the steps involved in the iRefIndex pipeline. This pipeline is like a big system for gathering and organizing information about how proteins interact with each other. It collects data from different places and puts it all together in one place. By going through each part of this process, we want to improve the automatization of using this dataset. That means fewer mistakes and quicker fixes if something goes wrong. This helps other users to get the protein interaction data they need faster and with fewer errors. + +##### Data management: +Use the FAIR principle (Table 2) as a guide to store and structure your data. +The structure of my data is stored in my GitHub, in its own repository. This can be found in the following link: https://github.com/EliasSteyaert/stage-VIB-irefindex.git + +##### Which aspects of the FAIR principle can be applied to your documentation system? +All of the aspects of the FAIR principle can be applied: +- Findability: The data used originates from the iRefIndex and Complex Portal databases. iRefIndex data, available at ‘https://irefindex.vib.be/wiki/index.php/iRefIndex', provides unique identifiers for interactors, taxonomy identifiers, interaction types, and interaction sources. Each protein interaction record and participant protein has a corresponding key. + - Accesiblity: The iRefIndex database, available at 'https://irefindex.vib.be/wiki/index.php/iRefIndex', provides version 19.0 data in PSI-MITAB format, specifically for human (taxon id: 9606). The dataset can be accessed at 'https://storage.googleapis.com/irefindex-data/archive/release_19.0/psi_mitab/MITAB2.6/9606.mitab.08-22-2022.txt'. Additionally, the dataset for all organisms in iRefIndex is available at 'https://storage.googleapis.com/irefindex-data/archive/release_19.0/psi_mitab/MITAB2.6/All.mitab.08-22-2022.txt'. +- Interoperability: The iRefIndex data is accessible in PSI-MITAB 2.6 tab-delimited format. +- Reproducibility: the iRefIndex data can be reused using the method that can be found on the VIB GitHub: https://github.com/EliasSteyaert/stage-VIB-irefindex.git + +##### Traceability of steps and methods: +Which platform or application (e.g. Markdown pages on Git, Word document ...) will be used to document the project steps and progression in a traceable manner? +There is a Rmarkdown file in my GitHub repository. In this Rmarkdown file is my ETN (Electronic Traineeship Notebook), where there is a description of the actions performed during each day. + +##### Version control of code: +Which Git repository will be used to store the code? +The code will be stored in my own GitHub repository which is a clone of the fork of the original project of the VIB. The GitHub repository can be found with the following link: https://github.com/EliasSteyaert/stage-VIB-irefindex.git +How will the repository be made available for the internal and external supervisors? +The status of the repository is set on public. Everyone can see the code on my GitHub repository. + + +### The Learning Outcome (LO) + +##### Development goal: +Trying to learn/become better at using ansible so I can automate processes easier. + +##### Development activity: +I will be working with ansible playbooks, find errors and parts of it that don’t work properly and fix them so it can be more automated. + +##### Desired results: +Being able to make a fully running ansible playbook so that other users don’t have to do anything except to run the playbook. + +##### Schedule: +These activities will be performed throughout the internship (on daily basis). + +##### Necessary support and facilities: +Normally, all this should be done in the traineeship and its 250 hours. Nothing should be bought for this goal to be achieved. + +##### SMART-principle: +- Specific: learning how to use ansible and make my own playbooks. +- Measurable: At the end of the traineeship I should be able to make my own working ansible playbook. +- Achievable: As I’m working daily with ansible for my traineeship, I should be able to achieve this goal. +- Relevant: In the context of the traineeship, this learning outcome is relevant. +- Time-bound: This should be achieved by the end of the traineeship. + + + + + + From 98e2fdd47ecffee3a14927034221f7107207e3fb Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 17 May 2024 14:46:38 +0200 Subject: [PATCH 32/74] changinng the urls of uniprot and innatedb so they can be downloaded properly --- usr/bin/irdata-config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usr/bin/irdata-config b/usr/bin/irdata-config index 13d5f3b5..5ff04fe4 100644 --- a/usr/bin/irdata-config +++ b/usr/bin/irdata-config @@ -326,7 +326,7 @@ INTCOMPLEX_XML_FILE_PATTERN='pmid/*/*.xml *.xml' # InnateDB # OK INNATEDB_DOWNLOAD_FILES="innatedb_all.mitab.gz" -INNATEDB_MIRROR='http://www.innatedb.com' +INNATEDB_MIRROR='https://www.innatedb.com' INNATEDB_RELEASE_URL="$INNATEDB_MIRROR/download/interactions/" INNATEDB_DIR_NAME="INNATEDB" @@ -563,7 +563,7 @@ TAXONOMY_SCHEMA='taxonomy' # UniProt # OK UNIPROT_DOWNLOAD_FILES="uniprot_sprot.dat.gz uniprot_trembl.dat.gz uniprot_sprot_varsplic.fasta.gz reldate.txt" -UNIPROT_MIRROR='ftp://ftp.ebi.ac.uk' +UNIPROT_MIRROR='https://ftp.ebi.ac.uk' UNIPROT_RELEASE='current_release' UNIPROT_RELEASE_URL="$UNIPROT_MIRROR/pub/databases/uniprot/$UNIPROT_RELEASE/knowledgebase/complete/" @@ -579,7 +579,7 @@ UNIPROT_MANIFEST_FILE='reldate.txt' # UniProt (Fly) # OK FLY_DOWNLOAD_FILES="fly.txt" -FLY_MIRROR='ftp://ftp.uniprot.org' +FLY_MIRROR='https://ftp.uniprot.org' FLY_RELEASE_URL="$FLY_MIRROR/pub/databases/uniprot/current_release/knowledgebase/complete/docs/" FLY_DIR_NAME="fly" @@ -594,7 +594,7 @@ FLY_MANIFEST_FILE='fly.txt' # UniProt (Yeast) # OK YEAST_DOWNLOAD_FILES="yeast.txt" -YEAST_MIRROR='ftp://ftp.uniprot.org' +YEAST_MIRROR='https://ftp.uniprot.org' YEAST_RELEASE_URL="$YEAST_MIRROR/pub/databases/uniprot/current_release/knowledgebase/complete/docs/" YEAST_DIR_NAME="yeast" From a8d944ad22c4ac5e97256ab3743966ddb5ca6a33 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 17 May 2024 14:47:25 +0200 Subject: [PATCH 33/74] deleting comments so more databases gets downloaded --- vsc/ansible/vars/sources.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vsc/ansible/vars/sources.yml b/vsc/ansible/vars/sources.yml index cd7d5f56..c6e150a6 100644 --- a/vsc/ansible/vars/sources.yml +++ b/vsc/ansible/vars/sources.yml @@ -70,10 +70,10 @@ jobs: # - { name: "corum", download_runner_async: 60, download_retries: 12, download_delay: 15 } # - { name: "matrixdb", download_runner_async: 300, download_retries: 30, download_delay: 30, parse_async: 200, parse_retries: 8, parse_delay: 25, import_async: 160, import_delay: 10, import_retries: 16 } - { name: "uniprotpp", download_runner_async: 300, download_retries: 30, download_delay: 30, parse_async: 120, import_async: 160, import_delay: 10, import_retries: 16 } - # - { name: "uniprot", download_runner_async: 2000, download_retries: 60, download_delay: 100, parse_async: 36000, parse_retries: 60, parse_delay: 600 } + - { name: "uniprot", download_runner_async: 2000, download_retries: 60, download_delay: 100, parse_async: 36000, parse_retries: 60, parse_delay: 600 } - { name: "mint", download_runner_async: 600, download_retries: 60, download_delay: 30, parse_async: 120, import_async: 400, import_delay: 30, import_retries: 14 } - { name: "gene", download_runner_async: 800, download_retries: 40, download_delay: 60, parse_async: 1600, parse_retries: 27, parse_delay: 60, import_async: 4000, import_retries: 34, import_delay: 120 } - { name: "reactome", download_runner_async: 300, download_retries: 30, download_delay: 30, import_async: 300, import_retries: 10, import_delay: 30 } - # - { name: "imex", download_runner_async: 10800, download_retries: 108, download_delay: 300, parse_async: 600, parse_retries: 10, parse_delay: 60, import_async: 1500, import_retries: 25, import_delay: 60 } - # - { name: "refseq", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 450, parse_retries: 15, parse_delay: 30, import_async: 5000, import_retries: 21, import_delay: 240 } - # - { name: "genpept", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 21000, parse_retries: 60, parse_delay: 350, import_async: 5000, import_retries: 21, import_delay: 240 } + - { name: "imex", download_runner_async: 10800, download_retries: 108, download_delay: 300, parse_async: 600, parse_retries: 10, parse_delay: 60, import_async: 1500, import_retries: 25, import_delay: 60 } + - { name: "refseq", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 450, parse_retries: 15, parse_delay: 30, import_async: 5000, import_retries: 21, import_delay: 240 } + - { name: "genpept", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 21000, parse_retries: 60, parse_delay: 350, import_async: 5000, import_retries: 21, import_delay: 240 } From e97c396bfc2a13127f87447bc318027a447e59c3 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 17 May 2024 14:48:50 +0200 Subject: [PATCH 34/74] recreating a part of the playbook to do problem-solving easier/quicker --- usr/bin/pythonscript.py | 12 +++++ usr/bin/shellscript.sh | 8 ++++ vsc/ansible/Tryout2.yml | 78 +++++++++++++++++++++++++++++++++ vsc/ansible/recreatingerror.yml | 3 ++ vsc/ansible/shellscript.yml | 77 ++++++++++++++++++++++++++++++++ 5 files changed, 178 insertions(+) create mode 100644 usr/bin/pythonscript.py create mode 100644 usr/bin/shellscript.sh create mode 100644 vsc/ansible/Tryout2.yml create mode 100644 vsc/ansible/recreatingerror.yml create mode 100644 vsc/ansible/shellscript.yml diff --git a/usr/bin/pythonscript.py b/usr/bin/pythonscript.py new file mode 100644 index 00000000..fcb351d5 --- /dev/null +++ b/usr/bin/pythonscript.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 + +from irdata import data as irdt +import subprocess + +print(subprocess.getoutput("irdata --version")) + +print("would be cool if this worked") + + + +#- pythonscript: import irdata, print irdata version, print a stupid text \ No newline at end of file diff --git a/usr/bin/shellscript.sh b/usr/bin/shellscript.sh new file mode 100644 index 00000000..cd88d9df --- /dev/null +++ b/usr/bin/shellscript.sh @@ -0,0 +1,8 @@ +#!/usr/bin/bash + +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + +${USE_PYTHON_INTERPRETER} /home/irefindex/usr/bin/pythonscript.py + + +#- shellscript: set up use_python_interpreter, run python script diff --git a/vsc/ansible/Tryout2.yml b/vsc/ansible/Tryout2.yml new file mode 100644 index 00000000..47b7d6b4 --- /dev/null +++ b/vsc/ansible/Tryout2.yml @@ -0,0 +1,78 @@ +--- +- name: shellscript + hosts: all + gather_facts: true # WE NEED THIS HERE FOR THE DATES FOR THE LOGS! + become: true + vars_files: + - ./vars/general.yml + + vars: + local_source_directory: "/home/guest/internship/stage-VIB-irefindex/usr/bin" # Directory containing the files on your local machine + remote_destination_directory: "/home/irefindex/usr/bin" # Destination directory on the remote machine + ansible_python_interpreter: /usr/bin/python3 # Path to Python 3 interpreter + + + tasks: + - name: Copy shellscript.sh from local to remote + ansible.builtin.copy: + src: "{{ local_source_directory }}/shellscript.sh" + dest: "{{ remote_destination_directory }}/shellscript.sh" + + - name: Copy pythonscript.py from local to remote + ansible.builtin.copy: + src: "{{ local_source_directory }}/pythonscript.py" + dest: "{{ remote_destination_directory }}/pythonscript.py" + + - name: Install python3-venv package + ansible.builtin.apt: + name: python3-venv + state: present + + - name: Create virtual environment + ansible.builtin.command: + cmd: python3 -m venv "{{ download_data }}/venv" + + - name: Activate virtual environment + ansible.builtin.command: + cmd: "bash -c 'source /mnt/disks/data/venv/bin/activate'" + + - name: Install pip for Python 3 + apt: + name: python3-pip + state: present + + - name: Install irdata package + pip: + name: irdata + state: present + + #- name: Install irdata package + # environment: + # python_interpreter: "{{ download_data }}/venv/bin/python3" + # ansible.builtin.pip: + # name: irdata + # state: present + + - name: Find shellscript.sh and pythonscript.py files + find: + paths: /home/irefindex/usr/bin + patterns: "shellscript.sh,pythonscript.py" + register: files_to_update + + - name: Give execute permissions to shellscript.sh and pythonscript.py files + file: + path: "{{ item.path }}" + mode: u+x,g+x,o+x + loop: "{{ files_to_update.files }}" + when: files_to_update.matched > 0 + + - name: Execute shell script + environment: + python_interpreter: "{{ download_data }}/venv/bin/python3" + command: "/home/irefindex/usr/bin/shellscript.sh" + + +#To do: +#- playbook: making a venv, running a shell script +#- shellscript: set up use_python_interpreter, run python script +#- pythonscript: import irdata, print irdata version, print a stupid text \ No newline at end of file diff --git a/vsc/ansible/recreatingerror.yml b/vsc/ansible/recreatingerror.yml new file mode 100644 index 00000000..56de088a --- /dev/null +++ b/vsc/ansible/recreatingerror.yml @@ -0,0 +1,3 @@ +--- +- name: Run shellscript playbook + import_playbook: shellscript.yml \ No newline at end of file diff --git a/vsc/ansible/shellscript.yml b/vsc/ansible/shellscript.yml new file mode 100644 index 00000000..dbb6e7b8 --- /dev/null +++ b/vsc/ansible/shellscript.yml @@ -0,0 +1,77 @@ +--- +- name: shellscript + hosts: all + gather_facts: true # WE NEED THIS HERE FOR THE DATES FOR THE LOGS! + become: true + vars_files: + - ./vars/general.yml + + vars: + local_source_directory: "/home/guest/internship/stage-VIB-irefindex/usr/bin" # Directory containing the files on your local machine + remote_destination_directory: "/home/irefindex/usr/bin" # Destination directory on the remote machine + + tasks: + - name: Copy shellscript.sh from local to remote + ansible.builtin.copy: + src: "{{ local_source_directory }}/shellscript.sh" + dest: "{{ remote_destination_directory }}/shellscript.sh" + + - name: Copy pythonscript.py from local to remote + ansible.builtin.copy: + src: "{{ local_source_directory }}/pythonscript.py" + dest: "{{ remote_destination_directory }}/pythonscript.py" + + - name: Install python3-venv package + ansible.builtin.apt: + name: python3-venv + state: present + + - name: Create virtual environment + ansible.builtin.command: + cmd: python3 -m venv "{{ download_data }}/venv" + + - name: Activate virtual environment + ansible.builtin.command: + cmd: "bash -c 'source /mnt/disks/data/venv/bin/activate'" + + - name: Install pip for Python 3 + apt: + name: python3-pip + state: present + + - name: Install irdata package + pip: + name: irdata + executable: pip3 # Use 'pip3' for Python 3 + state: present + + #- name: Install irdata package + # environment: + # python_interpreter: "{{ download_data }}/venv/bin/python3" + # ansible.builtin.pip: + # name: irdata + # state: present + + - name: Find shellscript.sh and pythonscript.py files + find: + paths: /home/irefindex/usr/bin + patterns: "shellscript.sh,pythonscript.py" + register: files_to_update + + - name: Give execute permissions to shellscript.sh and pythonscript.py files + file: + path: "{{ item.path }}" + mode: u+x,g+x,o+x + loop: "{{ files_to_update.files }}" + when: files_to_update.matched > 0 + + - name: Execute shell script + environment: + python_interpreter: "{{ download_data }}/venv/bin/python3" + command: "/home/irefindex/usr/bin/shellscript.sh" + + +#To do: +#- playbook: making a venv, running a shell script +#- shellscript: set up use_python_interpreter, run python script +#- pythonscript: import irdata, print irdata version, print a stupid text \ No newline at end of file From 04b8a6ad4049407ce300fc0c7c9e948cc6c99122 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 17 May 2024 19:34:23 +0200 Subject: [PATCH 35/74] changing the vars of sources to the sources2.yml one --- vsc/ansible/irdownload.yml | 83 ++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/vsc/ansible/irdownload.yml b/vsc/ansible/irdownload.yml index 2529fc54..4145d7a3 100644 --- a/vsc/ansible/irdownload.yml +++ b/vsc/ansible/irdownload.yml @@ -5,50 +5,50 @@ gather_facts: true # WE NEED THIS HERE FOR THE DATES FOR THE LOGS! vars_files: - ./vars/logs.yml - - ./vars/sources.yml + - ./vars/sources2.yml vars: log_location: "{{ logs_dir }}/irdownload" tasks: - - name: Update apt packages - ansible.builtin.apt: - update_cache: true - - - name: Install required packages - ansible.builtin.apt: - name: "{{ item }}" - state: present - loop: - - apt-transport-https - - ca-certificates - - gnupg - - curl - - sudo - - - name: Add Google Cloud SDK repository - ansible.builtin.lineinfile: - dest: /etc/apt/sources.list.d/google-cloud-sdk.list - line: "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main" - create: true - mode: "0644" - - - name: Download Google Cloud SDK key - ansible.builtin.get_url: - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg - dest: /usr/share/keyrings/cloud.google.asc - mode: "0644" - - - name: Update apt packages with Google Cloud SDK repository - ansible.builtin.apt: - update_cache: true - force_apt_get: true - - - name: Install Google Cloud CLI - ansible.builtin.apt: - name: google-cloud-cli - state: present - + #- name: Update apt packages + # ansible.builtin.apt: + # update_cache: true +# + #- name: Install required packages + # ansible.builtin.apt: + # name: "{{ item }}" + # state: present + # loop: + # - apt-transport-https + # - ca-certificates + # - gnupg + # - curl + # - sudo +# + #- name: Add Google Cloud SDK repository + # ansible.builtin.lineinfile: + # dest: /etc/apt/sources.list.d/google-cloud-sdk.list + # line: "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main" + # create: true + # mode: "0644" +# + #- name: Download Google Cloud SDK key + # ansible.builtin.get_url: + # url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + # dest: /usr/share/keyrings/cloud.google.asc + # mode: "0644" +# + #- name: Update apt packages with Google Cloud SDK repository + # ansible.builtin.apt: + # update_cache: true + # force_apt_get: true +# + #- name: Install Google Cloud CLI + # ansible.builtin.apt: + # name: google-cloud-cli + # state: present +# - name: Run jobs ansible.builtin.command: "./irdownload {{ item.name | upper }}" args: @@ -61,6 +61,7 @@ retries: "{{ item.download_runner_retries | default(18) }}" - name: Wait for jobs to complete + debugger: always ansible.builtin.async_status: jid: "{{ item.ansible_job_id }}" delay: "{{ item.item.download_delay | default(5) }}" @@ -70,6 +71,10 @@ until: job_result.finished == true loop: "{{ result.results }}" + - name: Debug job_result + ansible.builtin.debug: + var: job_result + - name: "Ensure that `log_location` exists" ansible.builtin.file: path: "{{ log_location }}" From c42db467d48aa0d6a578cb96fd16aa89db6c2a9c Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 17 May 2024 19:34:50 +0200 Subject: [PATCH 36/74] making a smaller sample of the jobs to make it run faster --- vsc/ansible/vars/sources2.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 vsc/ansible/vars/sources2.yml diff --git a/vsc/ansible/vars/sources2.yml b/vsc/ansible/vars/sources2.yml new file mode 100644 index 00000000..220fe82e --- /dev/null +++ b/vsc/ansible/vars/sources2.yml @@ -0,0 +1,2 @@ +jobs: + - { name: "mint", download_runner_async: 600, download_retries: 1, download_delay: 30, parse_async: 120, import_async: 400, import_delay: 30, import_retries: 14 } \ No newline at end of file From 46c7d6eb3f08b07ac9d29d9fa72542548318edd5 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 22 May 2024 10:36:52 +0200 Subject: [PATCH 37/74] New sources file to test irparse on one file at a time --- vsc/ansible/vars/sources3.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 vsc/ansible/vars/sources3.yml diff --git a/vsc/ansible/vars/sources3.yml b/vsc/ansible/vars/sources3.yml new file mode 100644 index 00000000..91db1364 --- /dev/null +++ b/vsc/ansible/vars/sources3.yml @@ -0,0 +1,3 @@ +jobs: + - { name: "athaliana" } + - { name: "bind" } \ No newline at end of file From 9a707bfb606cfd85d8d9175ba4712dbab4b68860 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 22 May 2024 10:38:30 +0200 Subject: [PATCH 38/74] last changes to make the mini script work --- usr/bin/pythonscript.py | 7 ++-- usr/bin/shellscript.sh | 1 - vsc/ansible/shellscript.yml | 65 ++++++++++++++++++++----------------- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/usr/bin/pythonscript.py b/usr/bin/pythonscript.py index fcb351d5..6ed810c7 100644 --- a/usr/bin/pythonscript.py +++ b/usr/bin/pythonscript.py @@ -1,12 +1,11 @@ #!/usr/bin/python3 -from irdata import data as irdt -import subprocess - -print(subprocess.getoutput("irdata --version")) +import irdata +print(irdata.__file__) print("would be cool if this worked") + #- pythonscript: import irdata, print irdata version, print a stupid text \ No newline at end of file diff --git a/usr/bin/shellscript.sh b/usr/bin/shellscript.sh index cd88d9df..b0d92a6d 100644 --- a/usr/bin/shellscript.sh +++ b/usr/bin/shellscript.sh @@ -1,7 +1,6 @@ #!/usr/bin/bash USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - ${USE_PYTHON_INTERPRETER} /home/irefindex/usr/bin/pythonscript.py diff --git a/vsc/ansible/shellscript.yml b/vsc/ansible/shellscript.yml index dbb6e7b8..254528b4 100644 --- a/vsc/ansible/shellscript.yml +++ b/vsc/ansible/shellscript.yml @@ -15,36 +15,50 @@ ansible.builtin.copy: src: "{{ local_source_directory }}/shellscript.sh" dest: "{{ remote_destination_directory }}/shellscript.sh" + mode: u+x,g+x,o+x - name: Copy pythonscript.py from local to remote ansible.builtin.copy: src: "{{ local_source_directory }}/pythonscript.py" dest: "{{ remote_destination_directory }}/pythonscript.py" + mode: u+x,g+x,o+x - - name: Install python3-venv package + - name: Ensure python3-venv and python3-pip are installed ansible.builtin.apt: - name: python3-venv + name: + - python3-venv + - python3-pip state: present - name: Create virtual environment ansible.builtin.command: - cmd: python3 -m venv "{{ download_data }}/venv" + cmd: python3 -m venv "{{ download_data }}/venv" + args: + creates: "{{ download_data }}/venv" - - name: Activate virtual environment + - name: install irdata ansible.builtin.command: - cmd: "bash -c 'source /mnt/disks/data/venv/bin/activate'" + cmd: "{{ download_data }}/venv/bin/python3 -m pip install irdata" - - name: Install pip for Python 3 - apt: - name: python3-pip - state: present - - - name: Install irdata package - pip: - name: irdata - executable: pip3 # Use 'pip3' for Python 3 - state: present + + #args: + # creates: "{{ download_data }}/venv/lib/python3.8/site-packages/irdata" + # - name: Activate virtual environment + # ansible.builtin.command: + # cmd: "bash -c 'source /mnt/disks/data/venv/bin/activate'" +# + # - name: Install pip for Python 3 + # apt: + # name: python3-pip + # state: present +# + # - name: Install irdata package + # pip: + # name: irdata + # executable: pip3 # Use 'pip3' for Python 3 + # state: present +# #- name: Install irdata package # environment: # python_interpreter: "{{ download_data }}/venv/bin/python3" @@ -52,23 +66,16 @@ # name: irdata # state: present - - name: Find shellscript.sh and pythonscript.py files - find: - paths: /home/irefindex/usr/bin - patterns: "shellscript.sh,pythonscript.py" - register: files_to_update - - - name: Give execute permissions to shellscript.sh and pythonscript.py files - file: - path: "{{ item.path }}" - mode: u+x,g+x,o+x - loop: "{{ files_to_update.files }}" - when: files_to_update.matched > 0 - - name: Execute shell script environment: python_interpreter: "{{ download_data }}/venv/bin/python3" - command: "/home/irefindex/usr/bin/shellscript.sh" + ansible.builtin.command: + cmd: "/home/irefindex/usr/bin/shellscript.sh" + register: output + + - name: debug + ansible.builtin.debug: + msg: "{{ output.stdout }}" #To do: From 5712b8ac49e43e2a37a31439308e2ae71fb7dfc6 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 22 May 2024 10:40:22 +0200 Subject: [PATCH 39/74] defining the interpreter's variable --- usr/bin/irdata_parse_A_thaliana.sh | 3 +++ usr/bin/irdata_parse_bind.sh | 3 +++ usr/bin/irinit | 3 +++ 3 files changed, 9 insertions(+) diff --git a/usr/bin/irdata_parse_A_thaliana.sh b/usr/bin/irdata_parse_A_thaliana.sh index 2c6c180f..50e1a751 100644 --- a/usr/bin/irdata_parse_A_thaliana.sh +++ b/usr/bin/irdata_parse_A_thaliana.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_bind.sh b/usr/bin/irdata_parse_bind.sh index 2eb9a10d..be4349f7 100644 --- a/usr/bin/irdata_parse_bind.sh +++ b/usr/bin/irdata_parse_bind.sh @@ -22,6 +22,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irinit b/usr/bin/irinit index 9cec478f..7ff65425 100644 --- a/usr/bin/irinit +++ b/usr/bin/irinit @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then From a67c0bb3e7d9062987b8488fd6ee91c3957720fb Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 22 May 2024 10:41:31 +0200 Subject: [PATCH 40/74] changes to make the irparse work and definitive way to install the irdata package --- vsc/ansible/irparse.yml | 75 +++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 689ccbfb..9133ede7 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -5,7 +5,7 @@ become: true vars_files: - ./vars/logs.yml - - ./vars/sources.yml + - ./vars/sources3.yml - ./vars/general.yml vars: @@ -13,36 +13,41 @@ USE_PYTHON_INTERPRETER: "{{ python_interpreter | default('/usr/bin/python3') }}" tasks: - - name: Set permissions on Ansible temporary directory - ansible.builtin.file: - path: "{{ home_dir }}/.ansible/tmp/" - state: directory - mode: "0755" - recurse: yes - become: yes - - - name: Install python3-venv package +#COMMENTED BECAUSE DOING THIS ONCE IS ENOUGH + #- name: Set permissions on Ansible temporary directory + # ansible.builtin.file: + # path: "{{ home_dir }}/.ansible/tmp/" + # state: directory + # mode: "0755" + # recurse: yes + # become: yes + + - name: Ensure python3-venv and python3-pip are installed ansible.builtin.apt: - name: python3-venv + name: + - python3-venv + - python3-pip state: present - name: Create virtual environment ansible.builtin.command: cmd: python3 -m venv "{{ download_data }}/{{ data2 }}/virtualenv" + args: + creates: "{{ download_data }}/{{ data2 }}/virtualenv" - - name: Install irdata module - pip: - name: irdata - executable: "{{ download_data }}/{{ data2 }}/virtualenv/bin/pip" - become: true + - name: install irdata + ansible.builtin.command: + cmd: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python3 -m pip install irdata" - - name: Find files containing "TOOLS" - find: - paths: /home/irefindex/usr/bin/ - patterns: "*" - recurse: yes - register: found_files +#THIS DOESN'T WORK YET BUT IS NEEDED FOR REPRODUCIBILTY + # - name: Find files containing "TOOLS" + # find: + # paths: /home/irefindex/usr/bin/ + # patterns: "*" + # recurse: yes + # register: found_files +#SAME AS THE COMMENT ABOVE, THESE BELONG TOGETHER # - name: Iterate over found files # vars: # use_python_interpreter: "{{ USE_PYTHON_INTERPRETER }}" @@ -52,17 +57,19 @@ # executable: /bin/bash # when: found_files.matched > 0 - - name: Find .sh and .py files - find: - paths: /home/irefindex/usr/bin - patterns: "*.sh,*.py" - register: files_to_update - - name: Give execute permissions to .sh and .py files - file: - path: "{{ item.path }}" - mode: u+x,g+x,o+x - loop: "{{ files_to_update.files }}" +# NEEDED FOR REPRODUCIBILTY, WORKS BUT TAKES A WHILE SO ITS COMMENTED + # - name: Find .sh and .py files + # find: + # paths: /home/irefindex/usr/bin + # patterns: "*.sh,*.py" + # register: files_to_update +# SAME AS THE COMMENT ABOVE, THESE BELONG TOGETHER + # - name: Give execute permissions to .sh and .py files + # file: + # path: "{{ item.path }}" + # mode: u+x,g+x,o+x + # loop: "{{ files_to_update.files }}" - name: Convert the downloaded data to files for each source ansible.builtin.command: "./irparse {{ item.name | upper }}" @@ -92,10 +99,6 @@ - name: Debug job_result ansible.builtin.debug: var: job_result - - - name: df -h - ansible.builtin.command: "df -h" - register: disk_space - name: "Ensure that `log_location` exists" ansible.builtin.file: From a8464e53dc0fa1b1c91aa93279bf28deb5d837a0 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Thu, 23 May 2024 14:56:40 +0200 Subject: [PATCH 41/74] adding the definition of the variable USE_PYTHON_INTERPRETER --- usr/bin/irdata-config | 2 +- usr/bin/irdata_parse_bhfucl.sh | 3 +++ usr/bin/irdata_parse_dig.sh | 3 +++ usr/bin/irdata_parse_fly.sh | 3 +++ usr/bin/irdata_parse_hpidb.sh | 4 ++++ usr/bin/irdata_parse_huri.sh | 3 +++ usr/bin/irdata_parse_imex.sh | 3 +++ usr/bin/irdata_parse_innatedb.sh | 3 +++ usr/bin/irdata_parse_mbinfo.sh | 3 +++ usr/bin/irdata_parse_mint.sh | 3 +++ usr/bin/irdata_parse_mpidb.sh | 3 +++ usr/bin/irdata_parse_pdb.sh | 3 +++ usr/bin/irdata_parse_quickgo.sh | 3 +++ usr/bin/irdata_parse_reactome.sh | 3 +++ usr/bin/irdata_parse_uniprotpp.sh | 3 +++ usr/bin/irdata_parse_virushost.sh | 3 +++ usr/bin/irdata_process_signatures.sh | 3 +++ 17 files changed, 50 insertions(+), 1 deletion(-) diff --git a/usr/bin/irdata-config b/usr/bin/irdata-config index 5ff04fe4..179751b2 100644 --- a/usr/bin/irdata-config +++ b/usr/bin/irdata-config @@ -18,7 +18,7 @@ EMAIL="alexander.botzki@vib.be" # e-mail address for anonymous FTP l DATABASE="irdata${RELEASE_MAJOR}" # database name DATA="${DATAPREFIX}dataext/irdata${RELEASE_MAJOR}" # user defined data directory location WIKIFORMAT="MediaWiki" # format for generated Wiki pages -CURL_OPT="-L" # Follow HTTP redirects +CURL_OPT="-kL" # Follow HTTP redirects WGET_WAIT="5" # time to wait between wget requests WGET_ATTEMPTS="5" # number of attempts to make using wget PSQL_OPTIONS="" # "--psql-options " diff --git a/usr/bin/irdata_parse_bhfucl.sh b/usr/bin/irdata_parse_bhfucl.sh index 2106c8a1..fbb93013 100644 --- a/usr/bin/irdata_parse_bhfucl.sh +++ b/usr/bin/irdata_parse_bhfucl.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_dig.sh b/usr/bin/irdata_parse_dig.sh index 9473c380..1ca3c021 100644 --- a/usr/bin/irdata_parse_dig.sh +++ b/usr/bin/irdata_parse_dig.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_fly.sh b/usr/bin/irdata_parse_fly.sh index ea114c8d..b105983d 100644 --- a/usr/bin/irdata_parse_fly.sh +++ b/usr/bin/irdata_parse_fly.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_hpidb.sh b/usr/bin/irdata_parse_hpidb.sh index f03d9bac..b4fec9d8 100644 --- a/usr/bin/irdata_parse_hpidb.sh +++ b/usr/bin/irdata_parse_hpidb.sh @@ -15,6 +15,10 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . + +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_huri.sh b/usr/bin/irdata_parse_huri.sh index 16aa5723..a03659eb 100644 --- a/usr/bin/irdata_parse_huri.sh +++ b/usr/bin/irdata_parse_huri.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_imex.sh b/usr/bin/irdata_parse_imex.sh index 1425912c..5447dd5d 100644 --- a/usr/bin/irdata_parse_imex.sh +++ b/usr/bin/irdata_parse_imex.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_innatedb.sh b/usr/bin/irdata_parse_innatedb.sh index d57723e1..567537b8 100644 --- a/usr/bin/irdata_parse_innatedb.sh +++ b/usr/bin/irdata_parse_innatedb.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_mbinfo.sh b/usr/bin/irdata_parse_mbinfo.sh index f3fd9f6c..a73da424 100644 --- a/usr/bin/irdata_parse_mbinfo.sh +++ b/usr/bin/irdata_parse_mbinfo.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_mint.sh b/usr/bin/irdata_parse_mint.sh index aa2246aa..24d1de57 100644 --- a/usr/bin/irdata_parse_mint.sh +++ b/usr/bin/irdata_parse_mint.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_mpidb.sh b/usr/bin/irdata_parse_mpidb.sh index 06159107..a0c26bae 100644 --- a/usr/bin/irdata_parse_mpidb.sh +++ b/usr/bin/irdata_parse_mpidb.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_pdb.sh b/usr/bin/irdata_parse_pdb.sh index f100aba3..9c48bdde 100644 --- a/usr/bin/irdata_parse_pdb.sh +++ b/usr/bin/irdata_parse_pdb.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_quickgo.sh b/usr/bin/irdata_parse_quickgo.sh index 6ec30804..8c4ce437 100644 --- a/usr/bin/irdata_parse_quickgo.sh +++ b/usr/bin/irdata_parse_quickgo.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_reactome.sh b/usr/bin/irdata_parse_reactome.sh index 1498bdba..0bfe20bd 100644 --- a/usr/bin/irdata_parse_reactome.sh +++ b/usr/bin/irdata_parse_reactome.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_uniprotpp.sh b/usr/bin/irdata_parse_uniprotpp.sh index c13341c8..38d7a094 100644 --- a/usr/bin/irdata_parse_uniprotpp.sh +++ b/usr/bin/irdata_parse_uniprotpp.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_virushost.sh b/usr/bin/irdata_parse_virushost.sh index 502d2483..e92f5b6e 100644 --- a/usr/bin/irdata_parse_virushost.sh +++ b/usr/bin/irdata_parse_virushost.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_process_signatures.sh b/usr/bin/irdata_process_signatures.sh index a9dc38f7..490bff83 100644 --- a/usr/bin/irdata_process_signatures.sh +++ b/usr/bin/irdata_process_signatures.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then From f9132b0582d870caadc03511886840dd70f89012 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 24 May 2024 17:14:07 +0200 Subject: [PATCH 42/74] adding a ansible config file to change the directory of /tmp/.ansible --- vsc/ansible/ansible.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 vsc/ansible/ansible.cfg diff --git a/vsc/ansible/ansible.cfg b/vsc/ansible/ansible.cfg new file mode 100644 index 00000000..cabd4459 --- /dev/null +++ b/vsc/ansible/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +remote_tmp = /mnt/disks/data \ No newline at end of file From 3dd481e9d9ac347633159fc4f7c682d7657be886 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 24 May 2024 17:15:11 +0200 Subject: [PATCH 43/74] script to see free disk space of /home/guest/tmp --- vsc/ansible/time_debug.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 vsc/ansible/time_debug.py diff --git a/vsc/ansible/time_debug.py b/vsc/ansible/time_debug.py new file mode 100644 index 00000000..e4f025b0 --- /dev/null +++ b/vsc/ansible/time_debug.py @@ -0,0 +1,16 @@ +import csv +import shutil +import os +import time + +log = open("/tmp/disk_free.csv", mode='w') +writer = csv.writer(log) + +while True: + # Get disk usage + home = os.environ.get("HOME") + free = shutil.disk_usage(f"{home}/.ansible").free + writer.writerow([free]) + time.sleep(1) + + From 2ad57c77a5786c24276d913df83ef2723c87c35a Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 29 May 2024 09:37:36 +0200 Subject: [PATCH 44/74] putting temporary echo's into the file to see what it loops specifically --- usr/bin/argument-per-line | 1 - 1 file changed, 1 deletion(-) diff --git a/usr/bin/argument-per-line b/usr/bin/argument-per-line index 1aa85152..bfc004f1 100755 --- a/usr/bin/argument-per-line +++ b/usr/bin/argument-per-line @@ -1,5 +1,4 @@ #!/bin/bash - for ARG in $*; do echo $ARG done From 49ab22c7ea4de631f0fcc5ddb26dca1f16814d51 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 29 May 2024 09:39:38 +0200 Subject: [PATCH 45/74] minor changes to replace a bit of code that didn't make sense --- usr/bin/irdata_parse_refseq.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/usr/bin/irdata_parse_refseq.py b/usr/bin/irdata_parse_refseq.py index 19519ff3..18817eb2 100755 --- a/usr/bin/irdata_parse_refseq.py +++ b/usr/bin/irdata_parse_refseq.py @@ -218,9 +218,8 @@ def write_record(self, record): progname = os.path.basename(sys.argv[0]) try: - i = 1 - data_directory = sys.argv[i] - filenames = sys.argv[i + 1 :] + data_directory = sys.argv[1] + filenames = sys.argv[2:] except IndexError: print( "Usage: %s ..." % progname, From 2c8da3147d2008af7b7d54f8cab36f7135738436 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 29 May 2024 09:42:25 +0200 Subject: [PATCH 46/74] big change for the interpreter to work with the irparallel script, needed for genpept, refseq and uniprot --- usr/bin/irparallel | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/usr/bin/irparallel b/usr/bin/irparallel index 77040569..26a36069 100755 --- a/usr/bin/irparallel +++ b/usr/bin/irparallel @@ -15,6 +15,10 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} +current_time=$(date +"%H:%M:%S") + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -41,15 +45,49 @@ EOF exit 1 fi + COMMAND=$1 +# Log the value of $1 +echo "irparallel: \$1 is $1" 1>&2 +# Log the value of $1 +echo "\$COMMAND is $COMMAND; current time: $current_time" > /home/irefindex/logfile2.log +echo "irparallel: \$1 is $1; current time: $current_time" >> /home/irefindex/logfile2.log # Make sure that certain arguments are defined. - if [ ! "$COMMAND" ]; then echo "$PROGNAME: Please specify a command to run on each file (to be given as {})." 1>&2 exit 1 fi + +EXPLODED=($COMMAND) +EXENAME=$(echo ${EXPLODED[0]} | sed 's/\"//g') +if [[ $(basename $EXENAME .py) == $(basename $EXENAME) ]]; then + COMMAND2="$COMMAND" +else + REST="\"$(echo ${EXPLODED[@]:1} | sed 's/\"//g')\""; + COMMAND2="$USE_PYTHON_INTERPRETER ${EXPLODED[0]} $REST" +fi +echo "\$REST is $REST" >> /home/irefindex/logfile2.log + +echo "$(basename $EXENAME .py)" >> /home/irefindex/logfile2.log +echo "$EXENAME" >> /home/irefindex/logfile2.log + +# Extract the first part of the COMMAND +#FIRST_PART=$(echo "$COMMAND" | awk '{print $1}' | tr -d '"') +# Check if the first part ends in .py +#if [[ "$FIRST_PART" == *.py ]]; then + # Prepend Python interpreter to the command +# COMMAND2="$USE_PYTHON_INTERPRETER $COMMAND" +#fi +#else + # Assign the command to COMMAND2 if it's not a Python script +# COMMAND2="$COMMAND" +#fi + +echo "The value of command2 after checking if it ends at .py: : $COMMAND2 ; current time: $current_time" >> /home/irefindex/logfile2.log + # The pipeline. +xargs -P $PROCESSES $XARGS_I'{}' sh -c "$COMMAND2" -xargs -P $PROCESSES $XARGS_I'{}' sh -c "$COMMAND" +echo "The end of the irparallel script" >> /home/irefindex/logfile2.log From 90dd59c2d5df887c77c8ea55981942c6f27affb1 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 29 May 2024 09:43:00 +0200 Subject: [PATCH 47/74] making changes to get the irparallel working together with the script --- usr/bin/irdata_parse_refseq.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/usr/bin/irdata_parse_refseq.sh b/usr/bin/irdata_parse_refseq.sh index 119b982f..b638eb75 100644 --- a/usr/bin/irdata_parse_refseq.sh +++ b/usr/bin/irdata_parse_refseq.sh @@ -16,6 +16,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -36,6 +39,9 @@ fi DATADIR=$1 +# Log the value of $1 +echo "main_script: \$1 is $1" > /home/irefindex/logfile.log + if [ ! "$DATADIR" ]; then echo "$PROGNAME: A data directory must be specified." 1>&2 exit 1 @@ -51,8 +57,9 @@ fi # Parse the data files. - "$SCRIPTS/argument-per-line" $FILENAMES \ -| "$SCRIPTS/irparallel" "\"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_refseq.py\" \"$DATADIR\" {}" +"$SCRIPTS/argument-per-line" $FILENAMES \ +| "$SCRIPTS/irparallel" "\"$TOOLS/irdata_parse_refseq.py\" \"$DATADIR\" {}" + # Concatenate the output data. echo "$DATADIR cat step" 1>&2 @@ -62,4 +69,4 @@ cat "$DATADIR"/*_identifiers > "$DATADIR/refseq_identifiers.txt" # Process the sequence data. "$TOOLS/irdata_process_signatures.sh" "$DATADIR" --append --append-length -exit $? +exit $? \ No newline at end of file From 54f31d328231f026abdd5c8ea7b788da35ad62b2 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 29 May 2024 09:43:38 +0200 Subject: [PATCH 48/74] making changes to get the irparallel working together with the script --- usr/bin/irdata_parse_genpept.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/usr/bin/irdata_parse_genpept.sh b/usr/bin/irdata_parse_genpept.sh index 836263dc..8e094875 100644 --- a/usr/bin/irdata_parse_genpept.sh +++ b/usr/bin/irdata_parse_genpept.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -44,7 +47,8 @@ if [ ! "$DATADIR" ] || [ ! "$FILENAMES" ]; then fi "$SCRIPTS/argument-per-line" $FILENAMES \ -| "$SCRIPTS/irparallel" "\"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fasta.py\" 'GENPEPT' \"$DATADIR\" 'acc,name,organism' 'acc,name,organism' {}" +| "$SCRIPTS/irparallel" "\"$TOOLS/irdata_parse_fasta.py\" 'GENPEPT' \"$DATADIR\" 'acc,name,organism' 'acc,name,organism' {}" + # Concatenate the output data # Tab character used in the sed command. From e6e9909d3cb29df6b9a8ec9691a8028f9e6d0e2e Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 29 May 2024 09:44:41 +0200 Subject: [PATCH 49/74] minor changes while getting the irparallel working --- vsc/ansible/irparse.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 9133ede7..37eeab10 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -11,6 +11,8 @@ vars: log_location: "{{ logs_dir }}/irparse" USE_PYTHON_INTERPRETER: "{{ python_interpreter | default('/usr/bin/python3') }}" + local_source_directory: "/home/guest/internship/stage-VIB-irefindex/usr/bin" # Directory containing the files on your local machine + remote_destination_directory: "/home/irefindex/usr/bin" # Destination directory on the remote machine tasks: #COMMENTED BECAUSE DOING THIS ONCE IS ENOUGH @@ -22,6 +24,17 @@ # recurse: yes # become: yes + #- name: Get list of files in local source directory + # shell: ls -1 {{ local_source_directory }} + # register: local_files_list + + #- name: Copy files to remote destination directory + # ansible.builtin.copy: + # src: "{{ local_source_directory }}{{ item }}" + # dest: "{{ remote_destination_directory }}/{{ item }}" + # mode: 'u+x,g+x,o+x' + # with_items: "{{ local_files_list.stdout_lines }}" + - name: Ensure python3-venv and python3-pip are installed ansible.builtin.apt: name: @@ -71,8 +84,9 @@ # mode: u+x,g+x,o+x # loop: "{{ files_to_update.files }}" - - name: Convert the downloaded data to files for each source + - name: Parse downloads for each source ansible.builtin.command: "./irparse {{ item.name | upper }}" + #ansible.builtin.command: "./echoscript {{ item.name | upper }}" args: chdir: "/home/irefindex/usr/bin" environment: From c75ca2215d27e314474d2a828f1c94f35b8e93a6 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 29 May 2024 09:45:36 +0200 Subject: [PATCH 50/74] adding minor changes to get the irparallel working --- usr/bin/irparse | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/bin/irparse b/usr/bin/irparse index baa03ff9..89acf82f 100755 --- a/usr/bin/irparse +++ b/usr/bin/irparse @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . + + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -75,6 +77,4 @@ for SOURCE in $SOURCES ; do done # The pipeline. - - "$SCRIPTS/argument-per-line" $PARSESOURCES \ -| "$SCRIPTS/irparallel" "\"$SCRIPTS/irparse-source\" $FIX $CHECK $NO_PARSE {}" +"$SCRIPTS/argument-per-line" $PARSESOURCES | "$SCRIPTS/irparallel" "\"$SCRIPTS/irparse-source\" $FIX $CHECK $NO_PARSE {}" From d143e16a959bd251b9dabf534955d2f63c262587 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 29 May 2024 09:46:17 +0200 Subject: [PATCH 51/74] making changes to gete irparallel working with the script --- usr/bin/irdata_parse_uniprot.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/usr/bin/irdata_parse_uniprot.sh b/usr/bin/irdata_parse_uniprot.sh index 298b1808..f73210b6 100644 --- a/usr/bin/irdata_parse_uniprot.sh +++ b/usr/bin/irdata_parse_uniprot.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -66,7 +69,7 @@ for FILENAME in $FILENAMES; do # Split the data file into $PROCESS pieces. # Pass each process its rank (0-based) and the total number of processes. seq 0 $((PROCESSES - 1)) \ - | "$SCRIPTS/irparallel" "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_altsplit.py $FILENAME {} $PROCESSES | $TOOLS/irdata_parse_uniprot.py $DATADIR - ${LEAFNAME}_%s-{}.txt" + | "$SCRIPTS/irparallel" "$TOOLS/irdata_altsplit.py $FILENAME {} $PROCESSES | ${USE_PYTHON_INTERPRETER} $TOOLS/irdata_parse_uniprot.py $DATADIR - ${LEAFNAME}_%s-{}.txt" # Merge the pieces. echo "$PROGNAME: Merging data files for $LEAFNAME..." 1>&2 From f6adfd1ef3db629a9ef6e32cae2d15b723033e6b Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 31 May 2024 14:14:07 +0200 Subject: [PATCH 52/74] drop --- usr/bin/irdata-config | 8 +++- usr/bin/irdata_parse_ipi.sh | 3 ++ usr/bin/irdata_parse_uniprot.sh | 15 ++++++- usr/bin/irparallel | 20 ++++++++-- vsc/ansible/irdownload.yml | 5 ++- vsc/ansible/vars/sources2.yml | 4 +- vsc/ansible/vars/sources3.yml | 69 ++++++++++++++++++++++++++++++++- 7 files changed, 114 insertions(+), 10 deletions(-) diff --git a/usr/bin/irdata-config b/usr/bin/irdata-config index 179751b2..03150f16 100644 --- a/usr/bin/irdata-config +++ b/usr/bin/irdata-config @@ -1,5 +1,11 @@ #!/bin/bash + + +############################################################ +# My own environments + + ############################################################ # Output and publication options. # (RELEASE is most likely to need changing.) @@ -18,7 +24,7 @@ EMAIL="alexander.botzki@vib.be" # e-mail address for anonymous FTP l DATABASE="irdata${RELEASE_MAJOR}" # database name DATA="${DATAPREFIX}dataext/irdata${RELEASE_MAJOR}" # user defined data directory location WIKIFORMAT="MediaWiki" # format for generated Wiki pages -CURL_OPT="-kL" # Follow HTTP redirects +CURL_OPT="-L" # Follow HTTP redirects WGET_WAIT="5" # time to wait between wget requests WGET_ATTEMPTS="5" # number of attempts to make using wget PSQL_OPTIONS="" # "--psql-options " diff --git a/usr/bin/irdata_parse_ipi.sh b/usr/bin/irdata_parse_ipi.sh index 80d08382..0ed9a0dc 100644 --- a/usr/bin/irdata_parse_ipi.sh +++ b/usr/bin/irdata_parse_ipi.sh @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +# My interpreter +USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} + if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_uniprot.sh b/usr/bin/irdata_parse_uniprot.sh index f73210b6..390ed1cb 100644 --- a/usr/bin/irdata_parse_uniprot.sh +++ b/usr/bin/irdata_parse_uniprot.sh @@ -68,8 +68,21 @@ for FILENAME in $FILENAMES; do # Split the data file into $PROCESS pieces. # Pass each process its rank (0-based) and the total number of processes. + echo "can I see this echo, which is situated in my uniprotscript" >> /home/irefindex/logfile2.log seq 0 $((PROCESSES - 1)) \ - | "$SCRIPTS/irparallel" "$TOOLS/irdata_altsplit.py $FILENAME {} $PROCESSES | ${USE_PYTHON_INTERPRETER} $TOOLS/irdata_parse_uniprot.py $DATADIR - ${LEAFNAME}_%s-{}.txt" + | "$SCRIPTS/irparallel" "$TOOLS/irdata_altsplit.py \"$FILENAME\" {} \"$PROCESSES\" | $TOOLS/irdata_parse_uniprot.py \"$DATADIR\" - \"${LEAFNAME}_%s-{}.txt\"" + + #for rank in $(seq 0 $((PROCESSES - 1))); do + # cmd="\"$TOOLS/irdata_altsplit.py\" \"$FILENAME\" \"$rank\" \"$PROCESSES\" | \"${USE_PYTHON_INTERPRETER}\" \"$TOOLS/irdata_parse_uniprot.py\" \"$DATADIR\" - \"${LEAFNAME}_%s-$rank.txt\"" + # echo "Executing: $cmd" # Debug: print the command being executed + # "$SCRIPTS/irparallel" "$cmd" + #done + + #for rank in $(seq 0 $((PROCESSES - 1))); do + # "$SCRIPTS/irparallel" "$TOOLS/irdata_altsplit.py" "$FILENAME" "$rank" "$PROCESSES" | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_uniprot.py" "$DATADIR" - "${LEAFNAME}_%s-${rank}.txt" + #done + + echo "uniprot.sh \echo: seq 0 $((PROCESSES - 1))" >> /home/irefindex/logfile2.log # Merge the pieces. echo "$PROGNAME: Merging data files for $LEAFNAME..." 1>&2 diff --git a/usr/bin/irparallel b/usr/bin/irparallel index 26a36069..261a84a2 100755 --- a/usr/bin/irparallel +++ b/usr/bin/irparallel @@ -77,7 +77,7 @@ echo "$EXENAME" >> /home/irefindex/logfile2.log #FIRST_PART=$(echo "$COMMAND" | awk '{print $1}' | tr -d '"') # Check if the first part ends in .py #if [[ "$FIRST_PART" == *.py ]]; then - # Prepend Python interpreter to the command + # Prepend Python interpreter to the commandx # COMMAND2="$USE_PYTHON_INTERPRETER $COMMAND" #fi #else @@ -87,7 +87,21 @@ echo "$EXENAME" >> /home/irefindex/logfile2.log echo "The value of command2 after checking if it ends at .py: : $COMMAND2 ; current time: $current_time" >> /home/irefindex/logfile2.log -# The pipeline. -xargs -P $PROCESSES $XARGS_I'{}' sh -c "$COMMAND2" +# Check if the command is for irdata_altsplit.py +#if [[ "$COMMAND" == *"irdata_altsplit.py"* ]]; then +# # Process each item differently +# while IFS= read -r item; do +# # Extract the filename, rank, and total from the item +# filename=$(echo "$item" | awk '{print $1}') +# rank=$(echo "$item" | awk '{print $2}') +# total=$(echo "$item" | awk '{print $3}') +# # Execute the command with the extracted arguments using xargs and sh -c +# echo "$filename $rank $total" | xargs -n 3 -P $PROCESSES sh -c "$COMMAND2" +# done +#else +# # Process items normally +# xargs -P $PROCESSES $XARGS_I'{}' sh -c "$COMMAND2" +#fi +xargs -P $PROCESSES $XARGS_I'{}' sh -c "$COMMAND2" echo "The end of the irparallel script" >> /home/irefindex/logfile2.log diff --git a/vsc/ansible/irdownload.yml b/vsc/ansible/irdownload.yml index 4145d7a3..d9cce9ff 100644 --- a/vsc/ansible/irdownload.yml +++ b/vsc/ansible/irdownload.yml @@ -5,12 +5,13 @@ gather_facts: true # WE NEED THIS HERE FOR THE DATES FOR THE LOGS! vars_files: - ./vars/logs.yml - - ./vars/sources2.yml + - ./vars/sources.yml #THIS SHOULD BE sources.yml vars: log_location: "{{ logs_dir }}/irdownload" tasks: +#ALL UNDERNEETH IS COMMENTED BECAUSE IT TAKES TO LONG, IT WORKS AND NEEDS TO BE UNCOMMENTED #- name: Update apt packages # ansible.builtin.apt: # update_cache: true @@ -61,7 +62,7 @@ retries: "{{ item.download_runner_retries | default(18) }}" - name: Wait for jobs to complete - debugger: always + # debugger: always ansible.builtin.async_status: jid: "{{ item.ansible_job_id }}" delay: "{{ item.item.download_delay | default(5) }}" diff --git a/vsc/ansible/vars/sources2.yml b/vsc/ansible/vars/sources2.yml index 220fe82e..e91d97bc 100644 --- a/vsc/ansible/vars/sources2.yml +++ b/vsc/ansible/vars/sources2.yml @@ -1,2 +1,4 @@ jobs: - - { name: "mint", download_runner_async: 600, download_retries: 1, download_delay: 30, parse_async: 120, import_async: 400, import_delay: 30, import_retries: 14 } \ No newline at end of file + # - { name: "mint", download_runner_async: 600, download_retries: 1, download_delay: 30, parse_async: 120, import_async: 400, import_delay: 30, import_retries: 14 } + # - { name: "bar" } + - { name: "gene", download_runner_async: 800, download_retries: 40, download_delay: 60, parse_async: 1600, parse_retries: 27, parse_delay: 60, import_async: 4000, import_retries: 34, import_delay: 120 } diff --git a/vsc/ansible/vars/sources3.yml b/vsc/ansible/vars/sources3.yml index 91db1364..dbc181c6 100644 --- a/vsc/ansible/vars/sources3.yml +++ b/vsc/ansible/vars/sources3.yml @@ -1,3 +1,68 @@ jobs: - - { name: "athaliana" } - - { name: "bind" } \ No newline at end of file + + # - { name: "ipi" } + # UNABLE TO DOWNLOAD SO UNABLE TO PARSE + # /data/irdata/logs/20240522T115852003408/irparse + + # - { name: "bar" } + # UNABLE TO DOWNLOAD SO UNABLE TO PARSE + + # - { name: "corum", download_runner_async: 60, download_retries: 12, download_delay: 15 } + # UNABLE TO DOWNLOAD SO UNABLE TO PARSE + # /data/irdata/logs/20240523T081112716280/irparse + + # - { name: "matrixdb", download_runner_async: 300, download_retries: 30, download_delay: 30, parse_async: 200, parse_retries: 8, parse_delay: 25, import_async: 160, import_delay: 10, import_retries: 16 } + # UNABLE TO DOWNLOAD SO UNABLE TO PARSE + #/data/irdata/logs/20240523T081306290855/irparse + + + + # - { name: "hprd", parse_async: 400, parse_delay: 30, parse_retries: 14, import_async: 300, import_delay: 30, import_retries: 10 } + # BIG ERROR FOR THIS ONE but looks like everything is in the files ?! + # " unassigned701 is not a pubmedID or DOI: citation suppressed" + # " unassigned1513 is not a pubmedID or DOI: citation suppressed" + # ALL THE FILES ARE NOT EMPTY + + # - { name: "bind_translation", parse_async: 120, parse_delay: 10 } + #ERROR FOR THIS ONE: irdata_xml2tab.py: Parsing failed with exception: /mnt/disks/data/dataext/irdata19/BIND_Translation/taxid10090_PSIMI25.xml:969431:746: not well-formed (invalid token) + #/data/irdata/logs/20240522T132332434617/irparse + + # - { name: "intcomplex", unpack_async: 60, unpack_delay: 10, parse_async: 350, parse_delay: 30, parse_retries: 12, import_async: 400, import_delay: 30, import_retries: 14 } + # irdata_xml2tab.py: Parsing failed with exception: 'utf-8' codec can't encode characters in position 50-52: surrogates not allowed + # irparse-source: XML processing for source INTCOMPLEX failed.non-zero return code + # /data/irdata/logs/20240523T074916877736/irparse/ + + # - { name: "huri", download_runner_async: 60, download_retries: 36, parse_async: 240, parse_retries: 16, parse_delay: 15, import_async: 400, import_delay: 30, import_retries: 14 } + #/data/irdata/logs/20240523T070911706214/irparse + # irparse-source: Data processing for source HURI failed.non-zero return + + + # - { name: "uniprot", download_runner_async: 2000, download_retries: 60, download_delay: 100, parse_async: 36000, parse_retries: 60, parse_delay: 600 } + # /data/irdata/logs/20240523T082021716989/irparse/uniprot.* + # irparallel: Please specify a command to run on each file (to be given as {}). + # irdata_parse_uniprot.sh: Merging data files for uniprot_sprot... + # cat: '/mnt/disks/data/dataext/irdata19/import/UniProt/uniprot_sprot_accessions-*.txt': such file or directory + # irdata_parse_uniprot.sh: Failed to concatenate data files for uniprot_sprot_accessions. + # irparse-source: Data processing for source UNIPROT failed.non-zero return code + + # - { name: "imex", download_runner_async: 10800, download_retries: 108, download_delay: 300, parse_async: 600, parse_retries: 10, parse_delay: 60, import_async: 1500, import_retries: 25, import_delay: 60 } + # " unassigned1513 is not a pubmedID or DOI: citation suppressed" + # " unassigned701 is not a pubmedID or DOI: citation suppressed" + # " unassigned2614 is not a pubmedID or DOI: citation suppressed", + # " unassigned2616 is not a pubmedID or DOI: citation suppressed", + # " unassigned2293 is not a pubmedID or DOI: citation suppressed", + # /root/.ansible_async/j832151890912.1559858 + + # - { name: "refseq", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 450, parse_retries: 5, parse_delay: 30, import_async: 5000, import_retries: 21, import_delay: 240 } + # /import/RefSeq is empty after parsing, BIG ERROR BUT NO LOG FILES + # "sh: 1: Syntax error: Unterminated quoted string" + # "cat: '/mnt/disks/data/dataext/irdata19/import/RefSeq/*_proteins': No such file or directory", + # "cat: '/mnt/disks/data/dataext/irdata19/import/RefSeq/*_identifiers': No such file or directory" + + - { name: "genpept", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 21000, parse_retries: 60, parse_delay: 350, import_async: 5000, import_retries: 21, import_delay: 240 } + # EMPTY AFTER PARSING + # "sh: 1: Syntax error: Unterminated quoted string", + # "cat: '/mnt/disks/data/dataext/irdata19/import/GenPept/*_proteins.txt': No such file or directory" + + + # - { name: "gene", download_runner_async: 800, download_retries: 40, download_delay: 60, parse_async: 1600, parse_retries: 27, parse_delay: 60, import_async: 4000, import_retries: 34, import_delay: 120 } \ No newline at end of file From 86040ee0a427dcdd6a4a00afe878d4e7042a2af9 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 31 May 2024 16:58:46 +0200 Subject: [PATCH 53/74] removing the interpreter --- usr/bin/argument-per-line | 1 + usr/bin/irbuild | 62 +++++++++++++------------- usr/bin/irbuild-output-resources | 8 ++-- usr/bin/irbuild2 | 42 ++++++++--------- usr/bin/irbuild3 | 8 ++-- usr/bin/irbuild4 | 56 +++++++++++------------ usr/bin/ircanonical | 8 ++-- usr/bin/irdata_archive_irefscape.sh | 2 +- usr/bin/irdata_fetch_refseq.sh | 4 +- usr/bin/irdata_index_irefscape_data.sh | 2 +- usr/bin/irdata_parse_A_thaliana.sh | 5 +-- usr/bin/irdata_parse_apid.sh | 2 +- usr/bin/irdata_parse_bar.sh | 2 +- usr/bin/irdata_parse_bhfucl.sh | 5 +-- usr/bin/irdata_parse_bind.sh | 11 ++--- usr/bin/irdata_parse_dig.sh | 5 +-- usr/bin/irdata_parse_dip.sh | 2 +- usr/bin/irdata_parse_fly.sh | 5 +-- usr/bin/irdata_parse_gene_info.sh | 4 +- usr/bin/irdata_parse_genpept.sh | 5 +-- usr/bin/irdata_parse_hpidb.sh | 6 +-- usr/bin/irdata_parse_huri.sh | 5 +-- usr/bin/irdata_parse_imex.sh | 5 +-- usr/bin/irdata_parse_innatedb.sh | 5 +-- usr/bin/irdata_parse_ipi.sh | 9 ++-- usr/bin/irdata_parse_matrixdb.sh | 2 +- usr/bin/irdata_parse_mbinfo.sh | 5 +-- usr/bin/irdata_parse_mint.sh | 5 +-- usr/bin/irdata_parse_molcon.sh | 2 +- usr/bin/irdata_parse_mpidb.sh | 5 +-- usr/bin/irdata_parse_pdb.sh | 5 +-- usr/bin/irdata_parse_quickgo.sh | 5 +-- usr/bin/irdata_parse_reactome.sh | 5 +-- usr/bin/irdata_parse_refseq.sh | 3 -- usr/bin/irdata_parse_spike.sh | 2 +- usr/bin/irdata_parse_uniprot.sh | 7 +-- usr/bin/irdata_parse_uniprotpp.sh | 5 +-- usr/bin/irdata_parse_virushost.sh | 6 +-- usr/bin/irdata_process_signatures.sh | 5 +-- usr/bin/irdrop | 36 +++++++-------- usr/bin/irefpro.sh | 2 +- usr/bin/irimport | 2 +- usr/bin/irimport-source | 4 +- usr/bin/irinit | 37 +++++++-------- usr/bin/irmanifest | 2 +- usr/bin/irmissing | 6 +-- usr/bin/iroutput | 8 ++-- usr/bin/irparallel | 55 +---------------------- usr/bin/irparse | 2 - usr/bin/irparse-source | 7 +-- usr/bin/irprevious | 8 ++-- usr/bin/irprevious-irefweb | 2 +- 52 files changed, 187 insertions(+), 315 deletions(-) mode change 100755 => 100644 usr/bin/irparse-source diff --git a/usr/bin/argument-per-line b/usr/bin/argument-per-line index bfc004f1..1aa85152 100755 --- a/usr/bin/argument-per-line +++ b/usr/bin/argument-per-line @@ -1,4 +1,5 @@ #!/bin/bash + for ARG in $*; do echo $ARG done diff --git a/usr/bin/irbuild b/usr/bin/irbuild index a376b5fc..9fbca55c 100644 --- a/usr/bin/irbuild +++ b/usr/bin/irbuild @@ -66,70 +66,70 @@ if [ "$BUILD" ]; then # Import any manifest information. - if [ -e "$DATA/irefindex_manifest" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_manifest.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/irefindex_manifest" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_manifest.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Manifest information could not be imported." 1>&2 exit 1 fi # Convert MITAB-based data to an XML-compatible representation. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/mitab_to_xml.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/mitab_to_xml.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not convert MITAB data to the common data representation." 1>&2 exit 1 fi # Finalise the XML-based data. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_constraints.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_constraints.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Constraints could not be applied to the common data representation." 1>&2 exit 1 fi # Import archived identifiers from a previous release. - if [ -e "$DATA/sequences_archived" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/sequences_archived" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import archived identifiers from the previous release." 1>&2 exit 1 fi # Build a sequence archive for the release. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_sequences.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build a sequence archive from the imported data." 1>&2 exit 1 fi # Identify the interactors. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactors.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactors.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interactor information from the imported data." 1>&2 exit 1 fi # Establish gene-to-protein mappings. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not create gene-to-protein mappings." 1>&2 exit 1 fi # Map identifiers to sequences in current databases. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences linkprefix '' ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences linkprefix '' ; then echo "$PROGNAME: Could not build sequence information from the imported data." 1>&2 exit 1 fi # Map identifiers to sequences in archived databases. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences_archived linkprefix 'archived/' ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences_archived linkprefix 'archived/' ; then echo "$PROGNAME: Could not build sequence information from archived data." 1>&2 exit 1 fi # Map interactors to sequences. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactor_sequences.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactor_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not map interactors to sequences." 1>&2 exit 1 fi @@ -146,14 +146,14 @@ if [ "$MAKE_REPORTS" ]; then # Make a tidied up manifest report. echo "$REPORTS $DATA $PSQL_OPTIONS" 1>&2 - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report manifest information." 1>&2 exit 1 fi # Make an interactor report. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report source interactor information." 1>&2 exit 1 fi @@ -172,7 +172,7 @@ if [ "$BUILD" ]; then # Assign definitive sequence information to interactors. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build assignment information from the imported data." 1>&2 exit 1 fi @@ -182,17 +182,17 @@ if [ "$MAKE_REPORTS" ]; then # Make assigned interactor reports, if requested. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report sequence assignment information." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier assignment information." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier scoring information." 1>&2 exit 1 fi @@ -204,34 +204,34 @@ if [ "$BUILD" ]; then # Export ROG information for RIG identifier calculation, make and import RIG identifiers. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export ROG identifiers for assigned sequences." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then + if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then echo "$PROGNAME: Could not generate RIG identifiers for the exported ROG identifiers." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated RIG identifiers." 1>&2 exit 1 fi # Make canonical interaction information. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export canonical ROG identifiers." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then + if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then echo "$PROGNAME: Could not generate canonical RIG identifiers for the exported canonical ROG identifiers." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated canonical RIG identifiers." 1>&2 exit 1 fi @@ -241,22 +241,22 @@ if [ "$MAKE_REPORTS" ]; then # Make an interactions report, if requested. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier assignment information." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier taxonomy statistics." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report interaction statistics." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then + if ! "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then echo "$PROGNAME: Could not generate statistics Wiki page." 1>&2 exit 1 fi @@ -268,29 +268,29 @@ if [ "$BUILD" ]; then # done in separate script # Make convenient representations for other entity types. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interaction information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate confidence score information." 1>&2 exit 1 fi # Import integer identifiers from a previous release. - if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import integer identifiers from the previous release." 1>&2 exit 1 fi diff --git a/usr/bin/irbuild-output-resources b/usr/bin/irbuild-output-resources index 5597497d..6b8b64d8 100644 --- a/usr/bin/irbuild-output-resources +++ b/usr/bin/irbuild-output-resources @@ -66,22 +66,22 @@ if [ "$BUILD" ]; then # Make convenient representations for other entity types. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interaction information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate confidence score information." 1>&2 exit 1 fi diff --git a/usr/bin/irbuild2 b/usr/bin/irbuild2 index 0377f8c3..3df46762 100644 --- a/usr/bin/irbuild2 +++ b/usr/bin/irbuild2 @@ -60,14 +60,14 @@ if [ "$MAKE_REPORTS" ]; then # Make a tidied up manifest report. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report manifest information." 1>&2 exit 1 fi # Make an interactor report. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report source interactor information." 1>&2 exit 1 fi @@ -86,7 +86,7 @@ if [ "$BUILD" ]; then # Assign definitive sequence information to interactors. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build assignment information from the imported data." 1>&2 exit 1 fi @@ -96,17 +96,17 @@ if [ "$MAKE_REPORTS" ]; then # Make assigned interactor reports, if requested. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report sequence assignment information." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier assignment information." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier scoring information." 1>&2 exit 1 fi @@ -118,34 +118,34 @@ if [ "$BUILD" ]; then # Export ROG information for RIG identifier calculation, make and import RIG identifiers. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export ROG identifiers for assigned sequences." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then + if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then echo "$PROGNAME: Could not generate RIG identifiers for the exported ROG identifiers." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated RIG identifiers." 1>&2 exit 1 fi # Make canonical interaction information. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export canonical ROG identifiers." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then + if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then echo "$PROGNAME: Could not generate canonical RIG identifiers for the exported canonical ROG identifiers." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated canonical RIG identifiers." 1>&2 exit 1 fi @@ -155,22 +155,22 @@ if [ "$MAKE_REPORTS" ]; then # Make an interactions report, if requested. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier assignment information." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier taxonomy statistics." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report interaction statistics." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then + if ! "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then echo "$PROGNAME: Could not generate statistics Wiki page." 1>&2 exit 1 fi @@ -182,29 +182,29 @@ if [ "$BUILD" ]; then # Make convenient representations for other entity types. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interaction information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate confidence score information." 1>&2 exit 1 fi # Import integer identifiers from a previous release. - if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import integer identifiers from the previous release." 1>&2 exit 1 fi diff --git a/usr/bin/irbuild3 b/usr/bin/irbuild3 index 44e00f81..f7aa8d29 100644 --- a/usr/bin/irbuild3 +++ b/usr/bin/irbuild3 @@ -59,24 +59,24 @@ if [ "$BUILD" ]; then # Make convenient representations for other entity types. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate confidence score information." 1>&2 exit 1 fi # Import integer identifiers from a previous release. - if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import integer identifiers from the previous release." 1>&2 exit 1 fi diff --git a/usr/bin/irbuild4 b/usr/bin/irbuild4 index 52f111c5..4c590605 100644 --- a/usr/bin/irbuild4 +++ b/usr/bin/irbuild4 @@ -66,49 +66,49 @@ if [ "$BUILD" ]; then # Import archived identifiers from a previous release. - if [ -e "$DATA/sequences_archived" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/sequences_archived" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import archived identifiers from the previous release." 1>&2 exit 1 fi # Build a sequence archive for the release. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_sequences.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build a sequence archive from the imported data." 1>&2 exit 1 fi # Identify the interactors. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactors.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactors.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build interactor information from the imported data." 1>&2 exit 1 fi # Establish gene-to-protein mappings. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not create gene-to-protein mappings." 1>&2 exit 1 fi # Map identifiers to sequences in current databases. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences linkprefix '' ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences linkprefix '' ; then echo "$PROGNAME: Could not build sequence information from the imported data." 1>&2 exit 1 fi # Map identifiers to sequences in archived databases. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences_archived linkprefix 'archived/' ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_identifier_sequences.sql" $PSQL_OPTIONS --defs sequences irefindex_sequences_archived linkprefix 'archived/' ; then echo "$PROGNAME: Could not build sequence information from archived data." 1>&2 exit 1 fi # Map interactors to sequences. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactor_sequences.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactor_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not map interactors to sequences." 1>&2 exit 1 fi @@ -125,14 +125,14 @@ if [ "$MAKE_REPORTS" ]; then # Make a tidied up manifest report. echo "$REPORTS $DATA $PSQL_OPTIONS" 1>&2 - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/manifest.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report manifest information." 1>&2 exit 1 fi # Make an interactor report. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactors_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report source interactor information." 1>&2 exit 1 fi @@ -151,7 +151,7 @@ if [ "$BUILD" ]; then # Assign definitive sequence information to interactors. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not build assignment information from the imported data." 1>&2 exit 1 fi @@ -161,17 +161,17 @@ if [ "$MAKE_REPORTS" ]; then # Make assigned interactor reports, if requested. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/assignments_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report sequence assignment information." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier assignment information." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rogids_by_score.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report ROG identifier scoring information." 1>&2 exit 1 fi @@ -183,34 +183,34 @@ if [ "$BUILD" ]; then # Export ROG information for RIG identifier calculation, make and import RIG identifiers. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export ROG identifiers for assigned sequences." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then + if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions" "$DATA/rigids_for_interactions" ; then echo "$PROGNAME: Could not generate RIG identifiers for the exported ROG identifiers." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated RIG identifiers." 1>&2 exit 1 fi # Make canonical interaction information. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_rogids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export canonical ROG identifiers." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then + if ! "$TOOLS/irdata_process_signatures.py" "$DATA/rogids_for_interactions_canonical" "$DATA/rigids_for_interactions_canonical" ; then echo "$PROGNAME: Could not generate canonical RIG identifiers for the exported canonical ROG identifiers." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_rigids_canonical.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import the generated canonical RIG identifiers." 1>&2 exit 1 fi @@ -220,22 +220,22 @@ if [ "$MAKE_REPORTS" ]; then # Make an interactions report, if requested. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier assignment information." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/rigids_by_taxonomy.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report RIG identifier taxonomy statistics." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$REPORTS/interactions_by_source.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not report interaction statistics." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then + if ! "$TOOLS/irdata_reports2wiki.py" "$DATA" "$DATA/Statistics.wiki" "$WIKIFORMAT" ; then echo "$PROGNAME: Could not generate statistics Wiki page." 1>&2 exit 1 fi @@ -247,29 +247,29 @@ if [ "$BUILD" ]; then # done in separate script # Make convenient representations for other entity types. - # if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then + # if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_interactions.sql" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not build interaction information from the imported data." 1>&2 # exit 1 # fi - # if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then + # if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_experiments.sql" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not build experiment information from the imported data." 1>&2 # exit 1 # fi - # if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then + # if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_participants.sql" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not build participant information from the imported data." 1>&2 # exit 1 # fi - # if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then + # if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_confidence.sql" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not generate confidence score information." 1>&2 # exit 1 # fi # Import integer identifiers from a previous release. - if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then + if [ -e "$DATA/rig2rigid" ] && [ -e "$DATA/rog2rogid" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_integer_identifiers.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import integer identifiers from the previous release." 1>&2 exit 1 fi diff --git a/usr/bin/ircanonical b/usr/bin/ircanonical index 144f5755..354d1290 100644 --- a/usr/bin/ircanonical +++ b/usr/bin/ircanonical @@ -33,14 +33,14 @@ EOF exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the canonical schema." 1>&2 exit 1 fi # Combine groups of genes until no more updates can be made. -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_iteration.sql" "$DATA" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_iteration.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not update the canonical schema." 1>&2 exit 1 fi @@ -48,14 +48,14 @@ fi while [ `cat "$DATA/canonical_updates"` != '0' ]; do echo "$PROGNAME: Updating the canonical schema..." 1>&2 - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_iteration.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_iteration.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not update the canonical schema." 1>&2 exit 1 fi done -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_final.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_irefindex_canonical_final.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not define the canonical representative interactors." 1>&2 exit 1 fi diff --git a/usr/bin/irdata_archive_irefscape.sh b/usr/bin/irdata_archive_irefscape.sh index 317b9cc0..781f1e4e 100644 --- a/usr/bin/irdata_archive_irefscape.sh +++ b/usr/bin/irdata_archive_irefscape.sh @@ -62,7 +62,7 @@ done # Convert the graph to the Java serialisation format. -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_convert_graph.py" "$DATADIR/graph.txt" "$DATADIR/graph" +"$TOOLS/irdata_convert_graph.py" "$DATADIR/graph.txt" "$DATADIR/graph" diff --git a/usr/bin/irdata_fetch_refseq.sh b/usr/bin/irdata_fetch_refseq.sh index c955b1d6..7a76830d 100644 --- a/usr/bin/irdata_fetch_refseq.sh +++ b/usr/bin/irdata_fetch_refseq.sh @@ -71,7 +71,7 @@ for ATTEMPT in `seq 1 $WGET_ATTEMPTS` ; do # Split the input file into manageable pieces and process each one in turn. # Note that this is done serially due to Entrez usage restrictions. - "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_split.py" -1 10000 "$WORKFILE" \ + "$TOOLS/irdata_split.py" -1 10000 "$WORKFILE" \ | xargs $XARGS_I'{}' sh -c "echo {} | \"$SCRIPTS/irslice\" \"$WORKFILE\" - | \"$TOOLS/irdata_fetch_eutils.sh\" \"$FILENAME\"" \ >> "$RESULTFILE" @@ -87,7 +87,7 @@ done # Parse the feature table output, producing files similar to those normally # available for RefSeq. -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_refseq.py" "$DATADIR" "$RESULTFILE" ; then +if ! "$TOOLS/irdata_parse_refseq.py" "$DATADIR" "$RESULTFILE" ; then echo "$PROGNAME: Could not parse the retrieved data." 1>&2 exit 1 fi diff --git a/usr/bin/irdata_index_irefscape_data.sh b/usr/bin/irdata_index_irefscape_data.sh index 592d6181..3a05c718 100644 --- a/usr/bin/irdata_index_irefscape_data.sh +++ b/usr/bin/irdata_index_irefscape_data.sh @@ -48,5 +48,5 @@ fi INTERVAL=100 - "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_index.py" $INTERVAL "$FILENAME" -d '|' \ + "$TOOLS/irdata_text_index.py" $INTERVAL "$FILENAME" -d '|' \ > "$OUTFILE" diff --git a/usr/bin/irdata_parse_A_thaliana.sh b/usr/bin/irdata_parse_A_thaliana.sh index 50e1a751..49fca643 100644 --- a/usr/bin/irdata_parse_A_thaliana.sh +++ b/usr/bin/irdata_parse_A_thaliana.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -60,5 +57,5 @@ echo "$FILETYPE" # cat the file and present it to the parser. echo "$DATADIR/$OUTFILE" 1>&2 zcat "$FILENAME" \ -| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_A_thaliana2.py" "$FILETYPE" --discard-ill-formed \ +| "$TOOLS/irdata_parse_A_thaliana2.py" "$FILETYPE" --discard-ill-formed \ > "$DATADIR/$OUTFILE" diff --git a/usr/bin/irdata_parse_apid.sh b/usr/bin/irdata_parse_apid.sh index 98a65c7b..d637bf22 100644 --- a/usr/bin/irdata_parse_apid.sh +++ b/usr/bin/irdata_parse_apid.sh @@ -43,4 +43,4 @@ if [ ! "$DATADIR" ] || [ ! "$FILENAMES" ]; then fi -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'APID' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'APID' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_bar.sh b/usr/bin/irdata_parse_bar.sh index 099b86b7..5515b67a 100644 --- a/usr/bin/irdata_parse_bar.sh +++ b/usr/bin/irdata_parse_bar.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'BAR' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'BAR' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_bhfucl.sh b/usr/bin/irdata_parse_bhfucl.sh index fbb93013..67cd8d66 100644 --- a/usr/bin/irdata_parse_bhfucl.sh +++ b/usr/bin/irdata_parse_bhfucl.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -60,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'BHF_UCL' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'BHF_UCL' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_bind.sh b/usr/bin/irdata_parse_bind.sh index be4349f7..c8e9f263 100644 --- a/usr/bin/irdata_parse_bind.sh +++ b/usr/bin/irdata_parse_bind.sh @@ -22,9 +22,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -94,7 +91,7 @@ for FILENAME in $FILENAMES; do # Pad the file in order to add missing methods. - "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_pad.py" -n 4 -p \\N "$FILENAME" \ + "$TOOLS/irdata_text_pad.py" -n 4 -p \\N "$FILENAME" \ > "$DATADIR/references.txt" elif [ "$FILETYPE" = 'complex2refs.txt' ]; then @@ -110,7 +107,7 @@ for FILENAME in $FILENAMES; do sed -n -e 'p;=' "$FILENAME" \ | sed -e 'N;s/\n/\t/' \ - | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 8 -t 8 -w '|' -s 0 - \ + | "$TOOLS/irdata_text_transpose.py" -f 8 -t 8 -w '|' -s 0 - \ | sed -e "s/^/$SED_FILENAME\t/" \ > "$DATADIR/complexes.txt" @@ -119,14 +116,14 @@ for FILENAME in $FILENAMES; do # First interactor (add position 0). cut -f 1,2,4 "$FILENAME" \ - | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 3 -w '|' -s 0 - \ + | "$TOOLS/irdata_text_transpose.py" -f 3 -w '|' -s 0 - \ | sed -e "s/^/0\t/" \ > "$DATADIR/labels.txt" # Second interactor (add position 1). cut -f 1,3,5 "$FILENAME" \ - | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 3 -w '|' -s 0 - \ + | "$TOOLS/irdata_text_transpose.py" -f 3 -w '|' -s 0 - \ | sed -e "s/^/1\t/" \ >> "$DATADIR/labels.txt" diff --git a/usr/bin/irdata_parse_dig.sh b/usr/bin/irdata_parse_dig.sh index 1ca3c021..99d96251 100644 --- a/usr/bin/irdata_parse_dig.sh +++ b/usr/bin/irdata_parse_dig.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -54,6 +51,6 @@ fi tail -n +2 "$FILENAME" \ | cut -f 2,7 \ -| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 1 -t 1 -w ', ' - \ +| "$TOOLS/irdata_text_transpose.py" -f 1 -t 1 -w ', ' - \ | sort -u \ > "$DATADIR/dig_genes.txt" diff --git a/usr/bin/irdata_parse_dip.sh b/usr/bin/irdata_parse_dip.sh index 10df6258..c89a84f0 100644 --- a/usr/bin/irdata_parse_dip.sh +++ b/usr/bin/irdata_parse_dip.sh @@ -57,4 +57,4 @@ fi # mv tmp $THISFILE #done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'DIP' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'DIP' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_fly.sh b/usr/bin/irdata_parse_fly.sh index b105983d..f2749dcd 100644 --- a/usr/bin/irdata_parse_fly.sh +++ b/usr/bin/irdata_parse_fly.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -60,5 +57,5 @@ END=`grep -ne '^----' "$FILENAME" | tail -n 2 | head -n 1 | cut -d ':' -f 1` head -n $((END - 1)) "$FILENAME" \ | tail -n "+$START" \ -| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fly.py" "$FILETYPE" --discard-ill-formed \ +| "$TOOLS/irdata_parse_fly.py" "$FILETYPE" --discard-ill-formed \ > "$DATADIR/$OUTFILE" diff --git a/usr/bin/irdata_parse_gene_info.sh b/usr/bin/irdata_parse_gene_info.sh index 84f780de..80bb3938 100644 --- a/usr/bin/irdata_parse_gene_info.sh +++ b/usr/bin/irdata_parse_gene_info.sh @@ -76,7 +76,7 @@ fi | tail -n +2 \ | cut -f 2,8 \ | grep -v -e 'NEWENTRY' \ -| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 2 -w '|' -s 0 - \ +| "$TOOLS/irdata_text_transpose.py" -f 2 -w '|' -s 0 - \ > "$DATADIR/$OUTFILE_MAPLOCATIONS" # For synonyms, remove the header, extract the geneid and synonyms. @@ -87,5 +87,5 @@ fi | tail -n +2 \ | cut -f 2,5 \ | grep -v -e 'NEWENTRY' \ -| "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 2 -w '|' -s 0 - \ +| "$TOOLS/irdata_text_transpose.py" -f 2 -w '|' -s 0 - \ > "$DATADIR/$OUTFILE_SYNONYMS" diff --git a/usr/bin/irdata_parse_genpept.sh b/usr/bin/irdata_parse_genpept.sh index 8e094875..c126c522 100644 --- a/usr/bin/irdata_parse_genpept.sh +++ b/usr/bin/irdata_parse_genpept.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -50,7 +47,7 @@ fi | "$SCRIPTS/irparallel" "\"$TOOLS/irdata_parse_fasta.py\" 'GENPEPT' \"$DATADIR\" 'acc,name,organism' 'acc,name,organism' {}" -# Concatenate the output data +# Concatenate the output data² # Tab character used in the sed command. #TAB=`printf '\t'` diff --git a/usr/bin/irdata_parse_hpidb.sh b/usr/bin/irdata_parse_hpidb.sh index b4fec9d8..281786a6 100644 --- a/usr/bin/irdata_parse_hpidb.sh +++ b/usr/bin/irdata_parse_hpidb.sh @@ -15,10 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . - -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -61,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'HPIDB' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'HPIDB' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_huri.sh b/usr/bin/irdata_parse_huri.sh index a03659eb..e94718a6 100644 --- a/usr/bin/irdata_parse_huri.sh +++ b/usr/bin/irdata_parse_huri.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -61,4 +58,4 @@ for THISFILE in $FILENAMES; do done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'HURI' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'HURI' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_imex.sh b/usr/bin/irdata_parse_imex.sh index 5447dd5d..4c1ffdd7 100644 --- a/usr/bin/irdata_parse_imex.sh +++ b/usr/bin/irdata_parse_imex.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -60,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'IMEX' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'IMEX' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_innatedb.sh b/usr/bin/irdata_parse_innatedb.sh index 567537b8..961dbf79 100644 --- a/usr/bin/irdata_parse_innatedb.sh +++ b/usr/bin/irdata_parse_innatedb.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -55,4 +52,4 @@ if [ $? != 0 ]; then exit 1 fi -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'INNATEDB' "$DATADIR" $DST +"$TOOLS/irdata_parse_mitab.py" 'INNATEDB' "$DATADIR" $DST diff --git a/usr/bin/irdata_parse_ipi.sh b/usr/bin/irdata_parse_ipi.sh index 0ed9a0dc..ad386383 100644 --- a/usr/bin/irdata_parse_ipi.sh +++ b/usr/bin/irdata_parse_ipi.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -50,7 +47,7 @@ for FILENAME in $FILENAMES; do BASENAME=`basename "$FILENAME"` FILETYPE=${BASENAME##*.} if [ "$FILETYPE" = 'fasta' ] || [ "$FILETYPE" = 'gz' ]; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fasta.py" 'IPI' "$DATADIR" 'acc' 'acc' "$FILENAME" ; then + if ! "$TOOLS/irdata_parse_fasta.py" 'IPI' "$DATADIR" 'acc' 'acc' "$FILENAME" ; then echo "$PROGNAME: FASTA parsing of $FILENAME failed." 1>&2 exit 1 fi @@ -77,10 +74,10 @@ for FILENAME in $FILENAMES; do | cut -d '>' -f 2- \ | cut -d ' ' -f 1,2 \ | sed -e 's/ /|/g' \ - | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 2 -d '|' - \ + | "$TOOLS/irdata_text_transpose.py" -f 2 -d '|' - \ | sed -e 's/[:=]/\t/g' \ | cut -f 2- \ - | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_text_transpose.py" -f 3 -w ';' - \ + | "$TOOLS/irdata_text_transpose.py" -f 3 -w ';' - \ > "$DATADIR/${BASENAME}_identifiers.txt" else diff --git a/usr/bin/irdata_parse_matrixdb.sh b/usr/bin/irdata_parse_matrixdb.sh index 7f96f7d3..ac39173c 100644 --- a/usr/bin/irdata_parse_matrixdb.sh +++ b/usr/bin/irdata_parse_matrixdb.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'MATRIXDB' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'MATRIXDB' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_mbinfo.sh b/usr/bin/irdata_parse_mbinfo.sh index a73da424..a8bef66b 100644 --- a/usr/bin/irdata_parse_mbinfo.sh +++ b/usr/bin/irdata_parse_mbinfo.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -60,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'MBINFO' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'MBINFO' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_mint.sh b/usr/bin/irdata_parse_mint.sh index 24d1de57..b04e5cec 100644 --- a/usr/bin/irdata_parse_mint.sh +++ b/usr/bin/irdata_parse_mint.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -46,4 +43,4 @@ if [ ! "$DATADIR" ] || [ ! "$FILENAMES" ]; then fi -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mpidb.py" 'MINT' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mpidb.py" 'MINT' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_molcon.sh b/usr/bin/irdata_parse_molcon.sh index df399989..b8b272d8 100644 --- a/usr/bin/irdata_parse_molcon.sh +++ b/usr/bin/irdata_parse_molcon.sh @@ -57,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'MOLCON' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'MOLCON' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_mpidb.sh b/usr/bin/irdata_parse_mpidb.sh index a0c26bae..77dbd44c 100644 --- a/usr/bin/irdata_parse_mpidb.sh +++ b/usr/bin/irdata_parse_mpidb.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -60,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'MPIDB' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'MPIDB' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_pdb.sh b/usr/bin/irdata_parse_pdb.sh index 9c48bdde..680943e3 100644 --- a/usr/bin/irdata_parse_pdb.sh +++ b/usr/bin/irdata_parse_pdb.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -50,7 +47,7 @@ for FILENAME in $FILENAMES; do BASENAME=`basename "$FILENAME"` FILETYPE=${BASENAME#*.} if [ "$FILETYPE" = 'fasta' ] || [ "$FILETYPE" = 'gz' ]; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fasta.py" 'PDB' "$DATADIR" 'acc,chain' 'acc,chain' "$FILENAME" ; then + if ! "$TOOLS/irdata_parse_fasta.py" 'PDB' "$DATADIR" 'acc,chain' 'acc,chain' "$FILENAME" ; then echo "$PROGNAME: FASTA parsing of $FILENAME failed." 1>&2 exit 1 fi diff --git a/usr/bin/irdata_parse_quickgo.sh b/usr/bin/irdata_parse_quickgo.sh index 8c4ce437..43fef988 100644 --- a/usr/bin/irdata_parse_quickgo.sh +++ b/usr/bin/irdata_parse_quickgo.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -60,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'QUICKGO' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'QUICKGO' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_reactome.sh b/usr/bin/irdata_parse_reactome.sh index 0bfe20bd..51327cb3 100644 --- a/usr/bin/irdata_parse_reactome.sh +++ b/usr/bin/irdata_parse_reactome.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -64,4 +61,4 @@ fi # mv tmp $THISFILE #done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'REACTOME' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'REACTOME' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_refseq.sh b/usr/bin/irdata_parse_refseq.sh index b638eb75..34c063a7 100644 --- a/usr/bin/irdata_parse_refseq.sh +++ b/usr/bin/irdata_parse_refseq.sh @@ -16,9 +16,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irdata_parse_spike.sh b/usr/bin/irdata_parse_spike.sh index 44847712..e2bb20e8 100644 --- a/usr/bin/irdata_parse_spike.sh +++ b/usr/bin/irdata_parse_spike.sh @@ -62,4 +62,4 @@ fi # mv tmp $THISFILE #done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'SPIKE' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'SPIKE' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_uniprot.sh b/usr/bin/irdata_parse_uniprot.sh index 390ed1cb..58bc4325 100644 --- a/usr/bin/irdata_parse_uniprot.sh +++ b/usr/bin/irdata_parse_uniprot.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -53,7 +50,7 @@ for FILENAME in $FILENAMES; do # Parse the FASTA files. if [ "$FILETYPE" = 'fasta' ] || [ "$FILETYPE" = 'fasta.gz' ]; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_fasta.py" 'UNIPROT' "$DATADIR" 'sp,acc,id' 'id,acc,date,taxid,mw' "$FILENAME" ; then + if ! "$TOOLS/irdata_parse_fasta.py" 'UNIPROT' "$DATADIR" 'sp,acc,id' 'id,acc,date,taxid,mw' "$FILENAME" ; then echo "$PROGNAME: FASTA parsing of $FILENAME failed." 1>&2 exit 1 fi @@ -79,7 +76,7 @@ for FILENAME in $FILENAMES; do #done #for rank in $(seq 0 $((PROCESSES - 1))); do - # "$SCRIPTS/irparallel" "$TOOLS/irdata_altsplit.py" "$FILENAME" "$rank" "$PROCESSES" | "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_uniprot.py" "$DATADIR" - "${LEAFNAME}_%s-${rank}.txt" + # "$SCRIPTS/irparallel" "$TOOLS/irdata_altsplit.py" "$FILENAME" "$rank" "$PROCESSES" | "$TOOLS/irdata_parse_uniprot.py" "$DATADIR" - "${LEAFNAME}_%s-${rank}.txt" #done echo "uniprot.sh \echo: seq 0 $((PROCESSES - 1))" >> /home/irefindex/logfile2.log diff --git a/usr/bin/irdata_parse_uniprotpp.sh b/usr/bin/irdata_parse_uniprotpp.sh index 38d7a094..47cb264a 100644 --- a/usr/bin/irdata_parse_uniprotpp.sh +++ b/usr/bin/irdata_parse_uniprotpp.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -60,4 +57,4 @@ for THISFILE in $FILENAMES; do mv tmp $THISFILE done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'UNIPROTPP' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'UNIPROTPP' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_parse_virushost.sh b/usr/bin/irdata_parse_virushost.sh index e92f5b6e..5f5dcda9 100644 --- a/usr/bin/irdata_parse_virushost.sh +++ b/usr/bin/irdata_parse_virushost.sh @@ -13,10 +13,6 @@ # PARTICULAR PURPOSE. See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program. If not, see . - -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} if [ -e "irdata-config" ]; then . "$PWD/irdata-config" @@ -65,4 +61,4 @@ fi # mv tmp $THISFILE #done -"${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_parse_mitab.py" 'VIRUSHOST' "$DATADIR" $FILENAMES +"$TOOLS/irdata_parse_mitab.py" 'VIRUSHOST' "$DATADIR" $FILENAMES diff --git a/usr/bin/irdata_process_signatures.sh b/usr/bin/irdata_process_signatures.sh index 490bff83..cccc27d1 100644 --- a/usr/bin/irdata_process_signatures.sh +++ b/usr/bin/irdata_process_signatures.sh @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -51,7 +48,7 @@ for FILENAME in "$DATADIR/"*_proteins.txt ; do exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_process_signatures.py" "$FILENAME" "$FILENAME.seq" $* ; then + if ! "$TOOLS/irdata_process_signatures.py" "$FILENAME" "$FILENAME.seq" $* ; then echo "$PROGNAME: Sequence digest processing of $FILENAME failed." 1>&2 exit 1 fi diff --git a/usr/bin/irdrop b/usr/bin/irdrop index a5238cb0..ff1166bf 100644 --- a/usr/bin/irdrop +++ b/usr/bin/irdrop @@ -53,9 +53,9 @@ fi if [ "$DROPALL" ]; then - if ! ( "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop.sql" $PSQL_OPTIONS \ - && "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_psixml.sql" $PSQL_OPTIONS \ - && "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_mitab.sql" $PSQL_OPTIONS \ + if ! ( "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop.sql" $PSQL_OPTIONS \ + && "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_psixml.sql" $PSQL_OPTIONS \ + && "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_mitab.sql" $PSQL_OPTIONS \ ) ; then echo "$PROGNAME: Could not remove the database schema." 1>&2 exit 1 @@ -67,7 +67,7 @@ if [ "$DROPALL" ]; then eval SCHEMA='$'"$SOURCE"_SCHEMA if [ "$SCHEMA" ]; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_${SCHEMA}.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_${SCHEMA}.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove schema for $SOURCE." 1>&2 [ ! "$FORCE" ] && exit 1 fi @@ -78,72 +78,72 @@ fi # Remove the different build tables. # Continue without exiting if the force option is given. -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_manifest.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_manifest.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the manifest schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if [ ! "$DROPALL" ] && ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_constraints.sql" $PSQL_OPTIONS ; then +if [ ! "$DROPALL" ] && ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_constraints.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the common representation's constraints." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_sequences.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the consolidated sequence information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_archived.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_archived.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the archived sequence information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_interactors.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_interactors.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the interactor information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the gene-to-protein mappings schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_canonical.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_canonical.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the canonical interactor information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_assignments.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the assignment information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_rigids.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_rigids.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the interaction information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_integer_identifiers.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_integer_identifiers.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the integer identifier mappings schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_interactions.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the additional interaction information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_experiments.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the additional experiment information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_participants.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the additional participant information schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_confidence.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/drop_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not remove the confidence score schema." 1>&2 [ ! "$FORCE" ] && exit 1 fi diff --git a/usr/bin/irefpro.sh b/usr/bin/irefpro.sh index 170421a3..2f8116c1 100644 --- a/usr/bin/irefpro.sh +++ b/usr/bin/irefpro.sh @@ -31,7 +31,7 @@ echo $SQL echo $PSQL_OPTIONS echo `whoami` -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefpro.sql" "$DATA" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefpro.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate iRefPro data." 1>&2 exit 1 fi diff --git a/usr/bin/irimport b/usr/bin/irimport index 2ba19982..144bc68b 100644 --- a/usr/bin/irimport +++ b/usr/bin/irimport @@ -72,7 +72,7 @@ elif [ ! "$SOURCES" ]; then fi if [ "$CHECK" ]; then - "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_sources.sql" "$DATA" $PSQL_OPTIONS + "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_sources.sql" "$DATA" $PSQL_OPTIONS IMPORTEDSOURCES=`cat "$DATA/imported_sources"` fi diff --git a/usr/bin/irimport-source b/usr/bin/irimport-source index 12cfc3eb..93e7a0ba 100644 --- a/usr/bin/irimport-source +++ b/usr/bin/irimport-source @@ -72,7 +72,7 @@ fi # Import XML sources using a common template. if [ "$XMLFILEPATTERN" ]; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_psixml.sql" "$IMPORTDATADIR" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_psixml.sql" "$IMPORTDATADIR" $PSQL_OPTIONS ; then echo "$PROGNAME: Import failed for XML files from $SOURCE." 1>&2 exit 1 fi @@ -86,7 +86,7 @@ else eval DEFS='$'"$SOURCE"_DEFS [[ -n ${DEFS} ]] && DEFS="--defs ${DEFS}" - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/$TEMPLATE" "$IMPORTDATADIR" $PSQL_OPTIONS $DEFS; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/$TEMPLATE" "$IMPORTDATADIR" $PSQL_OPTIONS $DEFS; then echo "$PROGNAME: Import failed for data files from $SOURCE." 1>&2 exit 1 fi diff --git a/usr/bin/irinit b/usr/bin/irinit index 7ff65425..147d93c0 100644 --- a/usr/bin/irinit +++ b/usr/bin/irinit @@ -15,9 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -47,9 +44,9 @@ fi if [ ! "$INITBUILD" ]; then - if ! ( "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init.sql" $PSQL_OPTIONS \ - && "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_psixml.sql" $PSQL_OPTIONS \ - && "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_mitab.sql" $PSQL_OPTIONS \ + if ! ( "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init.sql" $PSQL_OPTIONS \ + && "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_psixml.sql" $PSQL_OPTIONS \ + && "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_mitab.sql" $PSQL_OPTIONS \ ) ; then echo "$PROGNAME: Could not initialise the database schema." 1>&2 exit 1 @@ -61,7 +58,7 @@ if [ ! "$INITBUILD" ]; then eval SCHEMA='$'"$SOURCE"_SCHEMA if [ "$SCHEMA" ]; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_${SCHEMA}.sql" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_${SCHEMA}.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise schema for $SOURCE." 1>&2 exit 1 fi @@ -71,67 +68,67 @@ fi # Initialise the different build tables. -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_manifest.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_manifest.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the manifest schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_sequences.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_sequences.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the consolidated sequence information schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_archived.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_archived.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the archived sequence information schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_interactors.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_interactors.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the interactor information schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_gene_mappings.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the gene-to-protein mappings schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_canonical.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_canonical.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the canonical interactor information schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_assignments.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_assignments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the assignment information schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_rigids.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_rigids.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the interaction information schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_integer_identifiers.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_integer_identifiers.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the integer identifier mappings schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_interactions.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_interactions.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the additional interaction information schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_experiments.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_experiments.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the additional experiment information schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_participants.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_participants.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the additional participant information schema." 1>&2 exit 1 fi -if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_confidence.sql" $PSQL_OPTIONS ; then +if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/init_irefindex_confidence.sql" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not initialise the confidence score schema." 1>&2 exit 1 fi diff --git a/usr/bin/irmanifest b/usr/bin/irmanifest index 0a5e3f06..d7636d01 100644 --- a/usr/bin/irmanifest +++ b/usr/bin/irmanifest @@ -99,7 +99,7 @@ function irfiletime { source=$(echo $SOURCE | tr '[:upper:]' '[:lower:]') DETAILS="$MANIFESTS/irdata_manifest_$source.txt" - "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_manifest.py" "$URL" < "$DETAILS" \ + "$TOOLS/irdata_manifest.py" "$URL" < "$DETAILS" \ | sed -e "s/\(.*\)/$SOURCE\t\\1/" # If a source uses a manifest parser, use that to obtain information. diff --git a/usr/bin/irmissing b/usr/bin/irmissing index 53a998cf..6c9ca798 100644 --- a/usr/bin/irmissing +++ b/usr/bin/irmissing @@ -69,14 +69,14 @@ if [ ! -e "$DATADIR" ] ; then fi if [ "$CHECK" ] ; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_unknown_refseq.sql" "$DATADIR" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_unknown_refseq.sql" "$DATADIR" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export RefSeq identifiers not providing sequences." 1>&2 exit 1 fi # commented since gi no longer in use -# if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_unknown_gi.sql" "$DATADIR" $PSQL_OPTIONS ; then +# if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_unknown_gi.sql" "$DATADIR" $PSQL_OPTIONS ; then # echo "$PROGNAME: Could not export GenBank identifiers not providing sequences." 1>&2 # exit 1 # fi @@ -102,7 +102,7 @@ if [ "$IMPORT" ] ; then exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_unknown_refseq.sql" "$DATADIR" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/import_unknown_refseq.sql" "$DATADIR" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not import missing sequences." 1>&2 exit 1 fi diff --git a/usr/bin/iroutput b/usr/bin/iroutput index 5a5d4caa..42859bbf 100644 --- a/usr/bin/iroutput +++ b/usr/bin/iroutput @@ -76,7 +76,7 @@ if [ "$MITAB" ]; then cp "$SQL/organisms.txt" "$DATA" - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_mitab.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_mitab.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate MITAB data." 1>&2 exit 1 fi @@ -107,7 +107,7 @@ fi if [ "$MAPPING" ]; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_protein_identifier_mapping.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_protein_identifier_mapping.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate mapping file." 1>&2 exit 1 fi @@ -125,7 +125,7 @@ if [ "$IREFSCAPE" ]; then rm "$DATA/iRefScape/"* fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefscape.sql" "$DATA/iRefScape" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefscape.sql" "$DATA/iRefScape" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate iRefScape data." 1>&2 exit 1 fi @@ -147,7 +147,7 @@ if [ "$IREFWEB" ]; then rm "$DATA/iRefWeb/"* fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefweb.sql" "$DATA/iRefWeb" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefweb.sql" "$DATA/iRefWeb" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not generate iRefWeb data." 1>&2 exit 1 fi diff --git a/usr/bin/irparallel b/usr/bin/irparallel index 261a84a2..a71f1da8 100755 --- a/usr/bin/irparallel +++ b/usr/bin/irparallel @@ -15,10 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -# My interpreter -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} -current_time=$(date +"%H:%M:%S") - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then @@ -45,13 +41,7 @@ EOF exit 1 fi - COMMAND=$1 -# Log the value of $1 -echo "irparallel: \$1 is $1" 1>&2 -# Log the value of $1 -echo "\$COMMAND is $COMMAND; current time: $current_time" > /home/irefindex/logfile2.log -echo "irparallel: \$1 is $1; current time: $current_time" >> /home/irefindex/logfile2.log # Make sure that certain arguments are defined. if [ ! "$COMMAND" ]; then @@ -59,49 +49,6 @@ if [ ! "$COMMAND" ]; then exit 1 fi - -EXPLODED=($COMMAND) -EXENAME=$(echo ${EXPLODED[0]} | sed 's/\"//g') -if [[ $(basename $EXENAME .py) == $(basename $EXENAME) ]]; then - COMMAND2="$COMMAND" -else - REST="\"$(echo ${EXPLODED[@]:1} | sed 's/\"//g')\""; - COMMAND2="$USE_PYTHON_INTERPRETER ${EXPLODED[0]} $REST" -fi -echo "\$REST is $REST" >> /home/irefindex/logfile2.log - -echo "$(basename $EXENAME .py)" >> /home/irefindex/logfile2.log -echo "$EXENAME" >> /home/irefindex/logfile2.log - -# Extract the first part of the COMMAND -#FIRST_PART=$(echo "$COMMAND" | awk '{print $1}' | tr -d '"') -# Check if the first part ends in .py -#if [[ "$FIRST_PART" == *.py ]]; then - # Prepend Python interpreter to the commandx -# COMMAND2="$USE_PYTHON_INTERPRETER $COMMAND" -#fi -#else - # Assign the command to COMMAND2 if it's not a Python script -# COMMAND2="$COMMAND" -#fi - -echo "The value of command2 after checking if it ends at .py: : $COMMAND2 ; current time: $current_time" >> /home/irefindex/logfile2.log - -# Check if the command is for irdata_altsplit.py -#if [[ "$COMMAND" == *"irdata_altsplit.py"* ]]; then -# # Process each item differently -# while IFS= read -r item; do -# # Extract the filename, rank, and total from the item -# filename=$(echo "$item" | awk '{print $1}') -# rank=$(echo "$item" | awk '{print $2}') -# total=$(echo "$item" | awk '{print $3}') -# # Execute the command with the extracted arguments using xargs and sh -c -# echo "$filename $rank $total" | xargs -n 3 -P $PROCESSES sh -c "$COMMAND2" -# done -#else -# # Process items normally -# xargs -P $PROCESSES $XARGS_I'{}' sh -c "$COMMAND2" -#fi +# The pipeline. xargs -P $PROCESSES $XARGS_I'{}' sh -c "$COMMAND2" -echo "The end of the irparallel script" >> /home/irefindex/logfile2.log diff --git a/usr/bin/irparse b/usr/bin/irparse index 89acf82f..b6d98cf4 100755 --- a/usr/bin/irparse +++ b/usr/bin/irparse @@ -15,8 +15,6 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . - - if [ -e "irdata-config" ]; then . "$PWD/irdata-config" elif [ -e "scripts/irdata-config" ]; then diff --git a/usr/bin/irparse-source b/usr/bin/irparse-source old mode 100755 new mode 100644 index 2d81cbae..fcb03957 --- a/usr/bin/irparse-source +++ b/usr/bin/irparse-source @@ -24,9 +24,6 @@ else . 'irdata-config' fi -USE_PYTHON_INTERPRETER=${python_interpreter:-/usr/bin/python3} - - if [ "$1" = '--help' ]; then cat 1>&2 < @@ -147,7 +144,7 @@ for FILEPATTERN in $FILEPATTERNS; do # Where the check fails, run the fixing tool if requested. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_xmlcheck.py" "$FILE" > /dev/null 2> "$CHECKLOG" ; then + if ! "$TOOLS/irdata_xmlcheck.py" "$FILE" > /dev/null 2> "$CHECKLOG" ; then if [ "$FIX" ]; then "$TOOLS/irdata_fixencoding.sh" "$FILE" @@ -177,7 +174,7 @@ for FILEPATTERN in $FILEPATTERNS; do echo "$TOOLS/$PARSER" "$OUTPUTDIR" $FILES if [ ! "$NO_PARSE" ]; then if [ "$XMLFILEPATTERN" ]; then - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_xml2tab.py" "$OUTPUTDIR" "$SOURCE" $FILES ; then + if ! "$TOOLS/irdata_xml2tab.py" "$OUTPUTDIR" "$SOURCE" $FILES ; then echo "$PROGNAME: XML processing for source $SOURCE failed." 1>&2 exit 1 fi diff --git a/usr/bin/irprevious b/usr/bin/irprevious index f1f8884e..c1449096 100644 --- a/usr/bin/irprevious +++ b/usr/bin/irprevious @@ -55,24 +55,24 @@ if [ "$PGSQL" ]; then # Export identifiers from a PostgreSQL database. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_integer_rigids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_integer_rigids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export integer RIG identifiers from the previous database." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_integer_rogids.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_integer_rogids.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export integer ROG identifiers from the previous database." 1>&2 exit 1 fi # Export archived identifiers from a PostgreSQL database. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_archived.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export archived identifiers from the previous database." 1>&2 exit 1 fi - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_archived_sequences.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefindex_archived_sequences.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export archived sequences from the previous database." 1>&2 exit 1 fi diff --git a/usr/bin/irprevious-irefweb b/usr/bin/irprevious-irefweb index b8af5cec..37c40fe0 100644 --- a/usr/bin/irprevious-irefweb +++ b/usr/bin/irprevious-irefweb @@ -55,7 +55,7 @@ if [ "$PGSQL" ]; then # Export data from a PostgreSQL database. - if ! "${USE_PYTHON_INTERPRETER}" "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefweb.sql" "$DATA" $PSQL_OPTIONS ; then + if ! "$TOOLS/irdata_database_action.py" "$DATABASE" "$SQL/export_irefweb.sql" "$DATA" $PSQL_OPTIONS ; then echo "$PROGNAME: Could not export iRefWeb data from the previous database." 1>&2 exit 1 fi From 03a44d617cf6a20a591799bad2c2f1609217c230 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Sat, 1 Jun 2024 11:52:25 +0200 Subject: [PATCH 54/74] Update ETN-EliasSteyaert-stage.Rmd Adding week 4 to the ETN --- .../ETN-EliasSteyaert-stage.Rmd | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index 4cac1120..3f298d9c 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -167,6 +167,26 @@ After the /mnt/disks/data was set-up, the playbook could be played from my VM on - Trying little things like installing pip into the environment, even though it should be already there +#### Thursday 16/05/2024 + +##### Problem-solving new problems the Irparse playbook brings with it +- Making a smaller version of the ansible playbook where the same error occurs so it’s easier to try to get the error out of the script. To test this, an ansible playbook was made(.yml). This playbook executed a bash script (.sh) which on it's part executed a python script (.py) where the irdata module could be found in. +- Trying multiple things to fix the script but failing so. + + +#### Friday (17/05/2024) + +##### Problem-solving new problems the Irparse playbook brings with it +- Trying to get all the jobs running + +##### Downloading the "mint" database +- Making a sources2.yml where only mint is present so it can be downloaded quicker and the errors are better logged. +- Making the download work after seeing where the error occurs (the link had to be updated). + + + + + From 03951d00212ffb22a75e057a035a9000b5f12a13 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Sat, 1 Jun 2024 14:40:20 +0200 Subject: [PATCH 55/74] Update ETN-EliasSteyaert-stage.Rmd adding week 5 and 6 --- .../ETN-EliasSteyaert-stage.Rmd | 113 +++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index 3f298d9c..f16340aa 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -47,7 +47,6 @@ https://github.com/EliasSteyaert/stage-VIB-irefindex - Forking and cloning the VIB github repository into my own and setting up the needed folders/files into it (https://github.com/EliasSteyaert/stage-VIB-irefindex) - #### Thursday 25/04/2023 ##### Performing the "terraform-step" of the procedure: @@ -59,7 +58,6 @@ https://github.com/EliasSteyaert/stage-VIB-irefindex - Making a troubleshooting file and filling in all the issueas that were encountered so far - #### Friday 26/04/2023 ##### Trying to perform "terraform apply" @@ -73,6 +71,7 @@ Connecting to the instance. ##### Playing the first playbook +## Week 2 #### Monday 29/04/2024 ##### Running the playbook @@ -109,6 +108,7 @@ Connecting to the instance. - Running into new problems while running the irparse playbook: creating a venv and trying to get the irdata package downloaded +## Week 3 #### Monday 06/05/2024 ##### Problem-solving new problems the Irparse playbook brings with it @@ -143,6 +143,7 @@ Connecting to the instance. After the /mnt/disks/data was set-up, the playbook could be played from my VM onto the google cloud instance. +## Week 4 #### Monday 13/05/2024 ##### Playing the playbook all over again @@ -184,6 +185,112 @@ After the /mnt/disks/data was set-up, the playbook could be played from my VM on - Making the download work after seeing where the error occurs (the link had to be updated). +## Week 5 +#### Monday (20/05/2024) + +##### Pentecostmonday + + +#### Tuesday (21/05/2024) + +##### Irdownload for the databases that didn't work before +- Fully downloading the mint database +- Trying to get the irdownload running for “ipi”, “bar”, “corumdb” and “matrixdb” + +##### Fixing the “Irdata” problem +- Problem-solving the irdata "module not found" further on the basis of the mini-version of the scripts +- Learning more about the subprocess module: what it is and what it is used for (https://docs.python.org/3/library/subprocess.html) + + +#### Wednesday (22/05/2024) + +##### Learning more about the individual aspects of the scripts +- Figuring out what “dir” does, why it is usefull and how it is used + +##### Fixing the irpase playbook +- Finally fixing the irparse playbook so the "irmodule irdata could not be found" error is solved +- Piece of code that installed the irdata, with this solution, every .sh script where irdata is being used needs to be in the virtual environment: + - name: Ensure python3-venv and python3-pip are installed + ansible.builtin.apt: + name: + - python3-venv + - python3-pip + state: present + + - name: Create virtual environment + ansible.builtin.command: + cmd: python3 -m venv "{{ download_data }}/{{ data2 }}/virtualenv" + args: + creates: "{{ download_data }}/{{ data2 }}/virtualenv" + + - name: install irdata + ansible.builtin.command: + cmd: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python3 -m pip install irdata" + +##### Parsing the job "Athelian" +- With the knowledge that the smaller version of the playbook which I made earlier works, I can adjust the process of parsing "Athelian +- Figuring out how to get the job “Athelian” to get parsed and get passed the error + + +#### Thursday (23/05/2024) + +##### Running the irparse playbook +- Running the jobs one by one, fixing the easy errors and logging the harder errors. + + +#### Friday (24/05/2024) + +##### Fixing a parsing error +- Investigating the error: mkdir: cannot create directory '/home/elias.steyaert/.ansible/tmp/ansible-tmp-1716456592.068384-36656-83367091344262': No space left on device +- This was fixed with making an ansible.cfg file with the code “remote_tmp = /mnt/disks/data +- Investigating the error: "sh: 1: Syntax error: Unterminated quoted string" + + +## Week 6 +#### Monday (27/05/2024) + +##### Fixing the parsing step of the job “refseq” +- The error "sh: 1: Syntax error: Unterminated quoted string" occured and looking for a fix for that +- Figuring out how the irparallel script works + + +#### Tuesday (28/05/2024) + +##### Getting the irparallel script to work together with the jobs genpept, refseq and uniprot. +- The interpreter and the irparallel together caused an annoying error. +- To fix the error (specifically for refseq first), the following block of code was added to the irparallel script: +COMMAND=$1 +EXPLODED=($COMMAND) +EXENAME=$(echo ${EXPLODED[0]} | sed 's/\"//g') +if [[ $(basename $EXENAME .py) == $(basename $EXENAME) ]]; then + COMMAND2="$COMMAND" +else + REST="\"$(echo ${EXPLODED[@]:1} | sed 's/\"//g')\""; + COMMAND2="$USE_PYTHON_INTERPRETER ${EXPLODED[0]} $REST" +fi + + +#### Wednesday (29/05/2024) + +##### Using the fixed script for refseq to get the UniProt and GenPept parsing working +- Running the GenPept parsing step while trying to get the UniProt parsing step working +- Trying to get the irdata_parse_uniprot.sh compatible to run the irparallel script. Which is hard because the basis of the command is a whole lot bigger with an extra pipeline and two python scripts instead of one + + +#### Thursday (29/05/2024) + +##### Using the fixed script for refseq to get the UniProt and GenPept parsing working +- Running the GenPept parsing step while trying to get the UniProt parsing step working +- Trying to get the irdata_parse_uniprot.sh compatible to run the irparallel script. + + +#### Friday (30/05/2024) +##### Trying to get the irdata_parse_uniprot.sh compatible to run the irparallel script. +- Not succeeding at this and deciding to start all over again, just before the step where the python interpreter starts getting used. Cleaning up the files were there are traces of the usage of a virtual environment + + + + @@ -193,6 +300,8 @@ After the /mnt/disks/data was set-up, the playbook could be played from my VM on + + ### Trainee documentation plan (TDP) #####Tentative planning: From 27ddbc4dd9008c68ab923620b019b27d88fec4ff Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Sat, 1 Jun 2024 15:08:08 +0200 Subject: [PATCH 56/74] Update ETN-EliasSteyaert-stage.Rmd Getting the code blocks cleaner when knitted --- .../ETN-EliasSteyaert-stage.Rmd | 46 +++++++------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index f16340aa..68dedb2e 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -127,8 +127,11 @@ Connecting to the instance. ##### Problem-solving new problems the Irparse playbook brings with it - Creating a "for loop" to get the python interpreter into the scripts which need the environment to solve “from irdata import data ModuleNotFoundError: No module named 'irdata'” -- for filename in $(grep -rl "TOOLS" ./usr/bin/* | sort | uniq); do cat "$filename" | sed "s|\(\$TOOLS/.*.py\)|\${USE_PYTHON_INTERPRETER}\" \"\1|g" > "$filename.tmp" && mv "$filename.tmp" "$filename"; done - +```bash +for filename in $(grep -rl "TOOLS" ./usr/bin/* | sort | uniq); do +cat "$filename" | sed "s|\(\$TOOLS/.*.py\)|\${USE_PYTHON_INTERPRETER}\" \"\1|g" +> "$filename.tmp" && mv "$filename.tmp" "$filename"; done +``` #### Thursday 09/05/204 @@ -210,6 +213,7 @@ After the /mnt/disks/data was set-up, the playbook could be played from my VM on ##### Fixing the irpase playbook - Finally fixing the irparse playbook so the "irmodule irdata could not be found" error is solved - Piece of code that installed the irdata, with this solution, every .sh script where irdata is being used needs to be in the virtual environment: +```bash - name: Ensure python3-venv and python3-pip are installed ansible.builtin.apt: name: @@ -226,6 +230,7 @@ After the /mnt/disks/data was set-up, the playbook could be played from my VM on - name: install irdata ansible.builtin.command: cmd: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python3 -m pip install irdata" +``` ##### Parsing the job "Athelian" - With the knowledge that the smaller version of the playbook which I made earlier works, I can adjust the process of parsing "Athelian @@ -242,9 +247,14 @@ After the /mnt/disks/data was set-up, the playbook could be played from my VM on ##### Fixing a parsing error - Investigating the error: mkdir: cannot create directory '/home/elias.steyaert/.ansible/tmp/ansible-tmp-1716456592.068384-36656-83367091344262': No space left on device -- This was fixed with making an ansible.cfg file with the code “remote_tmp = /mnt/disks/data -- Investigating the error: "sh: 1: Syntax error: Unterminated quoted string" - +- This was fixed with making an ansible.cfg file with the code +```bash +“remote_tmp = /mnt/disks/data +``` +- Investigating the error: +```bash +"sh: 1: Syntax error: Unterminated quoted string" +``` ## Week 6 #### Monday (27/05/2024) @@ -259,6 +269,7 @@ After the /mnt/disks/data was set-up, the playbook could be played from my VM on ##### Getting the irparallel script to work together with the jobs genpept, refseq and uniprot. - The interpreter and the irparallel together caused an annoying error. - To fix the error (specifically for refseq first), the following block of code was added to the irparallel script: +```bash COMMAND=$1 EXPLODED=($COMMAND) EXENAME=$(echo ${EXPLODED[0]} | sed 's/\"//g') @@ -268,7 +279,7 @@ else REST="\"$(echo ${EXPLODED[@]:1} | sed 's/\"//g')\""; COMMAND2="$USE_PYTHON_INTERPRETER ${EXPLODED[0]} $REST" fi - +``` #### Wednesday (29/05/2024) @@ -289,17 +300,6 @@ fi - Not succeeding at this and deciding to start all over again, just before the step where the python interpreter starts getting used. Cleaning up the files were there are traces of the usage of a virtual environment - - - - - - - - - - - ### Trainee documentation plan (TDP) @@ -353,15 +353,3 @@ Normally, all this should be done in the traineeship and its 250 hours. Nothing - Achievable: As I’m working daily with ansible for my traineeship, I should be able to achieve this goal. - Relevant: In the context of the traineeship, this learning outcome is relevant. - Time-bound: This should be achieved by the end of the traineeship. - - - - - - - - - - - - From 682bbb7dd82e32a5a4b6c3070e0f19a293029b96 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Thu, 6 Jun 2024 12:21:41 +0200 Subject: [PATCH 57/74] minor adjustments to find the ongoing probleme, which was a wrong variable usage --- usr/bin/irdata-config | 13 ++++--------- usr/bin/irdata_parse_A_thaliana.sh | 4 ++++ usr/bin/irdata_parse_A_thaliana2.py | 4 +++- usr/bin/irparallel | 3 ++- usr/bin/irparse | 9 +++++++++ usr/bin/irparse-source | 4 ++++ 6 files changed, 26 insertions(+), 11 deletions(-) diff --git a/usr/bin/irdata-config b/usr/bin/irdata-config index 03150f16..98133bf3 100644 --- a/usr/bin/irdata-config +++ b/usr/bin/irdata-config @@ -1,11 +1,5 @@ #!/bin/bash - - -############################################################ -# My own environments - - ############################################################ # Output and publication options. # (RELEASE is most likely to need changing.) @@ -708,6 +702,7 @@ if [ "$1" = '--inspect-only' ]; then shift 1 fi + # Tools. if [ -e "$SCRIPTS/../tools" ]; then @@ -768,7 +763,7 @@ fi # Data directory. SYSDATA="$SYSPREFIX/var/lib/irdata/data" -echo "marker1 $DATA" + if [ "$DATA" ] && [ -e "$DATA" ]; then : elif [ -e "$SCRIPTS/../data" ]; then @@ -779,7 +774,6 @@ else DATA="$SYSDATA" fi -echo "marker $DATA" if [ ! -e "$DATA" ] && [ ! "$INSPECTONLY" ]; then echo "Cannot locate the irdata data directory." 1>&2 @@ -788,7 +782,7 @@ fi # Log file directory. -SYSLOGS="$SYSPREFIX/var/log/irdata" +SYSLOGS="$DATAPREFIX/var/log/irdata" if [ -e "$SCRIPTS/../logs" ]; then LOGS=`realpath "$SCRIPTS/../logs"` @@ -798,6 +792,7 @@ else LOGS="$SYSLOGS" fi + if [ ! -e "$LOGS" ] && [ ! "$INSPECTONLY" ]; then echo "Cannot locate the irdata logs directory." 1>&2 exit 1 diff --git a/usr/bin/irdata_parse_A_thaliana.sh b/usr/bin/irdata_parse_A_thaliana.sh index 49fca643..28f572a1 100644 --- a/usr/bin/irdata_parse_A_thaliana.sh +++ b/usr/bin/irdata_parse_A_thaliana.sh @@ -23,6 +23,8 @@ else . 'irdata-config' fi +echo "does it get till here 5" + if [ "$1" = '--help' ]; then cat 1>&2 < @@ -59,3 +61,5 @@ echo "$DATADIR/$OUTFILE" 1>&2 zcat "$FILENAME" \ | "$TOOLS/irdata_parse_A_thaliana2.py" "$FILETYPE" --discard-ill-formed \ > "$DATADIR/$OUTFILE" + +echo "does it get till here 6" \ No newline at end of file diff --git a/usr/bin/irdata_parse_A_thaliana2.py b/usr/bin/irdata_parse_A_thaliana2.py index d9d0924c..af1c0dc7 100755 --- a/usr/bin/irdata_parse_A_thaliana2.py +++ b/usr/bin/irdata_parse_A_thaliana2.py @@ -23,7 +23,6 @@ import re - class Parser: "A parser for UniProt Arabidopsis format text files." @@ -137,3 +136,6 @@ def write_log(self, level, message): sys.exit(1) # vim: tabstop=4 expandtab shiftwidth=4 + + +print("does it get till here 7") \ No newline at end of file diff --git a/usr/bin/irparallel b/usr/bin/irparallel index a71f1da8..f3f5a3f4 100755 --- a/usr/bin/irparallel +++ b/usr/bin/irparallel @@ -44,6 +44,7 @@ fi COMMAND=$1 # Make sure that certain arguments are defined. + if [ ! "$COMMAND" ]; then echo "$PROGNAME: Please specify a command to run on each file (to be given as {})." 1>&2 exit 1 @@ -51,4 +52,4 @@ fi # The pipeline. -xargs -P $PROCESSES $XARGS_I'{}' sh -c "$COMMAND2" +xargs -P $PROCESSES $XARGS_I'{}' sh -c "$COMMAND" \ No newline at end of file diff --git a/usr/bin/irparse b/usr/bin/irparse index b6d98cf4..f12f36db 100755 --- a/usr/bin/irparse +++ b/usr/bin/irparse @@ -23,6 +23,8 @@ else . 'irdata-config' fi +echo "does it get till here 1" + if [ "$1" = '--help' ]; then cat 1>&2 <... ) @@ -40,6 +42,8 @@ EOF exit 1 fi +echo "does it get till here 2" + if [ "$1" = '--fix' ]; then FIX=$1 shift 1 @@ -64,6 +68,9 @@ elif [ ! "$SOURCES" ]; then exit 1 fi + +echo "does it get till here 3" + # Exclude sources. PARSESOURCES= @@ -76,3 +83,5 @@ done # The pipeline. "$SCRIPTS/argument-per-line" $PARSESOURCES | "$SCRIPTS/irparallel" "\"$SCRIPTS/irparse-source\" $FIX $CHECK $NO_PARSE {}" + +echo "does it get till here 4" \ No newline at end of file diff --git a/usr/bin/irparse-source b/usr/bin/irparse-source index fcb03957..2f28feb4 100644 --- a/usr/bin/irparse-source +++ b/usr/bin/irparse-source @@ -24,6 +24,8 @@ else . 'irdata-config' fi +echo "does it get till here 5.05" + if [ "$1" = '--help' ]; then cat 1>&2 < @@ -201,3 +203,5 @@ else echo "$PROGNAME: Source $SOURCE did not provide any data." 1>&2 exit 1 fi + +echo "does it get till here 5.1" \ No newline at end of file From b51fd24fb2c2e440d15550c8b9f82c9525439e91 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Thu, 6 Jun 2024 12:22:55 +0200 Subject: [PATCH 58/74] the .ansible logs were going onto the / disk again --- vsc/ansible.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 vsc/ansible.cfg diff --git a/vsc/ansible.cfg b/vsc/ansible.cfg new file mode 100644 index 00000000..017fe796 --- /dev/null +++ b/vsc/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +remote_tmp = /mnt/disks/data From d75b2ae59fd68b8aa6804dd20139459bd09af91e Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Thu, 6 Jun 2024 12:26:02 +0200 Subject: [PATCH 59/74] minor adjustment to figure out if the parsing worked --- usr/bin/irdata_parse_uniprot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/irdata_parse_uniprot.sh b/usr/bin/irdata_parse_uniprot.sh index 58bc4325..373e85e2 100644 --- a/usr/bin/irdata_parse_uniprot.sh +++ b/usr/bin/irdata_parse_uniprot.sh @@ -70,7 +70,7 @@ for FILENAME in $FILENAMES; do | "$SCRIPTS/irparallel" "$TOOLS/irdata_altsplit.py \"$FILENAME\" {} \"$PROCESSES\" | $TOOLS/irdata_parse_uniprot.py \"$DATADIR\" - \"${LEAFNAME}_%s-{}.txt\"" #for rank in $(seq 0 $((PROCESSES - 1))); do - # cmd="\"$TOOLS/irdata_altsplit.py\" \"$FILENAME\" \"$rank\" \"$PROCESSES\" | \"${USE_PYTHON_INTERPRETER}\" \"$TOOLS/irdata_parse_uniprot.py\" \"$DATADIR\" - \"${LEAFNAME}_%s-$rank.txt\"" + # cmd="\"$TOOLS/irdata_altsplit.py\" \"$FILENAME\" \"$rank\" \"$PROCESSES\" | \"$TOOLS/irdata_parse_uniprot.py\" \"$DATADIR\" - \"${LEAFNAME}_%s-$rank.txt\"" # echo "Executing: $cmd" # Debug: print the command being executed # "$SCRIPTS/irparallel" "$cmd" #done From 2e62588b6459fc623100156741fa0a566f46f9d3 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Sat, 8 Jun 2024 17:32:23 +0200 Subject: [PATCH 60/74] Delete internships/ETN-Elias-howest/test deleting an empty file --- internships/ETN-Elias-howest/test | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 internships/ETN-Elias-howest/test diff --git a/internships/ETN-Elias-howest/test b/internships/ETN-Elias-howest/test deleted file mode 100644 index e69de29b..00000000 From 335da85ba7e65b1e31d5c49a25b62e57c6cc1a09 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 12 Jun 2024 14:32:15 +0200 Subject: [PATCH 61/74] Changing code so the data from postgresql gets written to the /mnt/disks/data instead of the home directory (through the postgresql.conf) --- vsc/ansible/irinit.yml | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/vsc/ansible/irinit.yml b/vsc/ansible/irinit.yml index 31099cb8..44e0fb35 100644 --- a/vsc/ansible/irinit.yml +++ b/vsc/ansible/irinit.yml @@ -97,16 +97,52 @@ when: irdata_db.rowcount == 0 changed_when: true + - name: Ensure specific settings are set in postgresql.conf + lineinfile: + path: /etc/postgresql/14/main/postgresql.conf + regexp: '^#?{{ item.setting_name }}\s*=' + line: '{{ item.setting_name }} = {{ item.desired_value }}' + state: present + with_items: + - { setting_name: 'data_directory', desired_value: '/mnt/disks/data/var/lib/postgresql/14/main' } #CHANGE THIS TO THE DISK YOU WANT TO WRITE IT TO + - { setting_name: 'stats_temp_directory', desired_value: '/mnt/disks/data/var/run/postgresql/14-main.pg_stat_tmp'} #CHANGE THIS TO THE DISK YOU WANT TO WRITE IT TO + + - name: Restart PostgreSQL service + service: + name: postgresql + state: restarted + - name: Install Git ansible.builtin.apt: name: git state: present + - name: Install rsync + ansible.builtin.apt: + name: rsync + state: present + - name: Copy the irefindex files from my local machine to the remote machine - ansible.builtin.copy: - src: "{{ local_source_directory }}" + ansible.builtin.synchronize: + src: "{{ local_source_directory }}/" dest: "{{ remote_destination_directory }}" - + delete: yes + rsync_opts: + - "--delete" + + - name: Find all files in /home/irefindex/usr/bin + find: + paths: /home/irefindex/usr/bin + recurse: yes + file_type: file + register: files_to_update + + - name: Set execute permissions on all files + file: + path: "{{ item.path }}" + mode: '0755' + loop: "{{ files_to_update.files }}" + - name: Create directories for irdata data ansible.builtin.file: path: "{{ item }}" From 918a47e506c99f55ddb4d7252d40aa5019c61c3f Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 12 Jun 2024 14:35:20 +0200 Subject: [PATCH 62/74] removing echos that were added to understand a problem better --- usr/bin/irdata_parse_A_thaliana.sh | 6 +----- usr/bin/irdata_parse_A_thaliana2.py | 3 --- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/usr/bin/irdata_parse_A_thaliana.sh b/usr/bin/irdata_parse_A_thaliana.sh index 28f572a1..b7f44eaa 100644 --- a/usr/bin/irdata_parse_A_thaliana.sh +++ b/usr/bin/irdata_parse_A_thaliana.sh @@ -23,8 +23,6 @@ else . 'irdata-config' fi -echo "does it get till here 5" - if [ "$1" = '--help' ]; then cat 1>&2 < @@ -60,6 +58,4 @@ echo "$FILETYPE" echo "$DATADIR/$OUTFILE" 1>&2 zcat "$FILENAME" \ | "$TOOLS/irdata_parse_A_thaliana2.py" "$FILETYPE" --discard-ill-formed \ -> "$DATADIR/$OUTFILE" - -echo "does it get till here 6" \ No newline at end of file +> "$DATADIR/$OUTFILE" \ No newline at end of file diff --git a/usr/bin/irdata_parse_A_thaliana2.py b/usr/bin/irdata_parse_A_thaliana2.py index af1c0dc7..4a40a367 100755 --- a/usr/bin/irdata_parse_A_thaliana2.py +++ b/usr/bin/irdata_parse_A_thaliana2.py @@ -136,6 +136,3 @@ def write_log(self, level, message): sys.exit(1) # vim: tabstop=4 expandtab shiftwidth=4 - - -print("does it get till here 7") \ No newline at end of file From e73343e31cf0a2440fccf5eb2cc58a944ed3b835 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 12 Jun 2024 14:36:34 +0200 Subject: [PATCH 63/74] adjusting so the .async_dir doesn't write on the / disk again --- vsc/ansible.cfg | 1 + vsc/ansible/ansible.cfg | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/vsc/ansible.cfg b/vsc/ansible.cfg index 017fe796..fca0b703 100644 --- a/vsc/ansible.cfg +++ b/vsc/ansible.cfg @@ -1,2 +1,3 @@ [defaults] remote_tmp = /mnt/disks/data +async_dir = /mnt/disks/data/.ansible_async \ No newline at end of file diff --git a/vsc/ansible/ansible.cfg b/vsc/ansible/ansible.cfg index cabd4459..fca0b703 100644 --- a/vsc/ansible/ansible.cfg +++ b/vsc/ansible/ansible.cfg @@ -1,2 +1,3 @@ [defaults] -remote_tmp = /mnt/disks/data \ No newline at end of file +remote_tmp = /mnt/disks/data +async_dir = /mnt/disks/data/.ansible_async \ No newline at end of file From 84e9489e1f02c8259e6d5faf99a1558eef4b1fcd Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 12 Jun 2024 14:37:32 +0200 Subject: [PATCH 64/74] adding a download_data variable --- vsc/ansible/vars/logs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vsc/ansible/vars/logs.yml b/vsc/ansible/vars/logs.yml index 33c29729..a5970f7a 100644 --- a/vsc/ansible/vars/logs.yml +++ b/vsc/ansible/vars/logs.yml @@ -1,3 +1,4 @@ date: "{{ ansible_date_time.iso8601_basic }}" -logs_root_dir: "/data/irdata/logs" +download_data: "/mnt/disks/data/" # The directory linked to /dev/sda +logs_root_dir: "{{ download_data }}/data/irdata/logs" logs_dir: "{{ logs_root_dir }}/{{ date }}" From a6a3e434f0721522c815cb8ca994a2ff4ea8b621 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 12 Jun 2024 14:39:19 +0200 Subject: [PATCH 65/74] minor or no changes when trying to find out how irimport really works --- usr/bin/irimport | 0 usr/bin/irparse | 10 +--------- usr/bin/irparse-source | 4 ---- 3 files changed, 1 insertion(+), 13 deletions(-) mode change 100644 => 100755 usr/bin/irimport diff --git a/usr/bin/irimport b/usr/bin/irimport old mode 100644 new mode 100755 diff --git a/usr/bin/irparse b/usr/bin/irparse index f12f36db..f67b9286 100755 --- a/usr/bin/irparse +++ b/usr/bin/irparse @@ -23,8 +23,6 @@ else . 'irdata-config' fi -echo "does it get till here 1" - if [ "$1" = '--help' ]; then cat 1>&2 <... ) @@ -42,8 +40,6 @@ EOF exit 1 fi -echo "does it get till here 2" - if [ "$1" = '--fix' ]; then FIX=$1 shift 1 @@ -69,8 +65,6 @@ elif [ ! "$SOURCES" ]; then fi -echo "does it get till here 3" - # Exclude sources. PARSESOURCES= @@ -82,6 +76,4 @@ for SOURCE in $SOURCES ; do done # The pipeline. -"$SCRIPTS/argument-per-line" $PARSESOURCES | "$SCRIPTS/irparallel" "\"$SCRIPTS/irparse-source\" $FIX $CHECK $NO_PARSE {}" - -echo "does it get till here 4" \ No newline at end of file +"$SCRIPTS/argument-per-line" $PARSESOURCES | "$SCRIPTS/irparallel" "\"$SCRIPTS/irparse-source\" $FIX $CHECK $NO_PARSE {}" \ No newline at end of file diff --git a/usr/bin/irparse-source b/usr/bin/irparse-source index 2f28feb4..fcb03957 100644 --- a/usr/bin/irparse-source +++ b/usr/bin/irparse-source @@ -24,8 +24,6 @@ else . 'irdata-config' fi -echo "does it get till here 5.05" - if [ "$1" = '--help' ]; then cat 1>&2 < @@ -203,5 +201,3 @@ else echo "$PROGNAME: Source $SOURCE did not provide any data." 1>&2 exit 1 fi - -echo "does it get till here 5.1" \ No newline at end of file From 60a73bf3c679a6c5315dc1c144e0f8cca4066428 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Wed, 12 Jun 2024 22:39:21 +0200 Subject: [PATCH 66/74] Update ETN-EliasSteyaert-stage.Rmd adding new info to the ETN up to wednesday 12/06/2024 --- .../ETN-EliasSteyaert-stage.Rmd | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index 68dedb2e..457ce832 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -299,6 +299,92 @@ fi ##### Trying to get the irdata_parse_uniprot.sh compatible to run the irparallel script. - Not succeeding at this and deciding to start all over again, just before the step where the python interpreter starts getting used. Cleaning up the files were there are traces of the usage of a virtual environment +## Week 7 +#### Monday (03/06/2024) +##### Running into an error while running irparse.yml +- Cannot locate the irdata logs directory. +- the problem could be tracked down with "grep -r" till it could be concluded that it occured here: +```bash +if [ ! -e "$LOGS" ] && [ ! "$INSPECTONLY" ]; then echo "Cannot locate the irdata logs directory." 1>&2 exit 1 fi +``` +it was trying to write to a directory that didn’t exist. +- solution, changing the $LOGS into a directory that does exist (on the mnt disk) +- Cleaning the playbooks up so all the signs to an external environment are gone + + +#### Tuesday (04/06/2024) +##### Running into an error while running irparse.yml +- cannot create directory ‘/home/elias.steyaert/.ansible/tmp/ansible-tmp-1717502869.8542364-50434-182160768545910’: No space left on device +Making another ansibl.cfg file but now on a different directory so that directory does also have a config file +- Irparse didn’t work, no error or message on why it was happening -> in irparallel there was a variable being used which didn’t exist anymore + +##### manual QC for the refseq parsing step +- Following the parsing step of the refseq database to make up a conclusion if everything is parsed or not + + +#### Wednesday (05/06/2024) +##### Running irimport +- Running irimport for the first time, first with all the jobs at once, after that only with the Athaliana +- Rapidly showing new errors + + +#### Thursday (06/06/2024) +##### Trying to download “gene” +- Which constantly fails or gives error messages like: Can’t find the job” while all the right things are included. + +##### The "/" directory was full again, this time not because of temporary files +- Figuring out a good CLI command to find the right output to see where the largest files are: +```bash +sudo du -hS / | grep -v "^/mnt" | sort -h -r | head -n 25 | more +``` +- The problem was a random /tmp file in the /home/irefindex/usr/bin directory + + +#### Friday (07/06/2024) +##### Checking for each downloaded source that everything is downloaded +- This is done manually, the size ("ls -lh" into the downloaded folder) is compared with how big the databases are online (the source) +- Trying to succesfully download Uniprot, which always stops at a maximum of 16G. Trying to adjust the custom settings of the job without succeeding + + +## Week 8 +#### Monday (10/06/2024) +##### Trying to get the database Uniprot fully downloaded, as of now it always stops around 16GB of the 171GB. +- For example tried the code: +```bash +# Run jobs for uniprot +- name: Run jobs for uniprot +ansible.builtin.command: "./irdownload {{ item.name | upper }}" +args: +chdir: "/home/irefindex/usr/bin" +async: "{{ item.download_runner_async | default(2000) }}" +poll: 0 +register: result +retries: "{{ item.download_retries | default(60) }}" +until: result.finished +delay: "{{ item.download_delay | default(100) }}" +loop: "{{ jobs }}" +when: item.name == "uniprot" +``` +- It finally went on to download all the GB's, let this run in a tmux session because downloading 172GB takes its time + + +#### Tuesday (11/06/2024) +##### Adjusting the playbook so the parsing of Uniprot can’t be timed-out by ansible. +- Parsing the Uniprot data, this took more than 24 houres and so was done in a tmux session + +#### Parsing Uniprot and trying to run irimport +- Running irimport on the jobs that were succesfull after downloading and parsing and having a look what goes wrong (the data gets written to the “/” disk instead on a disk with more space) + + +#### Wednesday (12/06/2024) +#####Working on powerpoint presentation + +##### adding code in irinit +- This is done so the data from irimport is written on the disk and not in the home directory (adjusting the postgresql.conf file through ansible so its automated) + + + + From d14366d9cf1d32dede6f2b287b56ca2410bfd394 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Wed, 12 Jun 2024 22:41:00 +0200 Subject: [PATCH 67/74] Update ETN-EliasSteyaert-stage.Rmd minor changes --- internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index 457ce832..4bf10bcc 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -377,7 +377,9 @@ when: item.name == "uniprot" #### Wednesday (12/06/2024) -#####Working on powerpoint presentation +##### Working on powerpoint presentation +- Presenting this ppt and answering questions about the project +- Going over the evaluation report together with James Collier (supervisor) ##### adding code in irinit - This is done so the data from irimport is written on the disk and not in the home directory (adjusting the postgresql.conf file through ansible so its automated) From 0410b79f7350b00642af7dc80dd7e0609ecc8948 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:43:28 +0200 Subject: [PATCH 68/74] Create README.md making a README file --- internships/ETN-Elias-howest/README.md | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 internships/ETN-Elias-howest/README.md diff --git a/internships/ETN-Elias-howest/README.md b/internships/ETN-Elias-howest/README.md new file mode 100644 index 00000000..7a2bfccb --- /dev/null +++ b/internships/ETN-Elias-howest/README.md @@ -0,0 +1,32 @@ +To get started: +Start of with forking the github of iRefIndex from VIB. (https://github.com/vibbits/irefindex.git) +Ask a supervisor to set up a google cloud instance on a remote server. +To connect with the server you will need the command: +```bash +ssh @ -i id_ed25519 +an example of this is the following: +ssh elias.steyaert@35.240.84.94 -i id_ed25519 +the id_ed25519 should be found in the ~/.ssh/ folder, if not, it can be added with the command: ssh-add ~/.ssh/id_ed25519 +``` +When the server is fully set up, the next thing should be to mount the "/dev/sda/" disk. +To do this, follow the documentation here: https://cloud.google.com/compute/docs/disks/format-mount-disk-linux +It is HIGHLY RECOMMENDED if the "/dev/sda" disk is mounted with "/mnt/disks/data". That way the variables linked to this disk won't have to be changed. +This is easily done if you follow the documentation and replace "MOUNT_DIR" with "data". + +Go into the directory: /irefindex/vsc and make a text file called "inventory" +In this text file, you paste the ip adress of the google cloud server. +in this same directory, you should also make an "ansible.cfg" file. This file should have: +```bash +[defaults] +remote_tmp = /mnt/disks/data +async_dir = /mnt/disks/data/.ansible_async +``` + +When these things are done, you can start the first playbook. This is done by going into the directory: /irefindex/vsc and performing the following command +```bash +ansible-playbook -i inventory -u '' --private-key=~/.ssh/id_ed25519 ansible/main1.yml +in my case this looked like this: +guest@fedora:~/internship/stage-VIB-irefindex/vsc]$ ansible-playbook -i inventory -u 'elias.steyaert' --private-key=~/.ssh/id_ed25519 ansible/after_main1.yml +(I forked the irefindex github into the directory "internship" and I changed the name of the irefindex directory to "stage-VIB-irefindex) +``` + From 0b384b93314afe144995539666272aa575617689 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:44:18 +0200 Subject: [PATCH 69/74] Update README.md minor changes --- internships/ETN-Elias-howest/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internships/ETN-Elias-howest/README.md b/internships/ETN-Elias-howest/README.md index 7a2bfccb..8b233df2 100644 --- a/internships/ETN-Elias-howest/README.md +++ b/internships/ETN-Elias-howest/README.md @@ -2,7 +2,7 @@ To get started: Start of with forking the github of iRefIndex from VIB. (https://github.com/vibbits/irefindex.git) Ask a supervisor to set up a google cloud instance on a remote server. To connect with the server you will need the command: -```bash +``` ssh @ -i id_ed25519 an example of this is the following: ssh elias.steyaert@35.240.84.94 -i id_ed25519 @@ -16,14 +16,14 @@ This is easily done if you follow the documentation and replace "MOUNT_DIR" with Go into the directory: /irefindex/vsc and make a text file called "inventory" In this text file, you paste the ip adress of the google cloud server. in this same directory, you should also make an "ansible.cfg" file. This file should have: -```bash +``` [defaults] remote_tmp = /mnt/disks/data async_dir = /mnt/disks/data/.ansible_async ``` When these things are done, you can start the first playbook. This is done by going into the directory: /irefindex/vsc and performing the following command -```bash +``` ansible-playbook -i inventory -u '' --private-key=~/.ssh/id_ed25519 ansible/main1.yml in my case this looked like this: guest@fedora:~/internship/stage-VIB-irefindex/vsc]$ ansible-playbook -i inventory -u 'elias.steyaert' --private-key=~/.ssh/id_ed25519 ansible/after_main1.yml From df8f727de5637c06158cbf145680dda7954330ee Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Thu, 13 Jun 2024 17:32:46 +0200 Subject: [PATCH 70/74] extra adjustments so that the path of the data gets changed without errors --- vsc/ansible/irinit.yml | 62 +++++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/vsc/ansible/irinit.yml b/vsc/ansible/irinit.yml index 44e0fb35..6c69ef26 100644 --- a/vsc/ansible/irinit.yml +++ b/vsc/ansible/irinit.yml @@ -68,6 +68,58 @@ name: postgresql state: started + - name: Ensure data directory exists + file: + path: /mnt/disks/data/var/lib/postgresql/14/main + state: directory + owner: postgres + group: postgres + mode: '0777' + + - name: Ensure stats temp directory exists + file: + path: /mnt/disks/data/var/run/postgresql/14-main.pg_stat_tmp + state: directory + owner: postgres + group: postgres + mode: '0777' + + - name: Ensure PostgreSQL instance is stopped before changing configuration + service: + name: postgresql@14-main + state: stopped + + - name: Ensure specific settings are set in postgresql.conf + ansible.builtin.lineinfile: + path: /etc/postgresql/14/main/postgresql.conf + regexp: '^#?\s*{{ item.setting_name }}\s*=' + line: "{{ item.setting_name }} = '{{ item.desired_value }}'" + state: present + loop: + - { setting_name: 'data_directory', desired_value: '/mnt/disks/data/var/lib/postgresql/14/main' } + - { setting_name: 'stats_temp_directory', desired_value: '/mnt/disks/data/var/run/postgresql/14-main.pg_stat_tmp' } + + - name: Ensure PostgreSQL data directory exists and is empty + become: true + become_user: postgres + shell: rm -rf /mnt/disks/data/var/lib/postgresql/14/main/* + ignore_errors: yes # Ignore errors if the directory is already empty + + - name: Initialize PostgreSQL data directory + become: true + become_user: postgres + command: /usr/lib/postgresql/14/bin/initdb -D /mnt/disks/data/var/lib/postgresql/14/main + + - name: Ensure PostgreSQL instance is running + service: + name: postgresql@14-main + state: started + + - name: Restart PostgreSQL + service: + name: postgresql@14-main + state: restarted + - name: Ensure 'root' user exists with superuser privileges community.postgresql.postgresql_user: name: root @@ -97,16 +149,6 @@ when: irdata_db.rowcount == 0 changed_when: true - - name: Ensure specific settings are set in postgresql.conf - lineinfile: - path: /etc/postgresql/14/main/postgresql.conf - regexp: '^#?{{ item.setting_name }}\s*=' - line: '{{ item.setting_name }} = {{ item.desired_value }}' - state: present - with_items: - - { setting_name: 'data_directory', desired_value: '/mnt/disks/data/var/lib/postgresql/14/main' } #CHANGE THIS TO THE DISK YOU WANT TO WRITE IT TO - - { setting_name: 'stats_temp_directory', desired_value: '/mnt/disks/data/var/run/postgresql/14-main.pg_stat_tmp'} #CHANGE THIS TO THE DISK YOU WANT TO WRITE IT TO - - name: Restart PostgreSQL service service: name: postgresql From 50d058114dc07c01c5774fa77cdf8cd316658a4e Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:32:48 +0200 Subject: [PATCH 71/74] Update ETN-EliasSteyaert-stage.Rmd Updated the ETN for the evaluation --- .../ETN-EliasSteyaert-stage.Rmd | 143 +++++++++++++++--- 1 file changed, 119 insertions(+), 24 deletions(-) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index 4bf10bcc..4cbbd209 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -384,17 +384,15 @@ when: item.name == "uniprot" ##### adding code in irinit - This is done so the data from irimport is written on the disk and not in the home directory (adjusting the postgresql.conf file through ansible so its automated) - - - - - +### Self-assesment of the intermediate evaluation +- There were a lot of negative points, which I agree with. It was a rough start and I didn't have the right problem-solving skills to solve the issues that were on my path. I had to ask a lot of times when I got stuck to being helped out. Most of the times I waited too long to ask for help since I didn't want my question to be embarrasingly easy to solve. ### Trainee documentation plan (TDP) #####Tentative planning: -Which tasks will be performed during the traineeship (briefly)? -The main task of this internship is to go look at all the steps involved in the iRefIndex pipeline. This pipeline is like a big system for gathering and organizing information about how proteins interact with each other. It collects data from different places and puts it all together in one place. By going through each part of this process, we want to improve the automatization of using this dataset. That means fewer mistakes and quicker fixes if something goes wrong. This helps other users to get the protein interaction data they need faster and with fewer errors. +The main task of this internship is to go look at all the steps involved in the iRefIndex pipeline. This pipeline is like a big system for gathering and organizing information about how proteins interact with each other. It collects data from different sources and puts it all together in one place. By going through each part of this process, we want to improve the automatization of using this dataset. That means fewer mistakes and quicker fixes if something goes wrong. This helps other users to get the protein interaction data they need faster and with fewer errors. + +The main goal is to fix the pipeline that is coded with ansible and automate it further so when a person wants to update the iRefIndex after me, it goes way smoother and it shouldn’t run into problems. Normally the environment should be made with Terraform and the Vlaamse Supercomputer, but this didn’t work out as previously thought it would work. The task to create the infrastructure got changed that way. It changed from using the Vlaamse Supercomputer as an instance to using google cloud as the instance and work around with that. ##### Data management: Use the FAIR principle (Table 2) as a guide to store and structure your data. @@ -417,27 +415,124 @@ The code will be stored in my own GitHub repository which is a clone of the fork How will the repository be made available for the internal and external supervisors? The status of the repository is set on public. Everyone can see the code on my GitHub repository. +``` +The Learning Outcome (LO) + +Development goal: + +Understanding the iRefIndex, what it does, what it is used for and why it is an important tool. + +The second goal is to update the iRefIndex, in a way that it is automated for the next time it has to be updated. Not only the iRefIndex must be automated, but the infrastructure that is used to execute the pipeline of building the iRefIndex must be automated aswell. + +Every error that occurs must be handled in a way that nothing gets hardcoded but that the individual steps are coded so that. + +Trying to learn/become better at using ansible and terraform (these are the main tools that are used to automate the pipeline and the infrastructure) so I can automate processes easier. + +Development activity: + +I will be working with ansible playbooks and code from terraform. I will also read about it on the internet to get more insights and information. On my internship I will find errors and parts of it that don’t work properly and fix them so it can be more automated. + +I will recreate and make my own smaller versions of the playbooks and code so that when an error occurs, I can fix it in smaller environments instead of going through the whole playbook. + +Desired results: + +Being able to make a fully running ansible playbook so that other users don’t have to do anything except to run the playbook. + +Being better at finding where the error occurs and at general problem-solving. Also to have more knowledge about iRefIndex, ansible and terraform. + +Schedule: + +These activities will be performed throughout the internship (on daily basis). I will start to use Terraform first to create the infrastructure. While doing this I will see how the infrastructure as code works and how the code defines the infrastructure. + +After that I will be using ansible to build and perfect the pipeline to update the iRefIndex. I will have a lot of problem-solving to do there and see the different techniques to use for the problem-solving itself. + +Necessary support and facilities: + +Normally, all this should be done in the traineeship and its 295 hours. Nothing should be bought for this goal to be achieved. -### The Learning Outcome (LO) +SMART-principle: -##### Development goal: -Trying to learn/become better at using ansible so I can automate processes easier. +Specific: First learning how to create an infrastructure, what it is and why all the different features are important. Learning how to use ansible and make my own playbooks. -##### Development activity: -I will be working with ansible playbooks, find errors and parts of it that don’t work properly and fix them so it can be more automated. +Measurable: At the end of the traineeship, I should be able to make my own working ansible playbook, see how errors occur more rapidly and being able to create a good instance with the correct amount of volume. -##### Desired results: -Being able to make a fully running ansible playbook so that other users don’t have to do anything except to run the playbook. +Achievable: As I’m working daily with ansible for my traineeship, I should be able to achieve this goal. I should be able to accomplish this before the internship ends so I can end the internship with a full product -##### Schedule: -These activities will be performed throughout the internship (on daily basis). +Relevant: In the context of the traineeship, this learning outcome is relevant. But it is also relevant for later as it is a great tool to work with and easy to use since it is with human-readable code and it is always handy to automate as much as possible. -##### Necessary support and facilities: -Normally, all this should be done in the traineeship and its 250 hours. Nothing should be bought for this goal to be achieved. +Time-bound: This should be achieved by the end of the traineeship. But it would be excellent if it is already achieved at the first half of the internship. + + +Result: + +I wasn’t able to update the iRefIndex due to a lot of difficulties and problems. I was able to give insights and better the pipeline till where I ended up. +``` + +### My interpretation of the evaluation excel sheet +``` +The student writes, adjusts or uses code on the command line to process (biological) data, to create or manipulate a database structure, to create a (web) interface, … independent of the programming language +Good "Through reading the code in the scripts, I was able to perform the tasks in the scripts through CLI, for example: +curl ""$CURL_OPT"" ""$RELEASE_URL"" -o ""$DATADIR/$DOWNLOAD"" +curl -L https://ftp.ebi.ac.uk/pub/databases/uniprot/current_release/knowledgebase/complete/ -o /mnt/disks/data/dataext/data19/UniProt" + +The code is written in an elegant way, is not redundant and/or makes use of loop structures and functions where possible +Sufficient +There are some parts that could be left out or more clear, but at the end of the day, every part of the script has a clear name and it is visible what that specific code does + +The student adjusts an analysis pipeline or script in a correct way, with eye for detail, with the goal to apply the pipeline or script within a new research context or on new data +Good +The pipeline to build iRefIndex was analysed and adjusted where it had to be without creating different problems + +The student is able to work with files containing data, to read data and write data to new files, to filter and/or transform data, … and to check the data for correctness +Good +The basics to work with files are met. Checking the data for correctness was for example done at the end to check if everything was parsed alright + +The student can deposit data in a (relational) database system and/or can query a database system, whether or not by using a programming language +Limited +The howto for this is very limited, I can do the basics but answering questions about this is difficult + +The student creates a (relational) database system and/or creates a directory structure to store and organize (biological) data to be able to efficiently select data from the database/directory structure +Good +The repository looks clean and is well organised without things that shouldn't be there + +The student makes use of (dynamic) programming to develop an interface to visualize, analyse or store (biological) data, independent of the programming language +NA + +The student looks independently for new possibilities to program more efficiently, to store, organize or process data, to analyze data, ... and is able to apply the new techniques, with or without results +Limited +There are a lot of new things being tried out, but these are never of a high level (and mostly don't work so something else has to be tried) + +The student learns new programming and data processing techniques to store select, organize or analyze biological data in a more efficient way +Sufficient +I work with postgresql but my knowledge to operate this is not super advanced, I can follow the steps to use this and after asking help I can change the config file + +The student knows and spontaneously applies the rules to protect non-public data +NA + +The student searches for and compares various programming and data processing techniques and has insight into the advantages and disadvantages of the different techniques +Sufficient +I know about the different tools to use but I don't really know about the differences between and which one has a bigger advantage over the other, I just know the positive sides of the tools, not the downsides + +The student himself can select the appropriate programming language or software to process data on the basis of advantages and disadvantages +Sufficient +I know about the different tools to use but I don't really know about the differences between and which one has a bigger advantage over the other, I just know the positive sides of the tools, not the downsides + +The student combines programming skills with computer science knowledge and bioinformatics techniques to solve a (biological) research question, but the focus of the project remains on the bioinformatics aspect +Sufficient +The project is IT heavy combined with bioinformatics, I could use my bio-informatics part of the education well and the linux skills + +The student can use his knowledge from multiple disciplines to summarize results in a scientifically correct way and to make a conclusion for the project +NA + +The student organizes his code, information and documents in an orderly manner, uses comments to document code, keeps track of results in an (electronic) lab notebook and/or uses version control (e.g. GIT) +Excellent +My ETN is uptodate, I pushed my commits and the commits are clear of what happened when pushed. Everything is organised. + +The student works fluently with colleagues, helps or asks for help wherever possible or needed, does more than just the bare minimum, … +Excellent +I don't ask the same question twice and I ask for help when needed +``` -##### SMART-principle: -- Specific: learning how to use ansible and make my own playbooks. -- Measurable: At the end of the traineeship I should be able to make my own working ansible playbook. -- Achievable: As I’m working daily with ansible for my traineeship, I should be able to achieve this goal. -- Relevant: In the context of the traineeship, this learning outcome is relevant. -- Time-bound: This should be achieved by the end of the traineeship. +### Internal aspects +- We never really got introduced to the other collegues, as their desks were 2 floors above us. We could rely on James Collier (our second supervisor) at all times, Mr. Collier worked from home but we were able to communicate through chat or teams meetings very easily. +- We saw Alexander Botzki (our main supervisor) on average once a day, except when he was abroad for congresses which was in total for 10 days. He almost never had a lot of time but was always happy to help us or give hints to proceed on the project From 894d7bc62ccc8eb425b61be11ea8a9355bb7e944 Mon Sep 17 00:00:00 2001 From: EliasSteyaert <151785444+EliasSteyaert@users.noreply.github.com> Date: Fri, 14 Jun 2024 09:48:30 +0200 Subject: [PATCH 72/74] Update ETN-EliasSteyaert-stage.Rmd updating the planning --- .../ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd index 4cbbd209..ad0b122c 100644 --- a/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd +++ b/internships/ETN-Elias-howest/ETN-EliasSteyaert-stage.Rmd @@ -536,3 +536,13 @@ I don't ask the same question twice and I ask for help when needed ### Internal aspects - We never really got introduced to the other collegues, as their desks were 2 floors above us. We could rely on James Collier (our second supervisor) at all times, Mr. Collier worked from home but we were able to communicate through chat or teams meetings very easily. - We saw Alexander Botzki (our main supervisor) on average once a day, except when he was abroad for congresses which was in total for 10 days. He almost never had a lot of time but was always happy to help us or give hints to proceed on the project + +### Planning +- Week 1: Getting to know the project, getting an VSC account and finding out a way to set up a working VPN +- Week 2: Setting up the infrastructure, running the terraform and running the first ansible playbook (main1) +- Week 3: Getting the first playbook fully ran and starting the second playbook (main2) +- Week 4: working on the second playbook to make it work +- Week 5: getting the second playbook fully ran +- Week 6: starting over and running the first playbook again. Using a different approach +- Week 7: Monitoring if the downloads are OK, if so, starting the parsing +- Week 8: Expanding the disk and trying to import the data with the third playbook (main3) From 668f7f115776dda143fc60c140e3f5a13523ded4 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Fri, 14 Jun 2024 16:42:48 +0200 Subject: [PATCH 73/74] pushing the final versions of the playbooks --- vsc/ansible/irdownload.yml | 85 +++++++++++++++++------------------ vsc/ansible/irimport.yml | 4 +- vsc/ansible/irparse.yml | 42 +++++------------ vsc/ansible/irunpack.yml | 4 -- vsc/ansible/vars/sources.yml | 18 +++++--- vsc/ansible/vars/sources2.yml | 15 ++++++- vsc/ansible/vars/sources3.yml | 19 ++------ 7 files changed, 86 insertions(+), 101 deletions(-) diff --git a/vsc/ansible/irdownload.yml b/vsc/ansible/irdownload.yml index d9cce9ff..1b48f547 100644 --- a/vsc/ansible/irdownload.yml +++ b/vsc/ansible/irdownload.yml @@ -11,63 +11,62 @@ log_location: "{{ logs_dir }}/irdownload" tasks: -#ALL UNDERNEETH IS COMMENTED BECAUSE IT TAKES TO LONG, IT WORKS AND NEEDS TO BE UNCOMMENTED - #- name: Update apt packages - # ansible.builtin.apt: - # update_cache: true -# - #- name: Install required packages - # ansible.builtin.apt: - # name: "{{ item }}" - # state: present - # loop: - # - apt-transport-https - # - ca-certificates - # - gnupg - # - curl - # - sudo -# - #- name: Add Google Cloud SDK repository - # ansible.builtin.lineinfile: - # dest: /etc/apt/sources.list.d/google-cloud-sdk.list - # line: "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main" - # create: true - # mode: "0644" -# - #- name: Download Google Cloud SDK key - # ansible.builtin.get_url: - # url: https://packages.cloud.google.com/apt/doc/apt-key.gpg - # dest: /usr/share/keyrings/cloud.google.asc - # mode: "0644" -# - #- name: Update apt packages with Google Cloud SDK repository - # ansible.builtin.apt: - # update_cache: true - # force_apt_get: true -# - #- name: Install Google Cloud CLI - # ansible.builtin.apt: - # name: google-cloud-cli - # state: present -# + - name: Update apt packages + ansible.builtin.apt: + update_cache: true + + - name: Install required packages + ansible.builtin.apt: + name: "{{ item }}" + state: present + loop: + - apt-transport-https + - ca-certificates + - gnupg + - curl + - sudo + + - name: Add Google Cloud SDK repository + ansible.builtin.lineinfile: + dest: /etc/apt/sources.list.d/google-cloud-sdk.list + line: "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main" + create: true + mode: "0644" + + - name: Download Google Cloud SDK key + ansible.builtin.get_url: + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + dest: /usr/share/keyrings/cloud.google.asc + mode: "0644" + + - name: Update apt packages with Google Cloud SDK repository + ansible.builtin.apt: + update_cache: true + force_apt_get: true + + - name: Install Google Cloud CLI + ansible.builtin.apt: + name: google-cloud-cli + state: present + - name: Run jobs ansible.builtin.command: "./irdownload {{ item.name | upper }}" args: chdir: "/home/irefindex/usr/bin" - async: "{{ item.download_runner_async | default(30) }}" + async: "{{ item.download_runner_async | default(300) }}" changed_when: false loop: "{{ jobs }}" poll: 0 register: result - retries: "{{ item.download_runner_retries | default(18) }}" + retries: "{{ item.download_runner_retries | default(180) }}" - name: Wait for jobs to complete # debugger: always ansible.builtin.async_status: jid: "{{ item.ansible_job_id }}" - delay: "{{ item.item.download_delay | default(5) }}" + delay: "{{ item.item.download_delay | default(50) }}" ignore_errors: true - retries: "{{ item.item.download_retries | default(18) }}" + retries: "{{ item.item.download_retries | default(180) }}" register: job_result until: job_result.finished == true loop: "{{ result.results }}" diff --git a/vsc/ansible/irimport.yml b/vsc/ansible/irimport.yml index 092db8a0..69a95cbf 100644 --- a/vsc/ansible/irimport.yml +++ b/vsc/ansible/irimport.yml @@ -27,8 +27,8 @@ register: job_result until: job_result.finished ignore_errors: true - retries: "{{ item.item.import_retries | default(18) }}" - delay: "{{ item.item.import_delay | default(5) }}" + retries: "{{ item.item.import_retries | default(600) }}" + delay: "{{ item.item.import_delay | default(50) }}" loop: "{{ job_result.results }}" - name: "Ensure that `log_location` exists" diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 37eeab10..4634d9ce 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -5,12 +5,11 @@ become: true vars_files: - ./vars/logs.yml - - ./vars/sources3.yml + - ./vars/sources.yml - ./vars/general.yml vars: log_location: "{{ logs_dir }}/irparse" - USE_PYTHON_INTERPRETER: "{{ python_interpreter | default('/usr/bin/python3') }}" local_source_directory: "/home/guest/internship/stage-VIB-irefindex/usr/bin" # Directory containing the files on your local machine remote_destination_directory: "/home/irefindex/usr/bin" # Destination directory on the remote machine @@ -33,25 +32,7 @@ # src: "{{ local_source_directory }}{{ item }}" # dest: "{{ remote_destination_directory }}/{{ item }}" # mode: 'u+x,g+x,o+x' - # with_items: "{{ local_files_list.stdout_lines }}" - - - name: Ensure python3-venv and python3-pip are installed - ansible.builtin.apt: - name: - - python3-venv - - python3-pip - state: present - - - name: Create virtual environment - ansible.builtin.command: - cmd: python3 -m venv "{{ download_data }}/{{ data2 }}/virtualenv" - args: - creates: "{{ download_data }}/{{ data2 }}/virtualenv" - - - name: install irdata - ansible.builtin.command: - cmd: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python3 -m pip install irdata" - + # with_items: "{{ local_files_list.stdout_lines }}" #THIS DOESN'T WORK YET BUT IS NEEDED FOR REPRODUCIBILTY # - name: Find files containing "TOOLS" @@ -70,7 +51,6 @@ # executable: /bin/bash # when: found_files.matched > 0 - # NEEDED FOR REPRODUCIBILTY, WORKS BUT TAKES A WHILE SO ITS COMMENTED # - name: Find .sh and .py files # find: @@ -84,13 +64,18 @@ # mode: u+x,g+x,o+x # loop: "{{ files_to_update.files }}" + - name: Ensure pip is installed + package: + name: python3-pip + state: present + + - name: Install irdata using pip with --break-system-packages + command: pip3 install --break-system-packages irdata + - name: Parse downloads for each source ansible.builtin.command: "./irparse {{ item.name | upper }}" - #ansible.builtin.command: "./echoscript {{ item.name | upper }}" args: chdir: "/home/irefindex/usr/bin" - environment: - python_interpreter: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python3" changed_when: false poll: 0 async: "{{ item.parse_async | default(60) }}" @@ -100,15 +85,12 @@ - name: Wait for all jobs to complete ansible.builtin.async_status: jid: "{{ item.ansible_job_id }}" - environment: - python_interpreter: "{{ download_data }}/{{ data2 }}/virtualenv/bin/python3" register: job_result until: job_result.finished ignore_errors: true - retries: "{{ item.item.parse_retries | default(12) }}" - delay: "{{ item.item.parse_delay | default(5) }}" + retries: "{{ item.item.parse_retries | default(120) }}" + delay: "{{ item.item.parse_delay | default(50) }}" loop: "{{ job_result.results }}" - become: yes - name: Debug job_result ansible.builtin.debug: diff --git a/vsc/ansible/irunpack.yml b/vsc/ansible/irunpack.yml index b363ca43..89a829ca 100644 --- a/vsc/ansible/irunpack.yml +++ b/vsc/ansible/irunpack.yml @@ -55,10 +55,6 @@ no_log: true loop: "{{ all_directories.files }}" - - name: wtf as in existing dirs - ansible.builtin.debug: - msg: "existing_dirs {{ existing_dirs is defined }}" - - name: Gather all directory contents ansible.builtin.find: paths: "{{ item.path }}" diff --git a/vsc/ansible/vars/sources.yml b/vsc/ansible/vars/sources.yml index c6e150a6..5cac2087 100644 --- a/vsc/ansible/vars/sources.yml +++ b/vsc/ansible/vars/sources.yml @@ -41,17 +41,19 @@ # import_delay[default=5]: Delay in seconds between the checks jobs: - { name: "athaliana" } - # - { name: "bar" } + # - { name: "bar" } # UNABLE TO DOWNLOAD SO UNABLE TO PARSE - { name: "bind" } - { name: "dig", import_async: 300, import_delay: 30, import_retries: 10 } - { name: "fly" } - { name: "innatedb" } - # - { name: "ipi" } + # - { name: "ipi" } # UNABLE TO DOWNLOAD SO UNABLE TO PARSE - { name: "mppi", import_async: 160, import_delay: 10, import_retries: 16 } - { name: "psi_mi", import_async: 160, import_delay: 10, import_retries: 16 } - { name: "quickgo", parse_async: 120, parse_delay: 10, import_async: 160, import_delay: 10, import_retries: 16 } - { name: "hpidb", parse_async: 300, parse_delay: 25, import_async: 160, import_retries: 16, import_delay: 10 } - { name: "hprd", parse_async: 400, parse_delay: 30, parse_retries: 14, import_async: 300, import_delay: 30, import_retries: 10 } + # "unassigned1513 is not a pubmedID or DOI: citation suppressed", is an example of an error, mostly got parsed but there are a few unassigned data + - { name: "mpact", parse_async: 120, parse_delay: 10, import_async: 160, import_delay: 10, import_retries: 16 } - { name: "mpidb", parse_async: 400, parse_delay: 30, parse_retries: 14, import_async: 160, import_delay: 10, import_retries: 16 } - { name: "intact", unpack_async: 180, unpack_delay: 30, parse_async: 6000, parse_delay: 300, parse_retries: 20, import_async: 6000, import_retries: 20, import_delay: 300 } @@ -66,14 +68,18 @@ jobs: - { name: "yeast", download_runner_async: 90, download_retries: 54 } - { name: "intcomplex", unpack_async: 60, unpack_delay: 10, parse_async: 350, parse_delay: 30, parse_retries: 12, import_async: 400, import_delay: 30, import_retries: 14 } - { name: "huri", download_runner_async: 60, download_retries: 36, parse_async: 240, parse_retries: 16, parse_delay: 15, import_async: 400, import_delay: 30, import_retries: 14 } + # After parsing: irparse-source: Data processing for source HURI failed.non-zero return; everything is parsed though + - { name: "biogrid", unpack_async: 120, unpack_retries: 8, unpack_delay: 15, download_runner_async: 60, download_retries: 12, download_delay: 15, parse_async: 4400, parse_retries: 15, parse_delay: 300, import_async: 5000, import_retries: 17, import_delay: 300 } - # - { name: "corum", download_runner_async: 60, download_retries: 12, download_delay: 15 } - # - { name: "matrixdb", download_runner_async: 300, download_retries: 30, download_delay: 30, parse_async: 200, parse_retries: 8, parse_delay: 25, import_async: 160, import_delay: 10, import_retries: 16 } + # - { name: "corum", download_runner_async: 60, download_retries: 12, download_delay: 15 } # UNABLE TO DOWNLOAD SO UNABLE TO PARSE + # - { name: "matrixdb", download_runner_async: 300, download_retries: 30, download_delay: 30, parse_async: 200, parse_retries: 8, parse_delay: 25, import_async: 160, import_delay: 10, import_retries: 16 } # UNABLE TO DOWNLOAD SO UNABLE TO PARSE - { name: "uniprotpp", download_runner_async: 300, download_retries: 30, download_delay: 30, parse_async: 120, import_async: 160, import_delay: 10, import_retries: 16 } - - { name: "uniprot", download_runner_async: 2000, download_retries: 60, download_delay: 100, parse_async: 36000, parse_retries: 60, parse_delay: 600 } + - { name: "uniprot", download_runner_async: 20000, download_retries: 600, download_delay: 100, parse_async: 36000, parse_retries: 600, parse_delay: 600 } - { name: "mint", download_runner_async: 600, download_retries: 60, download_delay: 30, parse_async: 120, import_async: 400, import_delay: 30, import_retries: 14 } - { name: "gene", download_runner_async: 800, download_retries: 40, download_delay: 60, parse_async: 1600, parse_retries: 27, parse_delay: 60, import_async: 4000, import_retries: 34, import_delay: 120 } - { name: "reactome", download_runner_async: 300, download_retries: 30, download_delay: 30, import_async: 300, import_retries: 10, import_delay: 30 } - { name: "imex", download_runner_async: 10800, download_retries: 108, download_delay: 300, parse_async: 600, parse_retries: 10, parse_delay: 60, import_async: 1500, import_retries: 25, import_delay: 60 } + # "unassigned1513 is not a pubmedID or DOI: citation suppressed", is an example of this type of error, mostly got parsed but there are a few unassigned data + - { name: "refseq", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 450, parse_retries: 15, parse_delay: 30, import_async: 5000, import_retries: 21, import_delay: 240 } - - { name: "genpept", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 21000, parse_retries: 60, parse_delay: 350, import_async: 5000, import_retries: 21, import_delay: 240 } + - { name: "genpept", download_runner_async: 36000, download_retries: 600, download_delay: 1800, parse_async: 21000, parse_retries: 600, parse_delay: 350, import_async: 5000, import_retries: 21, import_delay: 240 } diff --git a/vsc/ansible/vars/sources2.yml b/vsc/ansible/vars/sources2.yml index e91d97bc..6a1ba928 100644 --- a/vsc/ansible/vars/sources2.yml +++ b/vsc/ansible/vars/sources2.yml @@ -1,4 +1,17 @@ jobs: # - { name: "mint", download_runner_async: 600, download_retries: 1, download_delay: 30, parse_async: 120, import_async: 400, import_delay: 30, import_retries: 14 } # - { name: "bar" } - - { name: "gene", download_runner_async: 800, download_retries: 40, download_delay: 60, parse_async: 1600, parse_retries: 27, parse_delay: 60, import_async: 4000, import_retries: 34, import_delay: 120 } + # - { name: "gene", download_runner_async: 800, download_retries: 40, download_delay: 60, parse_async: 1600, parse_retries: 27, parse_delay: 60, import_async: 4000, import_retries: 34, import_delay: 120 } + # - { name: "athaliana" } + # - { name: "ipi" } + # - { name: "uniprot", download_runner_async: 2000, download_retries: 60, download_delay: 100, parse_async: 36000, parse_retries: 60, parse_delay: 600 } + # - { name: "mint", download_runner_async: 600, download_retries: 15, download_delay: 30, parse_async: 120, import_async: 400, import_delay: 30, import_retries: 14 } + # - { name: "gene", download_runner_async: 800, download_retries: 20, download_delay: 60, parse_async: 1600, parse_retries: 27, parse_delay: 60, import_async: 4000, import_retries: 34, import_delay: 120 } + # - { name: "reactome", download_runner_async: 300, download_retries: 15, download_delay: 30, import_async: 300, import_retries: 10, import_delay: 30 } + # - { name: "imex", download_runner_async: 10800, download_retries: 15, download_delay: 300, parse_async: 600, parse_retries: 10, parse_delay: 60, import_async: 1500, import_retries: 25, import_delay: 60 } + # - { name: "refseq", download_runner_async: 36000, download_retries: 30, download_delay: 1800, parse_async: 450, parse_retries: 15, parse_delay: 30, import_async: 5000, import_retries: 21, import_delay: 240 } + # - { name: "genpept", download_runner_async: 36000, download_retries: 30, download_delay: 1800, parse_async: 21000, parse_retries: 60, parse_delay: 350, import_async: 5000, import_retries: 21, import_delay: 240 } + # - { name: "huri", download_runner_async: 60, download_retries: 36, parse_async: 240, parse_retries: 16, parse_delay: 15, import_async: 400, import_delay: 30, import_retries: 14 } + - { name: "intact", unpack_async: 180, unpack_delay: 30, parse_async: 60000, parse_delay: 300, parse_retries: 200, import_async: 6000, import_retries: 20, import_delay: 300 } + + \ No newline at end of file diff --git a/vsc/ansible/vars/sources3.yml b/vsc/ansible/vars/sources3.yml index dbc181c6..89ee673c 100644 --- a/vsc/ansible/vars/sources3.yml +++ b/vsc/ansible/vars/sources3.yml @@ -16,7 +16,7 @@ jobs: #/data/irdata/logs/20240523T081306290855/irparse - + # - { name: "athaliana" } # - { name: "hprd", parse_async: 400, parse_delay: 30, parse_retries: 14, import_async: 300, import_delay: 30, import_retries: 10 } # BIG ERROR FOR THIS ONE but looks like everything is in the files ?! # " unassigned701 is not a pubmedID or DOI: citation suppressed" @@ -35,15 +35,8 @@ jobs: # - { name: "huri", download_runner_async: 60, download_retries: 36, parse_async: 240, parse_retries: 16, parse_delay: 15, import_async: 400, import_delay: 30, import_retries: 14 } #/data/irdata/logs/20240523T070911706214/irparse # irparse-source: Data processing for source HURI failed.non-zero return - - # - { name: "uniprot", download_runner_async: 2000, download_retries: 60, download_delay: 100, parse_async: 36000, parse_retries: 60, parse_delay: 600 } - # /data/irdata/logs/20240523T082021716989/irparse/uniprot.* - # irparallel: Please specify a command to run on each file (to be given as {}). - # irdata_parse_uniprot.sh: Merging data files for uniprot_sprot... - # cat: '/mnt/disks/data/dataext/irdata19/import/UniProt/uniprot_sprot_accessions-*.txt': such file or directory - # irdata_parse_uniprot.sh: Failed to concatenate data files for uniprot_sprot_accessions. - # irparse-source: Data processing for source UNIPROT failed.non-zero return code + # - { name: "uniprot", download_runner_async: 2000000, download_retries: 600, download_delay: 300, parse_async: 360000, parse_retries: 600, parse_delay: 600 } # - { name: "imex", download_runner_async: 10800, download_retries: 108, download_delay: 300, parse_async: 600, parse_retries: 10, parse_delay: 60, import_async: 1500, import_retries: 25, import_delay: 60 } # " unassigned1513 is not a pubmedID or DOI: citation suppressed" @@ -53,16 +46,12 @@ jobs: # " unassigned2293 is not a pubmedID or DOI: citation suppressed", # /root/.ansible_async/j832151890912.1559858 - # - { name: "refseq", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 450, parse_retries: 5, parse_delay: 30, import_async: 5000, import_retries: 21, import_delay: 240 } + - { name: "refseq", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 36000, parse_retries: 600, parse_delay: 600, import_async: 5000, import_retries: 21, import_delay: 240 } # /import/RefSeq is empty after parsing, BIG ERROR BUT NO LOG FILES # "sh: 1: Syntax error: Unterminated quoted string" # "cat: '/mnt/disks/data/dataext/irdata19/import/RefSeq/*_proteins': No such file or directory", # "cat: '/mnt/disks/data/dataext/irdata19/import/RefSeq/*_identifiers': No such file or directory" - - { name: "genpept", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 21000, parse_retries: 60, parse_delay: 350, import_async: 5000, import_retries: 21, import_delay: 240 } - # EMPTY AFTER PARSING - # "sh: 1: Syntax error: Unterminated quoted string", - # "cat: '/mnt/disks/data/dataext/irdata19/import/GenPept/*_proteins.txt': No such file or directory" - + # - { name: "genpept", download_runner_async: 36000, download_retries: 60, download_delay: 1800, parse_async: 21000, parse_retries: 60, parse_delay: 350, import_async: 5000, import_retries: 21, import_delay: 240 } # - { name: "gene", download_runner_async: 800, download_retries: 40, download_delay: 60, parse_async: 1600, parse_retries: 27, parse_delay: 60, import_async: 4000, import_retries: 34, import_delay: 120 } \ No newline at end of file From ed76b1fbdabed2cc79a2b79c4c864366654e32f7 Mon Sep 17 00:00:00 2001 From: Elias Steyaert Date: Wed, 19 Jun 2024 21:14:10 +0200 Subject: [PATCH 74/74] deleting commented lines --- vsc/ansible/irparse.yml | 50 ----------------------------------------- 1 file changed, 50 deletions(-) diff --git a/vsc/ansible/irparse.yml b/vsc/ansible/irparse.yml index 4634d9ce..0c7cf69d 100644 --- a/vsc/ansible/irparse.yml +++ b/vsc/ansible/irparse.yml @@ -14,56 +14,6 @@ remote_destination_directory: "/home/irefindex/usr/bin" # Destination directory on the remote machine tasks: -#COMMENTED BECAUSE DOING THIS ONCE IS ENOUGH - #- name: Set permissions on Ansible temporary directory - # ansible.builtin.file: - # path: "{{ home_dir }}/.ansible/tmp/" - # state: directory - # mode: "0755" - # recurse: yes - # become: yes - - #- name: Get list of files in local source directory - # shell: ls -1 {{ local_source_directory }} - # register: local_files_list - - #- name: Copy files to remote destination directory - # ansible.builtin.copy: - # src: "{{ local_source_directory }}{{ item }}" - # dest: "{{ remote_destination_directory }}/{{ item }}" - # mode: 'u+x,g+x,o+x' - # with_items: "{{ local_files_list.stdout_lines }}" - -#THIS DOESN'T WORK YET BUT IS NEEDED FOR REPRODUCIBILTY - # - name: Find files containing "TOOLS" - # find: - # paths: /home/irefindex/usr/bin/ - # patterns: "*" - # recurse: yes - # register: found_files -#SAME AS THE COMMENT ABOVE, THESE BELONG TOGETHER - # - name: Iterate over found files - # vars: - # use_python_interpreter: "{{ USE_PYTHON_INTERPRETER }}" - # ansible.builtin.shell: | - # for filename in {{ found_files.files | map(attribute='path') | join(' ') }}; do sed "s|\(\$TOOLS/.*.py\)|{{ use_python_interpreter }}\" \"\1|g" "{{ filename }}" > "{{ filename }}.tmp" && mv "{{ filename }}.tmp" "{{ filename }}"; done - # args: - # executable: /bin/bash - # when: found_files.matched > 0 - -# NEEDED FOR REPRODUCIBILTY, WORKS BUT TAKES A WHILE SO ITS COMMENTED - # - name: Find .sh and .py files - # find: - # paths: /home/irefindex/usr/bin - # patterns: "*.sh,*.py" - # register: files_to_update -# SAME AS THE COMMENT ABOVE, THESE BELONG TOGETHER - # - name: Give execute permissions to .sh and .py files - # file: - # path: "{{ item.path }}" - # mode: u+x,g+x,o+x - # loop: "{{ files_to_update.files }}" - - name: Ensure pip is installed package: name: python3-pip