diff --git a/bfabric/bfabric2.py b/bfabric/bfabric2.py index 4b26b503..62cbc7a3 100755 --- a/bfabric/bfabric2.py +++ b/bfabric/bfabric2.py @@ -299,6 +299,28 @@ def delete(self, endpoint: str, id: int | list[int], check: bool = True) -> Resu result.assert_success() return result + def upload_resource( + self, resource_name: str, content: bytes, workunit_id: int, check: bool = True + ) -> ResultContainer: + """Uploads a resource to B-Fabric, only intended for relatively small files that will be tracked by B-Fabric + and not one of the dedicated experimental data stores. + :param resource_name: the name of the resource to create (the same name can only exist once per workunit) + :param content: the content of the resource as bytes + :param workunit_id: the workunit ID to which the resource belongs + :param check: whether to check for errors in the response + """ + content_encoded = base64.b64encode(content).decode() + return self.save( + endpoint="resource", + obj={ + "base64": content_encoded, + "name": resource_name, + "description": "base64 encoded file", + "workunitid": workunit_id, + }, + check=check, + ) + def _read_page(self, readid: bool, endpoint: str, query: dict[str, Any], idonly: bool = False, page: int = 1): """Reads the specified page of objects from the specified endpoint that match the query.""" if readid: diff --git a/bfabric/scripts/bfabric_upload_resource.py b/bfabric/scripts/bfabric_upload_resource.py index fe8c6192..0228dec6 100755 --- a/bfabric/scripts/bfabric_upload_resource.py +++ b/bfabric/scripts/bfabric_upload_resource.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: latin1 -*- - """ Copyright (C) 2017,2020 Functional Genomics Center Zurich ETHZ|UZH. All rights reserved. @@ -12,15 +10,28 @@ this script takes a blob file and a workunit id as input and adds the file as resource to bfabric """ +import argparse +import json +from pathlib import Path + +from bfabric.bfabric2 import Bfabric + + +def bfabric_upload_resource(client: Bfabric, filename: Path, workunit_id: int) -> None: + """Uploads the specified file to the workunit with the name of the file as resource name.""" + result = client.upload_resource(resource_name=filename.name, content=filename.read_bytes(), workunit_id=workunit_id) + print(json.dumps(result.to_list_dict(), indent=2)) + + +def main() -> None: + """Parses the command line arguments and calls `bfabric_upload_resource`.""" + client = Bfabric.from_config(verbose=True) + parser = argparse.ArgumentParser() + parser.add_argument("filename", help="filename", type=Path) + parser.add_argument("workunitid", help="workunitid", type=int) + args = parser.parse_args() + bfabric_upload_resource(client=client, filename=args.filename, workunit_id=args.workunitid) -import sys -import os -from bfabric import Bfabric if __name__ == "__main__": - if len(sys.argv) == 3 and os.path.isfile(sys.argv[1]): - B = Bfabric() - B.print_json(B.upload_file(filename = sys.argv[1], workunitid = int(sys.argv[2]))) - else: - print("usage:\nbfabric_upload_resource.py ") - sys.exit(1) + main() diff --git a/bfabric/scripts/bfabric_upload_submitter_executable.py b/bfabric/scripts/bfabric_upload_submitter_executable.py index f44d9117..8aa6ecad 100755 --- a/bfabric/scripts/bfabric_upload_submitter_executable.py +++ b/bfabric/scripts/bfabric_upload_submitter_executable.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: latin1 -*- - """ Uploader for B-Fabric """ @@ -30,7 +28,7 @@ # # # Example of use: -# +# # For bfabric.__version__ < 0.10.22 # # ./bfabric_upload_submitter_executable.py bfabric_executable_submitter_functionalTest.py gridengine --name "Dummy - yaml / Grid Engine executable" --description "Dummy submitter for the bfabric functional test using Grid Engine." @@ -45,100 +43,101 @@ # ./bfabric_upload_submitter_executable.py bfabric_executable_submitter_functionalTest.py slurm --name "Dummy_-_yaml___Slurm_executable" --description "test new submitter's parameters" # -import os -import sys -import base64 -from bfabric import Bfabric import argparse +import base64 + +import yaml + +from bfabric.bfabric2 import Bfabric -SVN="$HeadURL: http://fgcz-svn.uzh.ch/repos/scripts/trunk/linux/bfabric/apps/python/bfabric/scripts/bfabric_upload_submitter_executable.py $" - -def setup(argv=sys.argv[1:]): - argparser = argparse.ArgumentParser(description="Arguments for new submitter executable.\nFor more details run: ./bfabric_upload_submitter_executable.py --help") - argparser.add_argument('filename', type=str, help="Bash executable of the submitter") - argparser.add_argument('engine', type=str, choices=['slurm', 'gridengine'], help="Valid engines for job handling are: slurm, gridengine") - argparser.add_argument('--name', type=str, help="Name of the submitter", required=False) - argparser.add_argument('--description', type=str, help="Description about the submitter", required=False) - if len(sys.argv) < 3: - argparser.print_help(sys.stderr) - sys.exit(1) - options = argparser.parse_args() - return options - -def main(options): + +def main_upload_submitter_executable(options) -> None: executableFileName = options.filename engine = options.engine - bfapp = Bfabric() + client = Bfabric.from_config(verbose=True) - with open(executableFileName, 'r') as f: + with open(executableFileName) as f: executable = f.read() - attr = { 'context': 'SUBMITTER', - 'parameter': [{'modifiable': 'true', - 'required': 'true', - 'type':'STRING'}, - {'modifiable': 'true', - 'required': 'true', - 'type':'STRING'}, - {'modifiable': 'true', - 'required': 'true', - 'type':'STRING'}], - 'masterexecutableid': 11871, - 'status': 'available', - 'enabled': 'true', - 'valid': 'true', - 'base64': base64.b64encode(executable.encode()).decode() } + attr = { + "context": "SUBMITTER", + "parameter": [ + {"modifiable": "true", "required": "true", "type": "STRING"}, + {"modifiable": "true", "required": "true", "type": "STRING"}, + {"modifiable": "true", "required": "true", "type": "STRING"}, + ], + "masterexecutableid": 11871, + "status": "available", + "enabled": "true", + "valid": "true", + "base64": base64.b64encode(executable.encode()).decode(), + } if engine == "slurm": - attr['name'] = 'yaml / Slurm executable' - attr['parameter'][0]['description'] = 'Which Slurm partition should be used.' - attr['parameter'][0]['enumeration'] = ['prx','maxquant','scaffold','mascot'] - attr['parameter'][0]['key'] = 'partition' - attr['parameter'][0]['label'] = 'partition' - attr['parameter'][0]['value'] = 'prx' - attr['parameter'][1]['description'] = 'Which Slurm nodelist should be used.' - attr['parameter'][1]['enumeration'] = ['fgcz-r-[035,028]','fgcz-r-035','fgcz-r-033','fgcz-r-028','fgcz-r-018'] - attr['parameter'][1]['key'] = 'nodelist' - attr['parameter'][1]['label'] = 'nodelist' - attr['parameter'][1]['value'] = 'fgcz-r-[035,028]' - attr['parameter'][2]['description'] = 'Which Slurm memory should be used.' - attr['parameter'][2]['enumeration'] = ['10G','50G','128G','256G','512G','960G'] - attr['parameter'][2]['key'] = 'memory' - attr['parameter'][2]['label'] = 'memory' - attr['parameter'][2]['value'] = '10G' - attr['version'] = 1.02 - attr['description'] = 'Stage the yaml config file to application using Slurm.' + attr["name"] = "yaml / Slurm executable" + attr["parameter"][0]["description"] = "Which Slurm partition should be used." + attr["parameter"][0]["enumeration"] = ["prx", "maxquant", "scaffold", "mascot"] + attr["parameter"][0]["key"] = "partition" + attr["parameter"][0]["label"] = "partition" + attr["parameter"][0]["value"] = "prx" + attr["parameter"][1]["description"] = "Which Slurm nodelist should be used." + attr["parameter"][1]["enumeration"] = [ + "fgcz-r-[035,028]", + "fgcz-r-035", + "fgcz-r-033", + "fgcz-r-028", + "fgcz-r-018", + ] + attr["parameter"][1]["key"] = "nodelist" + attr["parameter"][1]["label"] = "nodelist" + attr["parameter"][1]["value"] = "fgcz-r-[035,028]" + attr["parameter"][2]["description"] = "Which Slurm memory should be used." + attr["parameter"][2]["enumeration"] = ["10G", "50G", "128G", "256G", "512G", "960G"] + attr["parameter"][2]["key"] = "memory" + attr["parameter"][2]["label"] = "memory" + attr["parameter"][2]["value"] = "10G" + attr["version"] = 1.02 + attr["description"] = "Stage the yaml config file to application using Slurm." elif engine == "gridengine": - attr['name'] = 'yaml / Grid Engine executable' - attr['parameter'][0]['description'] = 'Which Grid Engine partition should be used.' - attr['parameter'][0]['enumeration'] = 'PRX' - attr['parameter'][0]['key'] = 'partition' - attr['parameter'][0]['label'] = 'partition' - attr['parameter'][0]['value'] = 'PRX' - attr['parameter'][1]['description'] = 'Which Grid Engine node should be used.' - attr['parameter'][1]['enumeration'] = ['fgcz-r-033','fgcz-r-028','fgcz-r-018'] - attr['parameter'][1]['key'] = 'nodelist' - attr['parameter'][1]['label'] = 'nodelist' - attr['parameter'][1]['value'] = 'fgcz-r-028' - attr['version'] = 1.00 - attr['description'] = 'Stage the yaml config file to an application using Grid Engine.' + attr["name"] = "yaml / Grid Engine executable" + attr["parameter"][0]["description"] = "Which Grid Engine partition should be used." + attr["parameter"][0]["enumeration"] = "PRX" + attr["parameter"][0]["key"] = "partition" + attr["parameter"][0]["label"] = "partition" + attr["parameter"][0]["value"] = "PRX" + attr["parameter"][1]["description"] = "Which Grid Engine node should be used." + attr["parameter"][1]["enumeration"] = ["fgcz-r-033", "fgcz-r-028", "fgcz-r-018"] + attr["parameter"][1]["key"] = "nodelist" + attr["parameter"][1]["label"] = "nodelist" + attr["parameter"][1]["value"] = "fgcz-r-028" + attr["version"] = 1.00 + attr["description"] = "Stage the yaml config file to an application using Grid Engine." if options.name: - attr['name'] = options.name - else: - pass + attr["name"] = options.name if options.description: - attr['description'] = options.description - else: - pass - - res = bfapp.save_object('executable', attr) - - bfapp.print_yaml(res) + attr["description"] = options.description + + res = client.save("executable", attr) + print(yaml.dump(res)) + + +def main() -> None: + """Parses command line arguments and calls `main_upload_submitter_executable`.""" + parser = argparse.ArgumentParser() + parser.add_argument("filename", type=str, help="Bash executable of the submitter") + parser.add_argument( + "engine", + type=str, + choices=["slurm", "gridengine"], + help="Valid engines for job handling are: slurm, gridengine", + ) + parser.add_argument("--name", type=str, help="Name of the submitter", required=False) + parser.add_argument("--description", type=str, help="Description about the submitter", required=False) + options = parser.parse_args() + main(options) if __name__ == "__main__": - options = setup() - main(options) - + main() diff --git a/bfabric/tests/integration/scripts/test_upload_resource.py b/bfabric/tests/integration/scripts/test_upload_resource.py new file mode 100644 index 00000000..7e99a282 --- /dev/null +++ b/bfabric/tests/integration/scripts/test_upload_resource.py @@ -0,0 +1,78 @@ +import contextlib +import datetime +import hashlib +import json +import unittest +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory + +from bfabric.bfabric2 import Bfabric +from bfabric.scripts.bfabric_upload_resource import bfabric_upload_resource +from bfabric.tests.integration.integration_test_helper import DeleteEntities + + +class TestUploadResource(unittest.TestCase): + def setUp(self): + self.client = Bfabric.from_config(config_env="TEST", verbose=True) + self.delete_results = DeleteEntities(client=self.client, created_entities=[]) + self.addCleanup(self.delete_results) + self.container_id = 3000 + + self.ts = datetime.datetime.now().isoformat() + + def _create_workunit(self): + # create workunit + workunit = self.client.save( + "workunit", {"containerid": self.container_id, "name": f"Testing {self.ts}", "applicationid": 1} + ).to_list_dict()[0] + self.delete_results.created_entities.append(("workunit", workunit["id"])) + return workunit["id"] + + def test_upload_resource(self): + with TemporaryDirectory() as work_dir: + work_dir = Path(work_dir) + file = work_dir / "test.txt" + file.write_text("Hello World!") + + workunit_id = self._create_workunit() + + # upload resource + out_text = StringIO() + with contextlib.redirect_stdout(out_text): + bfabric_upload_resource(client=self.client, filename=file, workunit_id=workunit_id) + resp = json.loads(out_text.getvalue())[0] + + # expected checksum + expected_checksum = hashlib.md5(file.read_bytes()).hexdigest() + + # check resource + resource = self.client.read("resource", {"id": resp["id"]}).to_list_dict()[0] + self.assertEqual(file.name, resource["name"]) + self.assertEqual("base64 encoded file", resource["description"]) + self.assertEqual(expected_checksum, resource["filechecksum"]) + + def test_upload_resource_when_already_exists(self): + with TemporaryDirectory() as work_dir: + work_dir = Path(work_dir) + file = work_dir / "test.txt" + file.write_text("Hello World!") + + workunit_id = self._create_workunit() + + # upload resource + out_text = StringIO() + with contextlib.redirect_stdout(out_text): + bfabric_upload_resource(client=self.client, filename=file, workunit_id=workunit_id) + resp = json.loads(out_text.getvalue())[0] + self.assertEqual(workunit_id, resp["workunit"]["id"]) + + # upload resource again + with self.assertRaises(RuntimeError) as error: + bfabric_upload_resource(client=self.client, filename=file, workunit_id=workunit_id) + + self.assertIn("Resource with the specified attribute combination already exists", str(error.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/bfabric/tests/unit/test_bfabric.py b/bfabric/tests/unit/test_bfabric.py index 58c2db03..937d731e 100644 --- a/bfabric/tests/unit/test_bfabric.py +++ b/bfabric/tests/unit/test_bfabric.py @@ -95,12 +95,12 @@ def test_with_auth_when_exception(self): @patch("bfabric.bfabric2.datetime") def test_add_query_timestamp_when_not_present(self, module_datetime): module_datetime.now.return_value = datetime.datetime(2020, 1, 2, 3, 4, 5) - query = self.mock_bfabric._add_query_timestamp( {"a": "b", "c": 1}) + query = self.mock_bfabric._add_query_timestamp({"a": "b", "c": 1}) self.assertDictEqual( - {"a": "b", "c": 1, 'createdbefore': '2020-01-02T03:04:05'}, + {"a": "b", "c": 1, "createdbefore": "2020-01-02T03:04:05"}, query, ) - module_datetime.now.assert_called_once_with(ZoneInfo('Pacific/Kiritimati')) + module_datetime.now.assert_called_once_with(ZoneInfo("Pacific/Kiritimati")) @patch("bfabric.bfabric2.datetime") def test_add_query_timestamp_when_set_and_past(self, module_datetime): @@ -113,7 +113,7 @@ def test_add_query_timestamp_when_set_and_past(self, module_datetime): {"a": "b", "createdbefore": "2019-12-31T23:59:59"}, query, ) - module_datetime.now.assert_called_once_with(ZoneInfo('Pacific/Kiritimati')) + module_datetime.now.assert_called_once_with(ZoneInfo("Pacific/Kiritimati")) @patch("bfabric.bfabric2.datetime") def test_add_query_timestamp_when_set_and_future(self, module_datetime): @@ -129,6 +129,24 @@ def test_add_query_timestamp_when_set_and_future(self, module_datetime): self.assertEqual(1, len(logs.output)) self.assertIn("Query timestamp is in the future: 2020-01-02 03:04:06", logs.output[0]) + @patch.object(Bfabric, "save") + def test_upload_resource(self, method_save): + resource_name = "hello_world.txt" + content = b"Hello, World!" + workunit_id = 123 + check = MagicMock(name="check") + self.mock_bfabric.upload_resource(resource_name, content, workunit_id, check) + method_save.assert_called_once_with( + endpoint="resource", + obj={ + "base64": "SGVsbG8sIFdvcmxkIQ==", + "workunitid": 123, + "name": "hello_world.txt", + "description": "base64 encoded file", + }, + check=check, + ) + def test_get_version_message(self): self.mock_config.base_url = "dummy_url" message = self.mock_bfabric.get_version_message() @@ -147,7 +165,9 @@ def test_print_version_message(self, method_get_version_message, mock_console): mock_stderr = MagicMock(name="mock_stderr") self.mock_bfabric.print_version_message(stderr=mock_stderr) mock_console.assert_called_once_with(stderr=mock_stderr, highlighter=ANY, theme=ANY) - mock_console.return_value.print.assert_called_once_with(method_get_version_message.return_value, style="bright_yellow") + mock_console.return_value.print.assert_called_once_with( + method_get_version_message.return_value, style="bright_yellow" + ) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index b7ecf1f3..78b1b2ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ Repository = "https://github.com/fgcz/bfabricPy" #bfabric_feeder_resource_autoQC="bfabric.scripts.bfabric_feeder_resource_autoQC:main" #bfabric_list_not_existing_storage_directories="bfabric.scripts.bfabric_list_not_existing_storage_directories:main" "bfabric_list_not_available_proteomics_workunits.py"="bfabric.scripts.bfabric_list_not_available_proteomics_workunits:main" -#bfabric_upload_resource="bfabric.scripts.bfabric_upload_resource:main" +"bfabric_upload_resource.py"="bfabric.scripts.bfabric_upload_resource:main" #bfabric_logthis="bfabric.scripts.bfabric_logthis:main" #bfabric_setResourceStatus_available="bfabric.scripts.bfabric_setResourceStatus_available:main" #bfabric_setExternalJobStatus_done="bfabric.scripts.bfabric_setExternalJobStatus_done:main"