Refactor upload scripts

bfabric/bfabric2.py

@@ -299,6 +299,28 @@ def delete(self, endpoint: str, id: int | list[int], check: bool = True) -> Resu
             result.assert_success()
         return result
 
+    def upload_resource(
+        self, resource_name: str, content: bytes, workunit_id: int, check: bool = True
+    ) -> ResultContainer:
+        """Uploads a resource to B-Fabric, only intended for relatively small files that will be tracked by B-Fabric
+        and not one of the dedicated experimental data stores.
+        :param resource_name: the name of the resource to create (the same name can only exist once per workunit)
+        :param content: the content of the resource as bytes
+        :param workunit_id: the workunit ID to which the resource belongs
+        :param check: whether to check for errors in the response
+        """
+        content_encoded = base64.b64encode(content).decode()
+        return self.save(
+            endpoint="resource",
+            obj={
+                "base64": content_encoded,
+                "name": resource_name,
+                "description": "base64 encoded file",
+                "workunitid": workunit_id,
+            },
+            check=check,
+        )
+
     def _read_page(self, readid: bool, endpoint: str, query: dict[str, Any], idonly: bool = False, page: int = 1):
         """Reads the specified page of objects from the specified endpoint that match the query."""
         if readid:

bfabric/scripts/bfabric_upload_resource.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: latin1 -*-
-
 """
 
 Copyright (C) 2017,2020 Functional Genomics Center Zurich ETHZ|UZH. All rights reserved.
@@ -12,15 +10,28 @@
 
 this script takes a blob file and a workunit id as input and adds the file as resource to bfabric
 """
+import argparse
+import json
+from pathlib import Path
+
+from bfabric.bfabric2 import Bfabric
+
+
+def bfabric_upload_resource(client: Bfabric, filename: Path, workunit_id: int) -> None:
+    """Uploads the specified file to the workunit with the name of the file as resource name."""
+    result = client.upload_resource(resource_name=filename.name, content=filename.read_bytes(), workunit_id=workunit_id)
+    print(json.dumps(result.to_list_dict(), indent=2))
+
+
+def main() -> None:
+    """Parses the command line arguments and calls `bfabric_upload_resource`."""
+    client = Bfabric.from_config(verbose=True)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("filename", help="filename", type=Path)
+    parser.add_argument("workunitid", help="workunitid", type=int)
+    args = parser.parse_args()
+    bfabric_upload_resource(client=client, filename=args.filename, workunit_id=args.workunitid)
 
-import sys
-import os
-from bfabric import Bfabric
 
 if __name__ == "__main__":
-    if len(sys.argv) == 3 and os.path.isfile(sys.argv[1]):
-        B = Bfabric()
-        B.print_json(B.upload_file(filename = sys.argv[1], workunitid = int(sys.argv[2])))
-    else:
-        print("usage:\nbfabric_upload_resource.py <filename> <workunitid>")
-        sys.exit(1)
+    main()

bfabric/scripts/bfabric_upload_submitter_executable.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: latin1 -*-
-
 """
 Uploader for B-Fabric
 """
@@ -30,7 +28,7 @@
 #
 #
 # Example of use:
-# 
+#
 # For bfabric.__version__ < 0.10.22
 #
 # ./bfabric_upload_submitter_executable.py bfabric_executable_submitter_functionalTest.py gridengine --name "Dummy - yaml / Grid Engine executable" --description "Dummy submitter for the bfabric functional test using Grid Engine."
@@ -45,100 +43,101 @@
 # ./bfabric_upload_submitter_executable.py bfabric_executable_submitter_functionalTest.py slurm --name "Dummy_-_yaml___Slurm_executable" --description "test new submitter's parameters"
 #
 
-import os
-import sys
-import base64
-from bfabric import Bfabric
 import argparse
+import base64
+
+import yaml
+
+from bfabric.bfabric2 import Bfabric
 
-SVN="$HeadURL: http://fgcz-svn.uzh.ch/repos/scripts/trunk/linux/bfabric/apps/python/bfabric/scripts/bfabric_upload_submitter_executable.py $"
-
-def setup(argv=sys.argv[1:]):
-    argparser = argparse.ArgumentParser(description="Arguments for new submitter executable.\nFor more details run: ./bfabric_upload_submitter_executable.py --help") 
-    argparser.add_argument('filename', type=str, help="Bash executable of the submitter")
-    argparser.add_argument('engine', type=str, choices=['slurm', 'gridengine'], help="Valid engines for job handling are: slurm, gridengine")
-    argparser.add_argument('--name', type=str, help="Name of the submitter", required=False)
-    argparser.add_argument('--description', type=str, help="Description about the submitter", required=False)
-    if len(sys.argv) < 3:
-        argparser.print_help(sys.stderr)
-        sys.exit(1)
-    options = argparser.parse_args()
-    return options
-
-def main(options):
+
+def main_upload_submitter_executable(options) -> None:
     executableFileName = options.filename
     engine = options.engine
 
-    bfapp = Bfabric()
+    client = Bfabric.from_config(verbose=True)
 
-    with open(executableFileName, 'r') as f:
+    with open(executableFileName) as f:
         executable = f.read()
 
-    attr = { 'context': 'SUBMITTER', 
-        'parameter': [{'modifiable': 'true', 
-            'required': 'true',
-            'type':'STRING'},
-            {'modifiable': 'true',
-            'required': 'true',
-            'type':'STRING'},
-            {'modifiable': 'true',
-            'required': 'true', 
-            'type':'STRING'}],
-        'masterexecutableid': 11871,
-        'status': 'available',
-        'enabled': 'true',
-        'valid': 'true',
-        'base64': base64.b64encode(executable.encode()).decode() }
+    attr = {
+        "context": "SUBMITTER",
+        "parameter": [
+            {"modifiable": "true", "required": "true", "type": "STRING"},
+            {"modifiable": "true", "required": "true", "type": "STRING"},
+            {"modifiable": "true", "required": "true", "type": "STRING"},
+        ],
+        "masterexecutableid": 11871,
+        "status": "available",
+        "enabled": "true",
+        "valid": "true",
+        "base64": base64.b64encode(executable.encode()).decode(),
+    }
 
     if engine == "slurm":
-        attr['name'] = 'yaml / Slurm executable'
-        attr['parameter'][0]['description'] = 'Which Slurm partition should be used.'
-        attr['parameter'][0]['enumeration'] = ['prx','maxquant','scaffold','mascot']
-        attr['parameter'][0]['key'] = 'partition'
-        attr['parameter'][0]['label'] = 'partition'
-        attr['parameter'][0]['value'] = 'prx'
-        attr['parameter'][1]['description'] = 'Which Slurm nodelist should be used.'
-        attr['parameter'][1]['enumeration'] = ['fgcz-r-[035,028]','fgcz-r-035','fgcz-r-033','fgcz-r-028','fgcz-r-018']
-        attr['parameter'][1]['key'] = 'nodelist'
-        attr['parameter'][1]['label'] = 'nodelist'
-        attr['parameter'][1]['value'] = 'fgcz-r-[035,028]'
-        attr['parameter'][2]['description'] = 'Which Slurm memory should be used.'
-        attr['parameter'][2]['enumeration'] = ['10G','50G','128G','256G','512G','960G']
-        attr['parameter'][2]['key'] = 'memory'
-        attr['parameter'][2]['label'] = 'memory'
-        attr['parameter'][2]['value'] = '10G'
-        attr['version'] = 1.02
-        attr['description'] = 'Stage the yaml config file to application using Slurm.'
+        attr["name"] = "yaml / Slurm executable"
+        attr["parameter"][0]["description"] = "Which Slurm partition should be used."
+        attr["parameter"][0]["enumeration"] = ["prx", "maxquant", "scaffold", "mascot"]
+        attr["parameter"][0]["key"] = "partition"
+        attr["parameter"][0]["label"] = "partition"
+        attr["parameter"][0]["value"] = "prx"
+        attr["parameter"][1]["description"] = "Which Slurm nodelist should be used."
+        attr["parameter"][1]["enumeration"] = [
+            "fgcz-r-[035,028]",
+            "fgcz-r-035",
+            "fgcz-r-033",
+            "fgcz-r-028",
+            "fgcz-r-018",
+        ]
+        attr["parameter"][1]["key"] = "nodelist"
+        attr["parameter"][1]["label"] = "nodelist"
+        attr["parameter"][1]["value"] = "fgcz-r-[035,028]"
+        attr["parameter"][2]["description"] = "Which Slurm memory should be used."
+        attr["parameter"][2]["enumeration"] = ["10G", "50G", "128G", "256G", "512G", "960G"]
+        attr["parameter"][2]["key"] = "memory"
+        attr["parameter"][2]["label"] = "memory"
+        attr["parameter"][2]["value"] = "10G"
+        attr["version"] = 1.02
+        attr["description"] = "Stage the yaml config file to application using Slurm."
     elif engine == "gridengine":
-        attr['name'] = 'yaml /  Grid Engine executable'
-        attr['parameter'][0]['description'] = 'Which Grid Engine partition should be used.'
-        attr['parameter'][0]['enumeration'] = 'PRX'
-        attr['parameter'][0]['key'] = 'partition'
-        attr['parameter'][0]['label'] = 'partition'
-        attr['parameter'][0]['value'] = 'PRX' 
-        attr['parameter'][1]['description'] = 'Which Grid Engine node should be used.'
-        attr['parameter'][1]['enumeration'] = ['fgcz-r-033','fgcz-r-028','fgcz-r-018']
-        attr['parameter'][1]['key'] = 'nodelist'
-        attr['parameter'][1]['label'] = 'nodelist'
-        attr['parameter'][1]['value'] = 'fgcz-r-028' 
-        attr['version'] = 1.00 
-        attr['description'] = 'Stage the yaml config file to an application using Grid Engine.' 
+        attr["name"] = "yaml /  Grid Engine executable"
+        attr["parameter"][0]["description"] = "Which Grid Engine partition should be used."
+        attr["parameter"][0]["enumeration"] = "PRX"
+        attr["parameter"][0]["key"] = "partition"
+        attr["parameter"][0]["label"] = "partition"
+        attr["parameter"][0]["value"] = "PRX"
+        attr["parameter"][1]["description"] = "Which Grid Engine node should be used."
+        attr["parameter"][1]["enumeration"] = ["fgcz-r-033", "fgcz-r-028", "fgcz-r-018"]
+        attr["parameter"][1]["key"] = "nodelist"
+        attr["parameter"][1]["label"] = "nodelist"
+        attr["parameter"][1]["value"] = "fgcz-r-028"
+        attr["version"] = 1.00
+        attr["description"] = "Stage the yaml config file to an application using Grid Engine."
 
     if options.name:
-        attr['name'] = options.name
-    else:
-        pass
+        attr["name"] = options.name
     if options.description:
-        attr['description'] = options.description
-    else:
-        pass
-
-    res = bfapp.save_object('executable', attr)
-
-    bfapp.print_yaml(res)
+        attr["description"] = options.description
+
+    res = client.save("executable", attr)
+    print(yaml.dump(res))
+
+
+def main() -> None:
+    """Parses command line arguments and calls `main_upload_submitter_executable`."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("filename", type=str, help="Bash executable of the submitter")
+    parser.add_argument(
+        "engine",
+        type=str,
+        choices=["slurm", "gridengine"],
+        help="Valid engines for job handling are: slurm, gridengine",
+    )
+    parser.add_argument("--name", type=str, help="Name of the submitter", required=False)
+    parser.add_argument("--description", type=str, help="Description about the submitter", required=False)
+    options = parser.parse_args()
+    main(options)
 
 
 if __name__ == "__main__":
-    options = setup()
-    main(options)
-
+    main()

bfabric/tests/integration/scripts/test_upload_resource.py

@@ -0,0 +1,78 @@
+import contextlib
+import datetime
+import hashlib
+import json
+import unittest
+from io import StringIO
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from bfabric.bfabric2 import Bfabric
+from bfabric.scripts.bfabric_upload_resource import bfabric_upload_resource
+from bfabric.tests.integration.integration_test_helper import DeleteEntities
+
+
+class TestUploadResource(unittest.TestCase):
+    def setUp(self):
+        self.client = Bfabric.from_config(config_env="TEST", verbose=True)
+        self.delete_results = DeleteEntities(client=self.client, created_entities=[])
+        self.addCleanup(self.delete_results)
+        self.container_id = 3000
+
+        self.ts = datetime.datetime.now().isoformat()
+
+    def _create_workunit(self):
+        # create workunit
+        workunit = self.client.save(
+            "workunit", {"containerid": self.container_id, "name": f"Testing {self.ts}", "applicationid": 1}
+        ).to_list_dict()[0]
+        self.delete_results.created_entities.append(("workunit", workunit["id"]))
+        return workunit["id"]
+
+    def test_upload_resource(self):
+        with TemporaryDirectory() as work_dir:
+            work_dir = Path(work_dir)
+            file = work_dir / "test.txt"
+            file.write_text("Hello World!")
+
+            workunit_id = self._create_workunit()
+
+            # upload resource
+            out_text = StringIO()
+            with contextlib.redirect_stdout(out_text):
+                bfabric_upload_resource(client=self.client, filename=file, workunit_id=workunit_id)
+            resp = json.loads(out_text.getvalue())[0]
+
+            # expected checksum
+            expected_checksum = hashlib.md5(file.read_bytes()).hexdigest()
+
+            # check resource
+            resource = self.client.read("resource", {"id": resp["id"]}).to_list_dict()[0]
+            self.assertEqual(file.name, resource["name"])
+            self.assertEqual("base64 encoded file", resource["description"])
+            self.assertEqual(expected_checksum, resource["filechecksum"])
+
+    def test_upload_resource_when_already_exists(self):
+        with TemporaryDirectory() as work_dir:
+            work_dir = Path(work_dir)
+            file = work_dir / "test.txt"
+            file.write_text("Hello World!")
+
+            workunit_id = self._create_workunit()
+
+            # upload resource
+            out_text = StringIO()
+            with contextlib.redirect_stdout(out_text):
+                bfabric_upload_resource(client=self.client, filename=file, workunit_id=workunit_id)
+            resp = json.loads(out_text.getvalue())[0]
+            self.assertEqual(workunit_id, resp["workunit"]["id"])
+
+            # upload resource again
+            with self.assertRaises(RuntimeError) as error:
+                bfabric_upload_resource(client=self.client, filename=file, workunit_id=workunit_id)
+
+            self.assertIn("Resource with the specified attribute combination already exists", str(error.exception))
+
+
+if __name__ == "__main__":
+    unittest.main()