test: added system tests of jobs with input data and parametric with …

…input data

src/DIRAC/tests/Utilities/testJobDefinitions.py

@@ -3,14 +3,13 @@
 
 # pylint: disable=invalid-name
 
+import errno
 import os
 import time
-import errno
 
 from DIRAC import rootPath
-from DIRAC.Interfaces.API.Job import Job
 from DIRAC.Interfaces.API.Dirac import Dirac
-
+from DIRAC.Interfaces.API.Job import Job
 from DIRAC.tests.Utilities.utils import find_all
 
 
@@ -65,6 +64,36 @@ def helloWorld():
     return endOfAllJobs(J)
 
 
+def helloWorld_input():
+    """simple hello world job with input"""
+
+    J = baseToAllJobs("helloWorld_input")
+    try:
+        J.setInputSandbox([find_all("exe-script-with-input.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script-with-input.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox([find_all("exe-script-with-input.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]])
+    J.setExecutable("exe-script-with-input.py", "", "helloWorld.log")
+    return endOfAllJobs(J)
+
+
+def helloWorld_input_single():
+    """simple hello world job with input"""
+
+    J = baseToAllJobs("helloWorld_input_single")
+    try:
+        J.setInputSandbox([find_all("exe-script-with-input-single-location.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script-with-input-single-location.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox([find_all("exe-script-with-input-single-location.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]])
+    J.setExecutable("exe-script-with-input-single-location.py", "", "helloWorld.log")
+    return endOfAllJobs(J)
+
+
 def helloWorldCERN():
     """simple hello world job to CERN"""
 
@@ -262,6 +291,29 @@ def parametricJob():
     return endOfAllJobs(J)
 
 
+def parametricJobInputData():
+    """Creates a parametric job with 3 subjobs which are simple hello world jobs, but with input data"""
+
+    J = baseToAllJobs("parametricJobInput")
+    try:
+        J.setInputSandbox([find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox([find_all("exe-script.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]])
+    J.setParameterSequence("args", ["one", "two", "three"])
+    J.setParameterSequence("iargs", [1, 2, 3])
+    J.setParameterSequence(
+        "InputData",
+        ["/dteam/user/f/fstagni/test/1.txt", "/dteam/user/f/fstagni/test/2.txt", "/dteam/user/f/fstagni/test/3.txt"],
+    )
+    J.setInputDataPolicy("download")
+    J.setExecutable("exe-script.py", arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log")
+    return endOfAllJobs(J)
+
+
+
 def jobWithOutput():
     """Creates a job that uploads an output.
     The output SE is not set here, so it would use the default /Resources/StorageElementGroups/SE-USER

tests/System/exe-script-with-input-single-location.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+###############################################################################
+# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+"""Script to run Executable application"""
+from os import system
+import sys
+
+# Main
+if __name__ == "__main__":
+    sys.exit(int(system("""cat testInputFileSingleLocation.txt""") / 256))

tests/System/exe-script-with-input.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+###############################################################################
+# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+"""Script to run Executable application"""
+import sys
+from os import system
+
+# Main
+if __name__ == "__main__":
+    sys.exit(int(system("""cat testInputFile.txt""") / 256))

tests/System/testInputFile.txt

@@ -0,0 +1 @@
+this is just a test

tests/System/testInputFileSingleLocation.txt

@@ -0,0 +1,3 @@
+this is just a test
+
+This file should be at /dteam/user/f/fstagni/test/testInputFileSingleLocation.txt in RAL-SE

tests/System/unitTestUserJobs.py

@@ -1,8 +1,9 @@
 """ Collection of user jobs for testing purposes
 """
 # pylint: disable=wrong-import-position, invalid-name
-import unittest
+import sys
 import time
+import unittest
 
 import DIRAC
 
@@ -22,7 +23,26 @@ class GridSubmissionTestCase(unittest.TestCase):
     """Base class for the Regression test cases"""
 
     def setUp(self):
-        pass
+        result = getProxyInfo()
+        if result["Value"]["group"] not in ["dteam_user", "gridpp_user"]:
+            print("GET A USER GROUP")
+            sys.exit(1)
+
+        res = DataManager().getReplicas(
+            ["/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt", "/dteam/user/f/fstagni/test/testInputFile.txt"]
+        )
+        if not res["OK"]:
+            print(f"DATAMANAGER.getRepicas failure: {res['Message']}")
+            sys.exit(1)
+        if res["Value"]["Failed"]:
+            print(f"DATAMANAGER.getRepicas failed for something: {res['Value']['Failed']}")
+            sys.exit(1)
+
+        replicas = res["Value"]["Successful"]
+        if list(replicas["/dteam/user/f/fstagni/test/testInputFile.txt"]) != ["RAL-SE", "UKI-LT2-IC-HEP-disk"]:
+            print("/dteam/user/f/fstagni/test/testInputFile.txt locations are not correct")
+        if list(replicas["/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt"]) != ["RAL-SE"]:
+            print("/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt locations are not correct")
 
     def tearDown(self):
         pass
@@ -37,6 +57,14 @@ def test_submit(self):
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
 
+        res = helloWorld_input()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
+        res = helloWorld_input_single()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
         res = helloWorldCERN()
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
@@ -77,6 +105,10 @@ def test_submit(self):
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
 
+        res = parametricJobInputData()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
         res = jobWithOutput()
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])