Merge pull request #1 from Illumina/4.3.6-el8

update to 4.3.6

app/source/dragen/Dockerfile

@@ -1,9 +1,11 @@
 # DRAGEN Quickstart Docker image generator --
-FROM public.ecr.aws/docker/library/centos:centos7.9.2009
-RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+FROM public.ecr.aws/docker/library/oraclelinux:8
+RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+
+RUN /usr/bin/crb enable
 
 # Install Basic packages needed for Dragen
-RUN yum -y install \
+RUN dnf -y install \
   perl \
   sos \
   coreutils \
@@ -15,25 +17,22 @@ RUN yum -y install \
   ca-certificates \
   ipmitool \
   smartmontools \
-  rsync && \
+  rsync \
+  tree && \
   yum clean all && \
   mkdir -m777 -p /var/log/dragen /var/run/dragen
 
+
 #########################################################
 # Now install the Edico WFMS "Wrapper" functions
 
 # Install necessary standard packages
 # Note: 'easy_install' used to workaround docker/kernel issue (https://github.com/moby/moby/issues/12327)
-RUN yum -y install \
-    python3-devel \
-    tree && \
-    yum clean all
+RUN dnf install python3.12 python3.12-pip && \
+    dnf clean all
 
 RUN python3 -m pip install --upgrade pip && \
-  python3 -m pip install future && \
-  python3 -m pip install six && \
-  python3 -m pip install requests && \
-  python3 -m pip install boto3
+    python3 -m pip install future six requests boto3
 
 # Install d_haul and dragen_job_execute wrapper functions and associated packages
 RUN mkdir -p /root/quickstart/scheduler && \

app/source/dragen/src/dragen_qs.py

@@ -85,6 +85,23 @@ def exec_cmd(cmd):
     err = p.wait()
     return err
 
+def exec_cmd(cmd, logfile=None):
+	printf("Executing %s" % cmd.strip())
+	p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+	output = ""
+	for line in p.stdout:
+		line = line.decode('utf-8')
+		print(line.strip())
+		output += line
+
+	err = p.wait()
+
+	if logfile:
+		print(logfile)
+		with open(logfile , 'w') as f:
+			f.write(output)
+	return err
 
 #########################################################################################
 # DragenJob - Dragen Job execution object
@@ -375,7 +392,7 @@ def check_board_state(self):
     # exec_download - Download a file from the given URL to the target directory
     #
     def exec_url_download(self, url, target_dir):
-        dl_cmd = '{bin} --mode download --url {url} --path {target}'.format(
+        dl_cmd = '{bin} --mode download --url {url} --path {target} -s'.format(
             bin=self.D_HAUL_UTIL,
             url=url,
             target=target_dir)
@@ -391,7 +408,7 @@ def exec_url_download(self, url, target_dir):
     # download_s3_object: Download an object from S3 bucket/key to spefific target file path
     #
     def download_s3_object(self, bucket, key, target_path):
-        dl_cmd = '{bin} --mode download --bucket {bucket} --key {key} --path {target}'.format(
+        dl_cmd = '{bin} --mode download --bucket {bucket} --key {key} --path {target} -s'.format(
             bin=self.D_HAUL_UTIL,
             bucket=bucket,
             key=key,
@@ -626,7 +643,7 @@ def download_ref_tables(self):
             sys.exit(1)
 
         target_path = self.DEFAULT_DATA_FOLDER  # Specifies the root
-        dl_cmd = '{bin} --mode download --bucket {bucket} --key {key} --path {target}'.format(
+        dl_cmd = '{bin} --mode download --bucket {bucket} --key {key} --path {target} -s'.format(
             bin=self.D_HAUL_UTIL,
             bucket=s3_bucket,
             key=s3_key,
@@ -731,12 +748,12 @@ def run_job(self):
 
         # Save the Dragen output to a file instead of stdout
         output_log_path = self.output_dir + '/' + self.DRAGEN_LOG_FILE_NAME % round(time.time())
-        redirect_cmd = self.REDIRECT_OUTPUT_CMD_SUFFIX % output_log_path
-        dragen_cmd = "%s %s" % (dragen_cmd, redirect_cmd)
+#        redirect_cmd = self.REDIRECT_OUTPUT_CMD_SUFFIX % output_log_path
+#        dragen_cmd = "%s %s" % (dragen_cmd, redirect_cmd)
 
         # Run the Dragen process
-        self.process_start_time = datetime.datetime.utcnow()
-        exit_code = exec_cmd(dragen_cmd)
+        self.process_start_time = datetime.datetime.now(datetime.UTC)
+        exit_code = exec_cmd(dragen_cmd, output_log_path)
 
         # Upload the results to S3 output bucket
         self.upload_job_outputs()
@@ -758,7 +775,7 @@ def run_job(self):
                     signum = -self.global_exit_code
                 printf("Job terminated due to signal %s" % signum)
 
-        self.process_end_time = datetime.datetime.utcnow()
+        self.process_end_time = datetime.datetime.now(datetime.UTC)
         return
 
     ########################################################################################

templates/batch.template.yaml

@@ -252,6 +252,8 @@ Resources:
               echo ECS_NUM_IMAGES_DELETE_PER_CYCLE=5>>/etc/ecs/ecs.config
               echo ECS_RESERVED_MEMORY=32>>/etc/ecs/ecs.config
               echo ECS_AVAILABLE_LOGGING_DRIVERS='["json-file","awslogs"]'>>/etc/ecs/ecs.config
+              qs_retry_command 10 yum downgrade -y docker-ce-25.0.5-1.el8 docker-ce-cli-25.0.5-1.el8 docker-ce-rootless-extras-25.0.5-1.el8
+              qs_retry_command 10 systemctl restart docker
               qs_retry_command 10 docker stop ecs-agent
               qs_retry_command 10 docker rm ecs-agent
               qs_retry_command 10 docker pull amazon/amazon-ecs-agent:latest
@@ -372,6 +374,9 @@ Resources:
           - ContainerPath: "/var/lib/edico"
             ReadOnly: False
             SourceVolume: docker_var_lib_edico
+          - ContainerPath: "/usr/lib64/libdragen.so.4.3.6"
+            ReadOnly: False
+            SourceVolume: docker_usr_lib64
         Volumes:
           - Name: docker_scratch
             Host:
@@ -385,6 +390,9 @@ Resources:
           - Name: docker_var_lib_edico
             Host:
               SourcePath: "/var/lib/edico"
+          - Name: docker_usr_lib64
+            Host:
+              SourcePath: "/usr/lib64/libdragen.so.4.3.6"
 
 Outputs:
   DragenComputeEnvironmentSpot:

templates/clean-bucket.template.yaml

@@ -22,7 +22,7 @@ Resources:
     Properties:
       Description: Removes non-versioned files from an S3 Bucket.
       Handler: index.handler
-      Runtime: python3.7
+      Runtime: python3.12
       Role: !Ref ServiceToken
       Timeout: 240
       Code:

templates/clean-repository.template.yaml

@@ -22,7 +22,7 @@ Resources:
     Properties:
       Description: Removes an ECR Repository.
       Handler: index.handler
-      Runtime: python3.7
+      Runtime: python3.12
       Role: !Ref ServiceToken
       Timeout: 240
       Code:

templates/copy.template.yaml

@@ -115,7 +115,7 @@ Resources:
     Properties:
       Description: Copies objects from a source S3 bucket to a destination
       Handler: index.handler
-      Runtime: python3.7
+      Runtime: python3.12
       Role: !GetAtt CopyRole.Arn
       Timeout: 240
       Code:

templates/dragen.template.yaml

@@ -78,15 +78,15 @@ Mappings:
     AMI:
       DRAGEN: dragen-completesuite-*
     us-east-1:
-      DRAGEN: ami-031e74a44cf78bbe0
+      DRAGEN: ami-03cbd53b0e88e058e
     us-west-2:
-      DRAGEN: ami-027d691833e1afb2d
+      DRAGEN: ami-044299ba1b39c5c13
     eu-central-1:
-      DRAGEN: ami-070807320b35acd1d
+      DRAGEN: ami-097ee7e6e7b6bbc0d
     eu-west-1:
-      DRAGEN: ami-0a3b1dcda746bd01d
+      DRAGEN: ami-03fa4250b37330bd1
     ap-southeast-2:
-      DRAGEN: ami-058a6e883edcdf5ee
+      DRAGEN: ami-01df63237b437a659
 
 Parameters:
   InstanceType: