#925: crontab for submitting the submit_pending_jobs job

cluster_provisioning/modules/common/grq_factotum_metrics.tf

@@ -388,4 +388,47 @@ resource "aws_instance" "factotum" {
     EOT
     ]
   }
+}
+
+resource "null_resource" "setup_cron_factotum" {
+  depends_on = [aws_instance.factotum]
+
+  connection {
+    type        = "ssh"
+    host        = aws_instance.factotum.private_ip
+    user        = "hysdsops"
+    private_key = file(var.private_key_file)
+  }
+
+  provisioner "remote-exec" {
+    inline = [<<-EOT
+      while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 5; done
+      source ~/.bash_profile
+      set -ex
+
+      pwd
+      mkdir -p cron
+      mkdir -p .local/bin/cron
+    EOT
+    ]
+  }
+
+  provisioner "file" {
+    source      = "${path.module}/../../../conf/sds/files/factotum/cron/" # NOTE trailing slash to upload dir contents
+    destination = "cron"
+  }
+
+  provisioner "remote-exec" {
+    inline = [<<-EOT
+      while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 5; done
+      source ~/.bash_profile
+      set -ex
+
+      chmod +x ~/cron/submit_job.py
+      mv ~/cron/submit_job.py ~/.local/bin/cron/
+
+      crontab ~/cron/hysdsops
+    EOT
+    ]
+  }
 }

conf/sds/files/factotum/cron/hysdsops

@@ -0,0 +1,14 @@
+SHELL=/bin/bash
+
+
+#######################################################################
+# hysdsops user crontab
+#
+# log output to system log using `logger`. On Oracle Linux 8, logs are
+#  written to /var/log/messages (on other systems: /var/log/syslog).
+#  System logging configuration at /etc/rsyslog.conf.
+#
+#
+#######################################################################
+
+#@hourly $HOME/.local/bin/cron/submit_job.py 2>&1 | logger -i -p user.notice -t submit_pending_jobs-task

conf/sds/files/factotum/cron/submit_job.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+import uuid
+import sys
+import hashlib
+import re
+import yaml
+from hysds_commons.job_utils import submit_mozart_job
+
+# have yaml parse regular expressions
+yaml.SafeLoader.add_constructor(
+    u"tag:yaml.org,2002:python/regexp", lambda l, n: re.compile(l.construct_scalar(n))
+)
+
+settings = yaml.safe_load(open("/export/home/hysdsops/verdi/etc/settings.yaml"))
+release_version = settings["RELEASE_VERSION"]
+generated_uuid = str(uuid.uuid4())
+payload_hash = hashlib.md5(generated_uuid.encode()).hexdigest()
+
+result = submit_mozart_job(
+        hysdsio={
+            "id": generated_uuid,
+            "params": [],
+            "job-specification": f"job-submit_pending_jobs:{release_version}",
+        },
+        product={},
+        rule={
+            "rule_name": f"trigger-submit_pending_jobs",
+            "queue": "factotum-job_worker-small", #"opera-job_worker-submit_pending_jobs"
+            "priority": "0",
+            "kwargs": "{}",
+            "enable_dedup": True
+        },
+        queue=None,
+        job_name="job-submit_pending_jobs",
+        payload_hash=payload_hash, # we have to provide a unique payload hash so that this doesn't dedup
+        enable_dedup=None,
+        soft_time_limit=None,
+        time_limit=None,
+        component=None
+    )
+
+print(result)