Devops 1593 fix

.gitignore

@@ -0,0 +1 @@
+.idea

deployment/bin/cron/container_reaper.py

@@ -8,7 +8,6 @@
 # SLACK_WEBHOOK_URL - The slack webhook url to send messages to
 """
 
-import json
 import os
 import socket
 import subprocess
@@ -17,21 +16,9 @@
 from typing import Set
 
 import docker
-import requests
 from docker.models.containers import Container
 
-
-def send_slack_message(message: str):
-    """
-    :param message: Escaped Message to send to slack
-    """
-    webhook_url = os.environ.get("SLACK_WEBHOOK_URL", None)
-    slack_data = {"text": message}
-    requests.post(
-        webhook_url,
-        data=json.dumps(slack_data),
-        headers={"Content-Type": "application/json"},
-    )
+from .send_slack_message import notify_kbase_slack
 
 
 def filter_containers_by_time(potential_containers, days=0, minutes=0):
@@ -74,7 +61,7 @@ def reap_containers_running_more_than_7_days(potential_containers: Set[Container
     if old_containers:
         for old_container in old_containers:
             message = get_running_time_message(old_container, title="reaper7daylimit")
-            send_slack_message(message)
+            notify_kbase_slack(message)
             remove_with_backoff(old_container, message)
 
 
@@ -91,7 +78,7 @@ def reap_containers_when_there_is_no_starter(potential_containers: Set[Container
     if runaway_containers:
         for runaway_container in runaway_containers:
             message = get_running_time_message(runaway_container, title="reaper_no_starter")
-            send_slack_message(message)
+            notify_kbase_slack(message)
             remove_with_backoff(container,message)
 
 

deployment/bin/cron/delete_exited_containers.py

@@ -1,26 +1,12 @@
 #!/miniconda/bin/python
 # This script is automatically run by the condor cronjob periodically
 # in order to clean up exited docker containers.
-import json
 import os
 import socket
 
 import docker
-import requests
-
-
-def send_slack_message(message: str):
-    """
-    :param message: Escaped Message to send to slack
-    """
-    webhook_url = os.environ.get("SLACK_WEBHOOK_URL", None)
-    slack_data = {"text": message}
-    requests.post(
-        webhook_url,
-        data=json.dumps(slack_data),
-        headers={"Content-Type": "application/json"},
-    )
 
+from .send_slack_message import notify_kbase_slack
 
 if __name__ == "__main__":
     hostname = socket.gethostname()
@@ -33,5 +19,5 @@ def send_slack_message(message: str):
             container.remove()
         debug_mode = os.environ.get("DEBUG", "false").lower() == "true"
         if debug_mode:
-            send_slack_message(
+            notify_kbase_slack(
                 f"Deleted {len(kbase_containers)} `exited` containers with 'kbase' in image name on {hostname}: {container_image_names}")

deployment/bin/cron/health_check.py

@@ -5,7 +5,6 @@
 """
 import datetime
 import inspect
-import json
 import logging
 import os
 import pwd
@@ -18,6 +17,8 @@
 import psutil
 import requests
 
+from .send_slack_message import notify_kbase_slack
+
 # Optional environment variables
 var_lib_docker = os.environ.get("DOCKER_CACHE", "/var/lib/docker/")
 scratch = os.environ.get("CONDOR_SUBMIT_WORKDIR", "/cdr")
@@ -38,17 +39,7 @@
 pid = pwd.getpwnam(user).pw_uid
 gid = pwd.getpwnam(user).pw_gid
 
-
-def send_slack_message(message: str):
-    """
-    :param message: Escaped Message to send to slack
-    """
-    slack_data = {"text": message}
-    requests.post(
-        webhook_url,
-        data=json.dumps(slack_data),
-        headers={"Content-Type": "application/json"},
-    )
+LOCKFILE_PATH = "/tmp/lockfile"
 
 
 def exit_unsuccessfully(message: str, send_to_slack=True):
@@ -58,22 +49,29 @@ def exit_unsuccessfully(message: str, send_to_slack=True):
     print("NODE_IS_HEALTHY = False")
     print(f'HEALTH_STATUS_MESSAGE = "{message}"')
     print("- update:true")
-    now = datetime.datetime.now()
 
     if send_to_slack:
-        try:
-            function_name = lambda: inspect.stack()[1][3]
-        except Exception:
+        if not os.path.exists(LOCKFILE_PATH):
+            # Lock file doesn't exist, meaning it's a new failure
+            # Notify Slack and create the lockfile to prevent future notifications for NODE_IS_HEALTHY = False
+            with open(LOCKFILE_PATH, 'w') as lock_file:
+                lock_file.write(message)  # Optionally write the failure message into the lockfile
             function_name = ""
-
-        send_slack_message(
-            f"POSSIBLE BLACK HOLE: {function_name} Ran healthcheck at {now} on {socket.gethostname()} with failure: {message}"
-        )
-
+            try:
+                function_name = inspect.stack()[1][3]
+            except Exception:
+                pass
+            notify_kbase_slack(
+                f"POSSIBLE BLACK HOLE: {function_name} on {socket.gethostname()} with failure: {message}"
+            )
+        # If the lock file exists, don't message slack, but continue the sys.exit(1)
     sys.exit(1)
 
 
 def exit_successfully():
+    if os.path.exists(LOCKFILE_PATH):
+        os.remove(LOCKFILE_PATH)  # Delete the lock file if it exists
+
     print("NODE_IS_HEALTHY = True")
     print(f'HEALTH_STATUS_MESSAGE = "Healthy {datetime.datetime.now()}"')
     print("- update:true")
@@ -197,7 +195,7 @@ def test_enough_space(mount_point, nickname, percentage):
 
 def check_kbase_endpoints():
     """
-    Check auth/njs/catalog/ws
+    Check Auth, Catalog, and Workspace status
     """
 
     post_services = {
@@ -232,7 +230,6 @@ def check_kbase_endpoints():
 
 def main():
     try:
-        # send_slack_message(f"Job HEALTH_CHECK is beginning at {datetime.datetime.now()}")
         test_docker_socket()
         test_docker_socket2()
         test_scratch_world_writeable()

deployment/bin/cron/send_slack_message.py

@@ -0,0 +1,16 @@
+import requests
+import os
+import json
+
+def notify_kbase_slack(message: str):
+    """
+    :param message: Escaped Message to send to slack
+    """
+    webhook_url = os.environ.get("SLACK_WEBHOOK_URL", None)
+    slack_data = {"text": message}
+    requests.post(
+        webhook_url,
+        data=json.dumps(slack_data),
+        headers={"Content-Type": "application/json"},
+    )
+

deployment/bin/cron/testing_create_dummy_containers.py

@@ -0,0 +1,22 @@
+import docker
+
+# Create a container with ee2 labels and username labels
+# Fake the creation time to be 7 days ago
+
+client = docker.from_env()
+c = client.containers.run("ubuntu:latest", "/bin/sleep infinity", labels={"user_name": "test_user", "ee2_endpoint":
+    "https://ci.kbase.us/services/ee2"},
+                      detach=True,
+                      name="test_container2",
+                      auto_remove=True)
+
+def remove_with_backoff(container,message,backoff=30):
+    try:
+        container.stop()
+        import time
+        time.sleep(backoff)  # Wait for backoff period before attempting to remove
+        container.remove()
+    except Exception as e:
+        pass
+
+remove_with_backoff(c,"test_container2")