cloud-gov · JasonTheMain · Nov 14, 2024 · Nov 7, 2024 · Nov 8, 2024 · Nov 12, 2024
@@ -27,6 +27,7 @@ consumes:
 templates:
   bin/generate-objects: bin/generate-objects
   bin/generate-tenants: bin/generate-tenants
+  bin/generate-ndjson: bin/generate-ndjson
   bin/import-objects: bin/import-objects
   bin/run: bin/run
   bin/pre-start.erb: bin/pre-start

@@ -0,0 +1,32 @@
+#!/var/vcap/packages/python3-requests/venv/bin/python3
+import os
+import json
+
+folders = [
+"/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/dashboard",
+"/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/search",
+"/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/visualization"
+]
+output_file = "/var/vcap/jobs/upload-dashboards-objects/dashboard.ndjson"
+
+
+with open(output_file, 'w', encoding='utf-8') as output:
+  for folder in folders:
+    if not os.path.exists(folder):
+      print(f"Folder not found: {folder}")
+      continue
+    for files in os.listdir(folder):
+      if files.endswith(".json"):
+        filepath = os.path.join(folder,files)
+
+        with open(filepath, 'r') as json_file:
+          try:
+            data = json.load(json_file)
+          except json.JSONDecodeError as e:
+            print(f"Error with file {files}: {e}")
+            continue
+
+          json.dump(data,output)
+          output.write("\n")
+
+
@@ -16,7 +16,7 @@ logging.basicConfig(level=logging.INFO)
 
 if __name__ == "__main__":
     session = requests.Session()
-
+    ndjson_output_path = "/var/vcap/jobs/upload-dashboards-objects/dashboard.ndjson"
     # set CA
     ca =  '<%= job_dir %>/config/ssl/opensearch.ca'
     cert = '<%= job_dir %>/config/ssl/dashboard-web.crt'
@@ -26,57 +26,72 @@ if __name__ == "__main__":
     with open('<%= store_dir %>/org-tenants.txt') as orgs:
       for tenant in orgs:
         headers = {
-          'content-type': 'application/json',
           'osd-xsrf': 'true',
           'x-proxy-roles': 'admin',
           'x-proxy-user': 'admin',
           'x-forwarded-for': '127.0.0.1',
-          'securitytenant': tenant.strip(),
+          'securitytenant': tenant.strip()
         }
 
-        <% p("dashboards_objects.upload_patterns").each do |pattern| %>
-        for filename in glob.iglob("<%= pattern.fetch("pattern") %>", recursive=True):
-            r = session.post(
-              '{}/api/saved_objects/{}/{}'.format(
-                logs_url,
-                os.path.basename(os.path.dirname(filename)),
-                os.path.basename(os.path.splitext(filename)[0])
+        with open(ndjson_output_path, 'rb') as ndjson_file:
+          ndjson_files= {'file':(ndjson_output_path, ndjson_file, 'application/x-ndjson')}
+          r = session.post(
+            '{}/api/saved_objects/_import?overwrite=true'.format(
+              logs_url
+            ),
+            files=ndjson_files,
+            headers=headers,
+            cert=(cert, key),
+            verify=ca
+          )
+          if r.status_code == 200:
+            logging.info(f'NDJSON FILE {ndjson_output_path} uploaded sucessfully')
+          elif r.status_code == 409:
+            put = session.put(
+                  '{}/api/saved_objects/_import?overwrite=true'.format(
+              logs_url
               ),
-              data=open(filename, 'rb'),
-              headers=headers,
-              cert=(cert, key),
-              verify=ca
+            data=ndjson_file,
+            headers=headers,
+            cert=(cert, key),
+            verify=ca
             )
-            if r.status_code == 409:
-                put = session.put(
-                  '{}/api/saved_objects/{}/{}'.format(
-                    logs_url,
-                    os.path.basename(os.path.dirname(filename)),
-                    os.path.basename(os.path.splitext(filename)[0])
-                  ),
-                  data=open(filename, 'rb'),
-                  headers=headers,
-                  cert=(cert, key),
-                  verify=ca
-                )
-                if put.status_code == 200:
-                    logging.info('Object %s Uploaded Successfully', filename)
-                else:
-                    logging.info('Object %s Failed to Upload', filename)
-            elif r.status_code == 200:
-                logging.info('Object %s Uploaded Successfully', filename)
+            if put.status_code == 200:
+                logging.info(f'NDJSON FILE {ndjson_output_path} uploaded successfully')
             else:
-                logging.info('Object %s Failed to Upload', filename)
-        <% end %>
+                logging.error(f"Failed to upload file {ndjson_output_path}, status code: {put.status_code}")
+                try:
+                   error_response = put.json()
+                   logging.error(f"Error detils: {error_response}")
+                except ValueError:
+                   logging.error(f"raw response: {put.text}")
+                logging.info(f'Failed to upload NDJSON FILE {ndjson_output_path}')
+          else:
+              logging.error(f"Failed to upload file {ndjson_output_path}, status code: {r.status_code}")
+              try:
+                  error_response = r.json()
+                  logging.error(f"Error detils: {error_response}")
+              except ValueError:
+                  logging.error(f"raw response: {r.text}")
+              logging.info(f'Failed to upload NDJSON FILE {ndjson_output_path}')
 
+        index_headers = {
+          'content-type': 'application/json',
+          'osd-xsrf': 'true',
+          'x-proxy-roles': 'admin',
+          'x-proxy-user': 'admin',
+          'x-forwarded-for': '127.0.0.1',
+          'securitytenant': tenant.strip(),
+        }
         r = session.post(
           '{}/api/opensearch-dashboards/settings'.format(logs_url),
           data='{"changes":{"defaultIndex":"<%= p('dashboards_objects.default_index') %>"}}',
-          headers=headers,
+          headers=index_headers,
           cert=(cert, key),
           verify=ca
         )
         if r.status_code == 200:
             logging.info('Successfully Set <%= p('dashboards_objects.default_index') %> as Default Index')
         else:
             logging.info('Failed to Set <%= p('dashboards_objects.default_index') %> as Default Index')
+
@@ -15,3 +15,9 @@ export JOB_DIR=/var/vcap/jobs/$JOB_NAME
 if [ -d ${JOB_DIR}/config/ssl ]; then
 	chown -R vcap:vcap ${JOB_DIR}/config/ssl
 fi
+
+export PATH=/var/vcap/packages/python3/bin:$PATH
+export PYTHONPATH="/var/vcap/packages/python3/lib/python3.12/site-packages${PYTHONPATH:+:$PYTHONPATH}"
+export LD_LIBRARY_PATH="/var/vcap/packages/python3/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+
+$(dirname $0)/generate-ndjson
@@ -92,6 +92,9 @@ ENDOFFJSON
   %>"
     }
   },
+  "type": "dashboard",
+  "title": "App Errors",
+  "id": "App-Errors",
   "references": [
     {
       "id": "App-errors-count-in-timeline-(top-5-apps)",

@@ -70,6 +70,9 @@ ENDOFFJSON
   %>"
     }
   },
+  "type": "dashboard",
+  "title": "App Location",
+  "id": "App-Location",
   "references": [
     {
       "id": "App-RTR-requests-map",

@@ -124,6 +124,9 @@ ENDOFFJSON
   %>"
     }
   },
+  "type": "dashboard",
+  "title": "App Logs",
+  "id": "App-Logs",
   "references": [
     {
       "id": "App-logs-count-by-source-type-(APP-or-cf-top-5-apps)",

@@ -108,6 +108,9 @@ ENDOFFJSON
   %>"
     }
   },
+  "type": "dashboard",
+  "title": "App Metrics",
+  "id": "App-Metrics",
   "references": [
     {
       "id": "App-METRIC-memory-usage",

@@ -102,6 +102,9 @@ ENDOFFJSON
   %>"
     }
   },
+  "type": "dashboard",
+  "title": "App Overview",
+  "id": "App-Overview",
   "references": [
     {
       "id": "App-links",

@@ -108,6 +108,9 @@ ENDOFFJSON
   %>"
     }
   },
+  "type": "dashboard",
+  "title": "App Performance",
+  "id": "App-Performance",
   "references": [
     {
       "id": "App-RTR-request-count-(top-10-apps)",

@@ -68,6 +68,10 @@
   ).to_json.gsub(/"/) { '\"' }
   %>"
     }
-  }
+  },
+  "id": "app-all-errors",
+  "references": [],
+  "title": "[app] All - ERRORs",
+  "type": "search"
 }
 <% end %>
@@ -19,7 +19,7 @@
       {
         "index": "#{p('opensearch_config.app_index_prefix')}*",
         "filter": [
-          
+
         ],
         "highlight": {
           "pre_tags": [
@@ -30,7 +30,7 @@
           ],
           "fields": {
             "*": {
-              
+
             }
           },
           "require_field_match": false,
@@ -46,5 +46,9 @@
   ).to_json.gsub(/"/) { '\"' }
   %>"
     }
-  }
+  },
+  "id": "app-all-messages",
+  "title": "[app] All Messages",
+  "references": [],
+  "type": "search"
 }
@@ -52,6 +52,10 @@
   ).to_json.gsub(/"/) { '\"' }
   %>"
     }
-  }
+  },
+  "id": "app-all-overview",
+  "references": [],
+  "title": "[app] All Overview",
+  "type": "search"
 }
 <% end %>
@@ -50,6 +50,10 @@
   ).to_json.gsub(/"/) { '\"' }
   %>"
     }
-  }
+  },
+  "id": "app-all",
+  "references": [],
+  "title": "[app] All",
+  "type": "search"
 }
 <% end %>
@@ -64,6 +64,10 @@
   ).to_json.gsub(/"/) { '\"' }
   %>"
     }
-  }
+  },
+  "id": "app-container-metric-cpu",
+  "references": [],
+  "title": "[app] Container Metric cpu",
+  "type": "search"
 }
 <% end %>
@@ -64,6 +64,10 @@
   ).to_json.gsub(/"/) { '\"' }
   %>"
     }
-  }
+  },
+  "id": "app-container-metric-disk",
+  "references": [],
+  "title": "[app] Container Metric disk",
+  "type": "search"
 }
 <% end %>