Merge branch 'master' into 7217-impact-parser

sirepo/http_util.py

@@ -46,3 +46,18 @@ def parse_auth_header(headers):
     if m := _AUTH_HEADER_RE.search(h):
         return m.group(1)
     return None
+
+
+def remote_ip(request):
+    """IP address of client from request.
+
+    Tornado covers 'X-Real-Ip' and 'X-Forwared-For'. This adds addition
+    headers to check.
+
+    Args:
+      request (tornado.httputil.HTTPServerRequest): Incoming request
+    Returns:
+      str:  IP address of client
+
+    """
+    return request.headers.get("proxy-for", request.remote_ip)

sirepo/package_data/static/html/raydata-run-analysis.html

@@ -5,7 +5,7 @@
         <div data-advanced-editor-pane="" data-view-name="'runAnalysis'" data-field-def="basic" data-want-buttons="true"></div>
       </div>
       <div class="col-md-5">
-        <div data-scan-detail="" data-scan="runAnalysis.raydataService.detailScan"></div>
+        <div data-ng-if="runAnalysis.raydataService.detailScan" data-scan-detail="" data-scan="runAnalysis.raydataService.detailScan"></div>
       </div>
       <div class="col-md-12">
         <div data-model-field="'scans'" data-model-name="'runAnalysis'"></div>

sirepo/package_data/static/js/raydata.js

@@ -1198,83 +1198,73 @@ SIREPO.app.directive('scanDetail', function() {
         template: `
             <div><strong>Scan Detail</strong></div>
             <div class="well" style="height: 250px; overflow: auto;">
-            <div data-ng-if="scan">
-              <div><strong>Scan Id:</strong> {{ scan.rduid }}</div>
-              <div data-ng-if="analysisElapsedTime()"><strong>Analysis Elapsed Time:</strong> {{ analysisElapsedTime() }} seconds</div>
-              <div data-ng-if="detailedStatusFile()">
-                <div><strong>Current Consecutive Failures:</strong> {{ consecutiveFailures() }}</div>
-              </div>
-              <div data-ng-if="detailedStatusFile()">
-                <div><strong>Most Recent Status</strong></div>
-                <pre>{{ currentStatus() }}</pre>
-              </div>
-              <div data-ng-if="detailedStatusFile()">
-                <div><strong>Detailed Status File</strong></div>
-                <pre>{{ detailedStatus() }}</pre>
+              <div data-ng-if="scan">
+                <div><strong>Scan Id:</strong> {{ scan.rduid }}</div>
+                <div data-ng-if="analysisElapsedTime()"><strong>Analysis Elapsed Time:</strong> {{ analysisElapsedTime() }} seconds</div>
+                <div>
+                  <div><strong>Current Status: </strong>{{ scan.status }}</div>
+                  <div data-ng-if="! isEmptyObject(latestDetailedStatus)">
+                    <strong>Detailed Status:</strong>
+                      <ul>
+                        <li data-ng-repeat="(stepName, stepInfo) in latestDetailedStatus">
+                          {{ stepName }}
+                          <ul>
+                            <li data-ng-repeat="key in ['start', 'stop']" data-ng-if="stepInfo[key]">
+                              {{ key }}: {{ parseTime(stepInfo[key]) }}
+                            </li>
+                            <li data-ng-if="stepElapsed(stepInfo)">elapsed: {{ stepElapsed(stepInfo) }}</li>
+                            <li data-ng-if="stepStatus(stepInfo)">status: {{ stepStatus(stepInfo) }}</li>
+                          </ul>
+                        </li>
+                      </ul>
+                  </div>
+                </div>
               </div>
             </div>
-            </div>
-`,
-        controller: function($scope, columnsService, utilities) {
-            function failureInRun(run) {
-                let r = false;
-                for (const f of Object.values($scope.detailedStatusFile()[run])) {
-                    if (f.status === 'failed') {
-                        r = true;
-                    }
-                }
-                return r;
-            }
-
-            function getSortedRunIndexes() {
-                return Object.keys($scope.detailedStatusFile()).map((x) => parseInt(x)).sort();
-            }
+	`,
+        controller: function($filter, $scope, columnsService, raydataService, utilities) {
+	    $scope.latestDetailedStatus = null;
 
-            function mostRecentAnalysisDetails() {
-                return $scope.detailedStatusFile()? $scope.detailedStatusFile()[Math.max(...getSortedRunIndexes())] : '';
-             }
+	    function setLatestDetailedStatus() {
+		$scope.latestDetailedStatus = $scope.scan.detailed_status[Math.max(Object.keys($scope.scan.detailed_status))];
+	    }
 
             $scope.analysisElapsedTime = () => {
                 return $scope.scan && $scope.scan.analysis_elapsed_time ? $scope.scan.analysis_elapsed_time : null;
             };
 
-            $scope.consecutiveFailures = () => {
-                if (! $scope.detailedStatusFile()) {
-                    return '';
-                }
-                let r = 0;
-                for (const k of getSortedRunIndexes().reverse()) {
-                    if (failureInRun(k)) {
-                        r += 1;
-                    } else {
-                        return r;
-                    }
-                }
-
-                return r;
-            };
-
-            $scope.currentStatus = () => {
-                let r = '';
-                for (const k of Object.keys(mostRecentAnalysisDetails())) {
-                    r += k + ': ' + mostRecentAnalysisDetails()[k].status + '\n';
-                }
-                return r;
-            };
-
-            $scope.detailedStatus = () => {
-                return utilities.objectToText($scope.detailedStatusFile()).replace(
-                    /(start:|stop:)(\s*)(\d+\.?\d*)/gi,
-                    (_, p1, p2, p3) => {
-                        return p1 + p2 + (new Date(parseFloat(p3)*1000)).toString();
-                    }
-                )
-                ;
-            };
-
-            $scope.detailedStatusFile = () => {
-                return $scope.scan && $scope.scan.detailed_status && Object.keys($scope.scan.detailed_status).length > 0 ? $scope.scan.detailed_status : null;
-            };
+	    $scope.isEmptyObject = (obj) => {
+		return $.isEmptyObject(obj);
+	    };
+
+	    $scope.parseTime = (unixTime) => {
+		return (new Date(unixTime * 1000)).toString();
+	    };
+
+	    $scope.stepElapsed = (stepInfo) => {
+		if (stepInfo.start && stepInfo.stop) {
+		    return $filter('date')((stepInfo.stop - stepInfo.start) * 1000, 'HH:mm:ss', 'UTC');
+		}
+		return null;
+	    };
+
+	    $scope.stepStatus = (stepInfo) => {
+		function pendingIfRunningElseError(scanStatus) {
+		    if (raydataService.ANALYSIS_STATUS_NON_STOPPED.includes(scanStatus)) {
+			return 'pending';
+		    }
+		    return 'error';
+		}
+
+		// start and stop have been set so the notebook was
+		// able to manage setting the status itself.
+		if (stepInfo.start && stepInfo.stop) {
+		    return stepInfo.status;
+		}
+		return pendingIfRunningElseError($scope.scan.status);
+	    };
+
+            $scope.$watch('scan', setLatestDetailedStatus);
         },
     };
 });

sirepo/pkcli/job_supervisor.py

@@ -15,6 +15,7 @@
 import sirepo.events
 import sirepo.feature_config
 import sirepo.global_resources.api
+import sirepo.http_util
 import sirepo.job
 import sirepo.job_driver
 import sirepo.job_supervisor
@@ -97,7 +98,7 @@ def open(self):
         pkdlog(
             "uri={} remote_ip={} ",
             self.request.uri,
-            self.request.remote_ip,
+            sirepo.http_util.remote_ip(self.request),
         )
 
     def sr_close(self):

sirepo/raydata/analysis_driver/__init__.py

@@ -39,6 +39,11 @@ def get_notebooks(self, *args, **kwargs):
         raise NotImplementedError("children must implement this method")
 
     def get_output(self):
+        def load_json(path):
+            d = pkjson.load_any(path)
+            # CHX json are double encoded so may need to load 2x
+            return pkjson.load_any(d) if isinstance(d, str) else d
+
         res = PKDict()
         for e in [
             PKDict(
@@ -53,7 +58,7 @@ def get_output(self):
             PKDict(
                 name="jsonFiles",
                 file_type="json",
-                op=pkjson.load_any,
+                op=load_json,
             ),
         ]:
             res[e.name] = [
@@ -69,12 +74,14 @@ def get_output_dir(self):
 
     def get_papermill_args(self):
         res = []
-        for n, v in [
-            ["uid", self.rduid],
-            ["scan", self.rduid],
+        for a in [
+            PKDict(name="uid", value=self.rduid),
+            PKDict(name="scan", value=self.rduid),
             *self._get_papermill_args(),
         ]:
-            res.extend(["-p", f"'{n}'", f"'{v}'"])
+            res.extend(
+                ["-r" if a.get("raw_param") else "-p", f"'{a.name}'", f"'{a.value}'"]
+            )
         res.extend(("--report-mode", "--log-output", "--progress-bar"))
         return res
 

sirepo/raydata/analysis_driver/chx.py

@@ -21,10 +21,14 @@ def get_conda_env(self):
 
     def get_detailed_status_file(self, rduid):
         p = self.get_output_dir().join(f"progress_dict_{rduid}.json")
-        if os.path.exists(p):
-            with open(p, "r") as f:
-                return pkjson.load_any(f)
-        return PKDict()
+        if not p.check():
+            return PKDict()
+        d = pkjson.load_any(p)
+        # The notebooks do json.dump(json.dumps(progress_dict), outfile)
+        # which double encodes the json object. So, we may
+        # need to decode it 2x. Be compliant either way in case this
+        # changes in the future.
+        return pkjson.load_any(d) if isinstance(d, str) else d
 
     def get_notebooks(self):
         return [
@@ -54,9 +58,19 @@ def get_output_dir(self):
 
     def _get_papermill_args(self, *args, **kwargs):
         return [
-            ["run_two_time", True],
-            ["run_dose", False],
-            ["username", self._scan_metadata.get_start_field("user")],
+            # Cycle can look like 2024_2 which is converted to int by papermill unless raw_param=True
+            PKDict(
+                name="cycle",
+                value=self._scan_metadata.get_start_field("cycle"),
+                raw_param=True,
+            ),
+            # POSIT: Same as AutoRun_functions.get_process_id
+            PKDict(name="process_id", value=f"{self.rduid}_0"),
+            PKDict(name="username", value=self._scan_metadata.get_start_field("user")),
+            PKDict(
+                name="user_group",
+                value=self._scan_metadata.get_start_field("user_group", unchecked=True),
+            ),
         ]
 
 

sirepo/request.py

@@ -11,9 +11,9 @@
 import pykern.pkcompat
 import pykern.pkjson
 import sirepo.const
+import sirepo.http_util
 import sirepo.quest
 import sirepo.util
-import urllib.parse
 import user_agents
 
 
@@ -186,7 +186,7 @@ def _parse_authorization(value):
             http_method=r.method,
             http_request_uri=r.full_url(),
             http_server_uri=f"{r.protocol}://{r.host}/",
-            remote_addr=r.remote_ip,
+            remote_addr=sirepo.http_util.remote_ip(r),
         )
 
     def body_as_bytes(self):

sirepo/smtp.py

@@ -1,11 +1,9 @@
-# -*- coding: utf-8 -*-
 """SMTP connection to send emails
 
-:copyright: Copyright (c) 2018-2019 RadiaSoft LLC.  All Rights Reserved.
+:copyright: Copyright (c) 2018-2024 RadiaSoft LLC.  All Rights Reserved.
 :license: http://www.apache.org/licenses/LICENSE-2.0.html
 """
 
-from __future__ import absolute_import, division, print_function
 from pykern.pkdebug import pkdp, pkdlog
 from pykern import pkconfig
 from pykern.pkcollections import PKDict
@@ -68,11 +66,12 @@ def _send_directly(msg):
             s.send_message(msg)
 
 
-def _send_with_auth(msg):
+def _send_via_relay_server(msg):
     with smtplib.SMTP(_cfg.server, _cfg.port) as s:
         s.starttls()
         s.ehlo()
-        s.login(_cfg.user, _cfg.password)
+        if _cfg.user and _cfg.password:
+            s.login(_cfg.user, _cfg.password)
         s.send_message(msg)
 
 
@@ -94,14 +93,18 @@ def _init():
         _cfg.server = "not " + _DEV_SMTP_SERVER
         _SEND = _send_directly
         return
-    _SEND = _send_with_auth
+    _SEND = _send_via_relay_server
     if pkconfig.in_dev_mode():
         if _cfg.server is None:
             _cfg.server = _DEV_SMTP_SERVER
         return
-    if _cfg.server is None or _cfg.user is None or _cfg.password is None:
+    if _cfg.server is None:
         pkconfig.raise_error(
-            f"server={_cfg.server}, user={_cfg.user}, and password={_cfg.password} must be defined",
+            f"server={_cfg.server} must be defined",
+        )
+    if bool(_cfg.user) != bool(_cfg.password):
+        pkconfig.raise_error(
+            f"user={_cfg.user} and password={_cfg.password} must be both set or not"
         )
 
 

sirepo/uri_router.py

@@ -7,25 +7,21 @@
 from pykern import pkcollections
 from pykern import pkconfig
 from pykern import pkinspect
-from pykern import pkjson
 from pykern.pkcollections import PKDict
 from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp, pkdformat
 import asyncio
-import contextlib
 import importlib
 import inspect
-import os
-import pkgutil
 import re
 import sirepo.api_auth
 import sirepo.auth
 import sirepo.const
 import sirepo.events
 import sirepo.feature_config
+import sirepo.http_util
 import sirepo.spa_session
 import sirepo.uri
 import sirepo.util
-import urllib.parse
 
 #: prefix for api functions
 _FUNC_PREFIX = "api_"
@@ -225,7 +221,7 @@ def open(self):
             self.__headers = PKDict(r.headers)
             self.cookie_state = self.__headers.get("Cookie")
             self.http_server_uri = f"{r.protocol}://{r.host}/"
-            self.remote_addr = r.remote_ip
+            self.remote_addr = sirepo.http_util.remote_ip(r)
             self.ws_id = ws_count
             self.sr_log(None, "open", fmt=" ip={}", args=[_remote_peer(r)])
 
@@ -354,7 +350,7 @@ def _remote_peer(request):
             # socket is not set on stream for websockets.
             if hasattr(c, "stream") and hasattr(c.stream, "socket"):
                 return "{}:{}".format(*c.stream.socket.getpeername())
-        return f"{request.remote_ip}:0"
+        return f"{sirepo.http_util.remote_ip(request)}:0"
 
     sirepo.modules.import_and_init("sirepo.server").init_tornado()
     s = httpserver.HTTPServer(