Skip to content

Commit

Permalink
Fix #7231: Raydata fixes for scan detailed status
Browse files Browse the repository at this point in the history
  • Loading branch information
@e-carlin
e-carlin committed Aug 30, 2024
1 parent 9df44ef commit f679c05
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 82 deletions.
2 changes: 1 addition & 1 deletion sirepo/package_data/static/html/raydata-run-analysis.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<div data-advanced-editor-pane="" data-view-name="'runAnalysis'" data-field-def="basic" data-want-buttons="true"></div>
</div>
<div class="col-md-5">
<div data-scan-detail="" data-scan="runAnalysis.raydataService.detailScan"></div>
<div data-ng-if="runAnalysis.raydataService.detailScan" data-scan-detail="" data-scan="runAnalysis.raydataService.detailScan"></div>
</div>
<div class="col-md-12">
<div data-model-field="'scans'" data-model-name="'runAnalysis'"></div>
Expand Down
128 changes: 59 additions & 69 deletions sirepo/package_data/static/js/raydata.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1198,83 +1198,73 @@ SIREPO.app.directive('scanDetail', function() {
template: `
<div><strong>Scan Detail</strong></div>
<div class="well" style="height: 250px; overflow: auto;">
<div data-ng-if="scan">
<div><strong>Scan Id:</strong> {{ scan.rduid }}</div>
<div data-ng-if="analysisElapsedTime()"><strong>Analysis Elapsed Time:</strong> {{ analysisElapsedTime() }} seconds</div>
<div data-ng-if="detailedStatusFile()">
<div><strong>Current Consecutive Failures:</strong> {{ consecutiveFailures() }}</div>
</div>
<div data-ng-if="detailedStatusFile()">
<div><strong>Most Recent Status</strong></div>
<pre>{{ currentStatus() }}</pre>
</div>
<div data-ng-if="detailedStatusFile()">
<div><strong>Detailed Status File</strong></div>
<pre>{{ detailedStatus() }}</pre>
<div data-ng-if="scan">
<div><strong>Scan Id:</strong> {{ scan.rduid }}</div>
<div data-ng-if="analysisElapsedTime()"><strong>Analysis Elapsed Time:</strong> {{ analysisElapsedTime() }} seconds</div>
<div>
<div><strong>Current Status: </strong>{{ scan.status }}</div>
<div data-ng-if="! isEmptyObject(latestDetailedStatus)">
<strong>Detailed Status:</strong>
<ul>
<li data-ng-repeat="(stepName, stepInfo) in latestDetailedStatus">
{{ stepName }}
<ul>
<li data-ng-repeat="key in ['start', 'stop']" data-ng-if="stepInfo[key]">
{{ key }}: {{ parseTime(stepInfo[key]) }}
</li>
<li data-ng-if="stepElapsed(stepInfo)">elapsed: {{ stepElapsed(stepInfo) }}</li>
<li data-ng-if="stepStatus(stepInfo)">status: {{ stepStatus(stepInfo) }}</li>
</ul>
</li>
</ul>
</div>
</div>
</div>
</div>
</div>
`,
controller: function($scope, columnsService, utilities) {
function failureInRun(run) {
let r = false;
for (const f of Object.values($scope.detailedStatusFile()[run])) {
if (f.status === 'failed') {
r = true;
}
}
return r;
}

function getSortedRunIndexes() {
return Object.keys($scope.detailedStatusFile()).map((x) => parseInt(x)).sort();
}
`,
controller: function($filter, $scope, columnsService, raydataService, utilities) {
$scope.latestDetailedStatus = null;

function mostRecentAnalysisDetails() {
return $scope.detailedStatusFile()? $scope.detailedStatusFile()[Math.max(...getSortedRunIndexes())] : '';
}
function setLatestDetailedStatus() {
$scope.latestDetailedStatus = $scope.scan.detailed_status[Math.max(Object.keys($scope.scan.detailed_status))];
}

$scope.analysisElapsedTime = () => {
return $scope.scan && $scope.scan.analysis_elapsed_time ? $scope.scan.analysis_elapsed_time : null;
};

$scope.consecutiveFailures = () => {
if (! $scope.detailedStatusFile()) {
return '';
}
let r = 0;
for (const k of getSortedRunIndexes().reverse()) {
if (failureInRun(k)) {
r += 1;
} else {
return r;
}
}

return r;
};

$scope.currentStatus = () => {
let r = '';
for (const k of Object.keys(mostRecentAnalysisDetails())) {
r += k + ': ' + mostRecentAnalysisDetails()[k].status + '\n';
}
return r;
};

$scope.detailedStatus = () => {
return utilities.objectToText($scope.detailedStatusFile()).replace(
/(start:|stop:)(\s*)(\d+\.?\d*)/gi,
(_, p1, p2, p3) => {
return p1 + p2 + (new Date(parseFloat(p3)*1000)).toString();
}
)
;
};

$scope.detailedStatusFile = () => {
return $scope.scan && $scope.scan.detailed_status && Object.keys($scope.scan.detailed_status).length > 0 ? $scope.scan.detailed_status : null;
};
$scope.isEmptyObject = (obj) => {
return $.isEmptyObject(obj)
}

$scope.parseTime = (unixTime) => {
return (new Date(unixTime * 1000)).toString()
};

$scope.stepElapsed = (stepInfo) => {
if (stepInfo['start'] && stepInfo['stop']) {
return $filter('date')((stepInfo['stop'] - stepInfo['start']) * 1000, 'HH:mm:ss', 'UTC');
}
return null;
};

$scope.stepStatus = (stepInfo) => {
function pendingIfRunningElseError(scanStatus) {
if (raydataService.ANALYSIS_STATUS_NON_STOPPED.includes(scanStatus)) {
return 'pending';
}
return 'error';
}

// start and stop have been set so the notebook was
// able to manage setting the status itself.
if (stepInfo['start'] && stepInfo['stop']) {
return stepInfo['status'];
}
return pendingIfRunningElseError($scope.scan.status);
};

$scope.$watch('scan', setLatestDetailedStatus);
},
};
});
Expand Down
17 changes: 12 additions & 5 deletions sirepo/raydata/analysis_driver/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ def get_notebooks(self, *args, **kwargs):
raise NotImplementedError("children must implement this method")

def get_output(self):
def load_json(path):
d = pkjson.load_any(path)
# CHX json are double encoded so may need to load 2x
return pkjson.load_any(d) if isinstance(d, str) else d

res = PKDict()
for e in [
PKDict(
Expand All @@ -53,7 +58,7 @@ def get_output(self):
PKDict(
name="jsonFiles",
file_type="json",
op=pkjson.load_any,
op=load_json,
),
]:
res[e.name] = [
Expand All @@ -69,12 +74,14 @@ def get_output_dir(self):

def get_papermill_args(self):
res = []
for n, v in [
["uid", self.rduid],
["scan", self.rduid],
for a in [
PKDict(name="uid", value=self.rduid),
PKDict(name="scan", value=self.rduid),
*self._get_papermill_args(),
]:
res.extend(["-p", f"'{n}'", f"'{v}'"])
res.extend(
["-r" if a.get("raw_param") else "-p", f"'{a.name}'", f"'{a.value}'"]
)
res.extend(("--report-mode", "--log-output", "--progress-bar"))
return res

Expand Down
28 changes: 21 additions & 7 deletions sirepo/raydata/analysis_driver/chx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,14 @@ def get_conda_env(self):

def get_detailed_status_file(self, rduid):
p = self.get_output_dir().join(f"progress_dict_{rduid}.json")
if os.path.exists(p):
with open(p, "r") as f:
return pkjson.load_any(f)
return PKDict()
if not p.check():
return PKDict()
d = pkjson.load_any(p)
# The notebooks do json.dump(json.dumps(progress_dict), outfile)
# which double encodes the json object. So, we may
# need to decode it 2x. Be compliant either way in case this
# changes in the future.
return pkjson.load_any(d) if isinstance(d, str) else d

def get_notebooks(self):
return [
Expand Down Expand Up @@ -54,9 +58,19 @@ def get_output_dir(self):

def _get_papermill_args(self, *args, **kwargs):
return [
["run_two_time", True],
["run_dose", False],
["username", self._scan_metadata.get_start_field("user")],
# Cycle can look like 2024_2 which is converted to int by papermill unless raw_param=True
PKDict(
name="cycle",
value=self._scan_metadata.get_start_field("cycle"),
raw_param=True,
),
# POSIT: Same as AutoRun_functions.get_process_id
PKDict(name="process_id", value=f"{self.rduid}_0"),
PKDict(name="username", value=self._scan_metadata.get_start_field("user")),
PKDict(
name="user_group",
value=self._scan_metadata.get_start_field("user_group", unchecked=True),
),
]


Expand Down

0 comments on commit f679c05

Please sign in to comment.