From 0390822144b6fde4190ef1b4cf0c8f00fb17dcb6 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:52:40 -0700 Subject: [PATCH 1/6] Increasing Azure App Function Timeout to 12 Hours (#3599) * add * Set function timeout to 12 hours. --- src/ApiService/ApiService/host.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ApiService/ApiService/host.json b/src/ApiService/ApiService/host.json index beb2e4020b..7b83b41ae4 100644 --- a/src/ApiService/ApiService/host.json +++ b/src/ApiService/ApiService/host.json @@ -1,5 +1,6 @@ { "version": "2.0", + "functionTimeout": "12:00:00", "logging": { "applicationInsights": { "samplingSettings": { From 909b1309d8b855763248d4faaad6fbe9dc8b7c63 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:30:42 -0700 Subject: [PATCH 2/6] JobResult Table Re-design - Improved task- and machine- granularity (#3539) * Release 8.7.1 (hotfix) (#3459) * Remove the retention policy setting (#3452) --------- Co-authored-by: Cheick Keita * Revert "Release 8.7.1 (hotfix) (#3459)" (#3468) This reverts commit c69deed50e81cc1805f6f82ebb10513a211cbbe2. * Redo 8.7.1 (#3469) * Redo-8.7.1-hotfix --------- Co-authored-by: Cheick Keita * Support custom ado fields that mark work items as duplicate (#3467) * Add field to ado config for checking duplicate work items * Make duplicate fields nullable and add it to python models * Update broken tests * Update docs to include new ado_duplicate_fields property * Update readme with archive message (#3408) Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> * Bump tokio from 1.30.0 to 1.32.0 in /src/proxy-manager (#3425) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.30.0 to 1.32.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.30.0...tokio-1.32.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tokio from 1.30.0 to 1.32.0 in /src/agent (#3424) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.30.0 to 1.32.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.30.0...tokio-1.32.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Remove unnecessary method argument (#3473) * Bump elsa from 1.8.1 to 1.9.0 in /src/agent (#3411) Bumps [elsa](https://github.com/manishearth/elsa) from 1.8.1 to 1.9.0. - [Commits](https://github.com/manishearth/elsa/compare/v1.8.1...v1.9.0) --- updated-dependencies: - dependency-name: elsa dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tempfile from 3.7.1 to 3.8.0 in /src/agent (#3437) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.7.1 to 3.8.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.7.1...v3.8.0) --- updated-dependencies: - dependency-name: tempfile dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tempfile from 3.7.1 to 3.8.0 in /src/proxy-manager (#3436) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.7.1 to 3.8.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.7.1...v3.8.0) --- updated-dependencies: - dependency-name: tempfile dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Updating requirements.txt to accept >= onefuzztypes. (#3477) * Updating requirements.txt to accept >= onefuzztypes. * Trying to loosen restriction. * Bump notify from 6.0.1 to 6.1.1 in /src/agent (#3435) Bumps [notify](https://github.com/notify-rs/notify) from 6.0.1 to 6.1.1. - [Release notes](https://github.com/notify-rs/notify/releases) - [Changelog](https://github.com/notify-rs/notify/blob/main/CHANGELOG.md) - [Commits](https://github.com/notify-rs/notify/compare/notify-6.0.1...notify-6.1.1) --- updated-dependencies: - dependency-name: notify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure_* crates (#3478) * Release 8.8.0 (#3466) * Release 8.8.0 * Bump clap from 4.3.21 to 4.4.2 in /src/agent (#3484) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.21 to 4.4.2. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.21...v4.4.2) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump gimli from 0.27.3 to 0.28.0 in /src/agent (#3414) Bumps [gimli](https://github.com/gimli-rs/gimli) from 0.27.3 to 0.28.0. - [Changelog](https://github.com/gimli-rs/gimli/blob/master/CHANGELOG.md) - [Commits](https://github.com/gimli-rs/gimli/compare/0.27.3...0.28.0) --- updated-dependencies: - dependency-name: gimli dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump clap from 4.3.21 to 4.4.2 in /src/proxy-manager (#3474) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.21 to 4.4.2. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.21...v4.4.2) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump winreg from 0.50.0 to 0.51.0 in /src/agent (#3434) Bumps [winreg](https://github.com/gentoo90/winreg-rs) from 0.50.0 to 0.51.0. - [Release notes](https://github.com/gentoo90/winreg-rs/releases) - [Changelog](https://github.com/gentoo90/winreg-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/gentoo90/winreg-rs/compare/v0.50.0...v0.51.0) --- updated-dependencies: - dependency-name: winreg dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> * Starting integration tests (#3438) * Starting integration tests * Ready to test the test * Parametrize test * checkpoint * Test works * Run integration tests in pipeline * fmt * . * -p * Install clang * quotes not required in yaml? * Hopefully fixed windows? * Try without killondrop * lint * small test * another test * Reuse core name * Wrong step * bump tokio? * Try with rust * make build happy * Bump pete and small clean up * Clean up and make the test pass regularly * fix broken ci * Lower the poll timeout * Set the timeout in a nicer way * fix windows * fmt * Include and copy pdbs * Ignore if pdb is missing on linux * It takes too long for coverage to be generated * lint * Only warn on missing coverage since it's flaky * Fix windows build * Small clean up * Try lowering the poll delay * fix coverage * PR comments * . * Apparently make is missing? * Remove aggressive step skipping in CI * Fix sed checks for CLI versioning (#3486) * Fix sed checks for CLI versioning * Fix. * Fix. * Changing build_cli * Trying greater than * Tring once more. * Trying major minor * trying to replace major minor * Using major minor * Bump bytes from 1.4.0 to 1.5.0 in /src/agent (#3488) Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/compare/v1.4.0...v1.5.0) --- updated-dependencies: - dependency-name: bytes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Improve area/iteration path validation (#3489) * Add more comprehensive checks and better error messages to area/iteration path validation * Join invalid chars with space instead of comma * Make tree path validation more testable * Add error code for invalid ADO project in config * Write unit tests for tree path validation * Format tree path unit tests * Merge escape character and control character checks and clarify error message * Improve handling of unexpected breakpoints (#3493) * Improve handling of unexpected breakpoints * fmt * Update azure_* crates (#3503) * Fuzz coverage recording (#3322) * Fuzz coverage recording * Update cargo.toml * Update src/agent/coverage/fuzz/fuzz_targets/fuzz_target_record_coverage.rs Co-authored-by: George Pollard * Fix fuzz --------- Co-authored-by: George Pollard * Reporting coverage on task start up (#3502) * Reporting coverage on task start up * Moving metric up. * Remove feature flag from heartbeat metrics. (#3505) * Update archive notice. (#3507) * Add onefuzz service version to job created events (#3504) * Tevoinea/add version checking in local tasks (#3517) * Compare task version to service version * Swallow output when looking for appropriate name * Create directories if they don't exist in the template (#3522) * Create directories if they don't exist in the template * fmt * Support for retention policies on containers (#3501) - [x] ability to specify a retention period on a container, which applies to newly-created blobs - [x] specify default retention periods in templates from CLI side There's a small breaking change to the Python JobHelper class. * Bump rayon from 1.7.0 to 1.8.0 in /src/agent (#3520) Bumps [rayon](https://github.com/rayon-rs/rayon) from 1.7.0 to 1.8.0. - [Changelog](https://github.com/rayon-rs/rayon/blob/master/RELEASES.md) - [Commits](https://github.com/rayon-rs/rayon/compare/rayon-core-v1.7.0...rayon-core-v1.8.0) --- updated-dependencies: - dependency-name: rayon dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump insta from 1.31.0 to 1.32.0 in /src/agent (#3521) Bumps [insta](https://github.com/mitsuhiko/insta) from 1.31.0 to 1.32.0. - [Changelog](https://github.com/mitsuhiko/insta/blob/master/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/insta/compare/1.31.0...1.32.0) --- updated-dependencies: - dependency-name: insta dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Disable `repro` and `debug` VM CLI commands. (#3494) * Disable and VM CLI commands. * Formatting. * More formatting. * More formatting. * Removing Repro check. * Make modules case insenstive on windows (#3527) * Make modules and coverage allowlist case insensitive on Windows * Tests and fmt * PR comments * fmt * Debugging missing file coverage * fmt * Broken linux test * Add a case insensitive transformer for better perf * cargo fix * Update windows interceptor list (#3528) * Template creation command (#3531) * Tasks are selectable * Almost there * It works * fmt * remove dead code * Remove unnecessary comments * Improve instructions * fix bug * Add some dummy values for paths * Terminate process on timeout in windows for the coverage task (#3529) * Terminate process on timeout in windows for the coverage task * set the timeout before we start the debugger * split the target launch from the debugger initialization wait for the process to finish on a separate thread * fix build * move comments * Ignore regression update when the work item is in some states (#3532) * Ignore regression update when the work item is in some states * format * formatting * don't hide messages in the poison queue * fix typo * update regression logic update test_template to support regression * build fix * mypy fix * build fix * move regression ignore state under ADODuplicateTemplate * replace extend with append * update set_tcp_keepalive * mke mypy happy * copy ADODuplicateTemplate.OnDuplicate.RegressionIgnoreStates * Updating IterationCount to be Task-based. * Changing to machine_id based * Fixing repro event name. * Updating iteration pr. * Single entry results. * Retry. * trying with unique guid. * Generic string type. * putting it back. * removing old update code. * removing comment. * Attempting to partition on task_id, machine_id, and event type. * Using replace and update. * Add logging statement. * UPdating such that we query. * attempting to try update. * Trying different update mechanism. * Checking previous value. * cleanup. * Removing old model. * Case guard for better readability. * Fix import ordering. * Removing duplicate code. * accidentally didn't include update. * Constructing the wrong way. * Moving back to standard switch. * Removing old code. * Removing more old code. * Using constants. * Addressing comments. * Corrected. * Updating. * Comment. * Replacing with Update to deal with edge cases. * Using timestamp. * Adding CreatedAt field. * Comparing timestamps. * Setting propery. * Adding check for CreatedAt. * Making created_at optional. * Trying again. * Remove. * Add log statement. * Remove function handle.: * Updating * Concat names * Set version. --------- Signed-off-by: dependabot[bot] Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> Co-authored-by: Cheick Keita Co-authored-by: Kanan B <32438208+kananb@users.noreply.github.com> Co-authored-by: Marc Greisen Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: George Pollard Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com> Co-authored-by: George Pollard --- .../ApiService/Functions/QueueJobResult.cs | 9 +- .../ApiService/OneFuzzTypes/Model.cs | 49 +++----- .../onefuzzlib/JobResultOperations.cs | 118 +++++++----------- src/agent/Cargo.lock | 2 + src/agent/onefuzz-result/Cargo.toml | 6 +- src/agent/onefuzz-result/src/job_result.rs | 9 +- 6 files changed, 87 insertions(+), 106 deletions(-) diff --git a/src/ApiService/ApiService/Functions/QueueJobResult.cs b/src/ApiService/ApiService/Functions/QueueJobResult.cs index 5e3bec0048..31b39802d6 100644 --- a/src/ApiService/ApiService/Functions/QueueJobResult.cs +++ b/src/ApiService/ApiService/Functions/QueueJobResult.cs @@ -31,7 +31,12 @@ public async Async.Task Run([QueueTrigger("job-result", Connection = "AzureWebJo var job = await _jobs.Get(task.JobId); if (job == null) { - _log.LogWarning("invalid {JobId}", task.JobId); + _log.LogWarning("invalid message {JobId}", task.JobId); + return; + } + + if (jr.CreatedAt == null) { + _log.LogWarning("invalid message, no created_at field {JobId}", task.JobId); return; } @@ -52,7 +57,7 @@ public async Async.Task Run([QueueTrigger("job-result", Connection = "AzureWebJo return; } - var jobResult = await _context.JobResultOperations.CreateOrUpdate(job.JobId, jobResultType, value); + var jobResult = await _context.JobResultOperations.CreateOrUpdate(job.JobId, jr.TaskId, jr.MachineId, jr.CreatedAt.Value, jr.Version, jobResultType, value); if (!jobResult.IsOk) { _log.LogError("failed to create or update with job result {JobId}", job.JobId); } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index ab41853a74..4dd4000283 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -34,19 +34,6 @@ public enum HeartbeatType { TaskAlive, } -[SkipRename] -public enum JobResultType { - NewCrashingInput, - NoReproCrashingInput, - NewReport, - NewUniqueReport, - NewRegressionReport, - NewCoverage, - NewCrashDump, - CoverageData, - RuntimeStats, -} - public record HeartbeatData(HeartbeatType Type); public record TaskHeartbeatEntry( @@ -55,12 +42,14 @@ public record TaskHeartbeatEntry( [property: Required] Guid MachineId, HeartbeatData[] Data); -public record JobResultData(JobResultType Type); +public record JobResultData(string Type); public record TaskJobResultEntry( Guid TaskId, Guid? JobId, Guid MachineId, + DateTime? CreatedAt, + double Version, JobResultData Data, Dictionary Value ); @@ -921,26 +910,24 @@ public record SecretAddress(Uri Url) : ISecret { public record SecretData(ISecret Secret) { } +[SkipRename] +public enum JobResultType { + CoverageData, + RuntimeStats, +} + public record JobResult( - [PartitionKey][RowKey] Guid JobId, + [PartitionKey] Guid JobId, + [RowKey] string TaskIdMachineIdMetric, + Guid TaskId, + Guid MachineId, + DateTime CreatedAt, string Project, string Name, - double NewCrashingInput = 0, - double NoReproCrashingInput = 0, - double NewReport = 0, - double NewUniqueReport = 0, - double NewRegressionReport = 0, - double NewCrashDump = 0, - double InstructionsCovered = 0, - double TotalInstructions = 0, - double CoverageRate = 0, - double IterationCount = 0 -) : EntityBase() { - public JobResult(Guid JobId, string Project, string Name) : this( - JobId: JobId, - Project: Project, - Name: Name, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) { } -} + string Type, + double Version, + Dictionary MetricValue +) : EntityBase(); public record JobConfig( string Project, diff --git a/src/ApiService/ApiService/onefuzzlib/JobResultOperations.cs b/src/ApiService/ApiService/onefuzzlib/JobResultOperations.cs index 1166cf91d4..b39c654642 100644 --- a/src/ApiService/ApiService/onefuzzlib/JobResultOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/JobResultOperations.cs @@ -2,99 +2,75 @@ using Microsoft.Extensions.Logging; using Polly; namespace Microsoft.OneFuzz.Service; +using System.Net; public interface IJobResultOperations : IOrm { - Async.Task GetJobResult(Guid jobId); - Async.Task CreateOrUpdate(Guid jobId, JobResultType resultType, Dictionary resultValue); + Async.Task GetJobResult(Guid jobId, Guid taskId, Guid machineId, string metricType); + Async.Task CreateOrUpdate(Guid jobId, Guid taskId, Guid machineId, DateTime createdAt, double version, string resultType, Dictionary resultValue); } public class JobResultOperations : Orm, IJobResultOperations { + const string COVERAGE_DATA = "CoverageData"; + const string RUNTIME_STATS = "RuntimeStats"; + public JobResultOperations(ILogger log, IOnefuzzContext context) : base(log, context) { } - public async Async.Task GetJobResult(Guid jobId) { - return await SearchByPartitionKeys(new[] { jobId.ToString() }).SingleOrDefaultAsync(); + public async Async.Task GetJobResult(Guid jobId, Guid taskId, Guid machineId, string metricType) { + var data = QueryAsync(Query.SingleEntity(jobId.ToString(), string.Concat(taskId, "-", machineId, "-", metricType))); + return await data.FirstOrDefaultAsync(); } - private JobResult UpdateResult(JobResult result, JobResultType type, Dictionary resultValue) { - - var newResult = result; - double newValue; - switch (type) { - case JobResultType.NewCrashingInput: - newValue = result.NewCrashingInput + resultValue["count"]; - newResult = result with { NewCrashingInput = newValue }; - break; - case JobResultType.NewReport: - newValue = result.NewReport + resultValue["count"]; - newResult = result with { NewReport = newValue }; - break; - case JobResultType.NewUniqueReport: - newValue = result.NewUniqueReport + resultValue["count"]; - newResult = result with { NewUniqueReport = newValue }; - break; - case JobResultType.NewRegressionReport: - newValue = result.NewRegressionReport + resultValue["count"]; - newResult = result with { NewRegressionReport = newValue }; - break; - case JobResultType.NewCrashDump: - newValue = result.NewCrashDump + resultValue["count"]; - newResult = result with { NewCrashDump = newValue }; - break; - case JobResultType.CoverageData: - double newCovered = resultValue["covered"]; - double newTotalCovered = resultValue["features"]; - double newCoverageRate = resultValue["rate"]; - newResult = result with { InstructionsCovered = newCovered, TotalInstructions = newTotalCovered, CoverageRate = newCoverageRate }; - break; - case JobResultType.RuntimeStats: - double newTotalIterations = resultValue["total_count"]; - newResult = result with { IterationCount = newTotalIterations }; - break; - default: - _logTracer.LogWarning($"Invalid Field {type}."); - break; - } - _logTracer.LogInformation($"Attempting to log new result: {newResult}"); - return newResult; - } - - private async Async.Task TryUpdate(Job job, JobResultType resultType, Dictionary resultValue) { + private async Async.Task TryUpdate(Job job, Guid taskId, Guid machineId, DateTime createdAt, double version, string resultType, Dictionary resultValue) { var jobId = job.JobId; + var taskIdMachineIdMetric = string.Concat(taskId, "-", machineId, "-", resultType); - var jobResult = await GetJobResult(jobId); - - if (jobResult == null) { - _logTracer.LogInformation("Creating new JobResult for Job {JobId}", jobId); - - var entry = new JobResult(JobId: jobId, Project: job.Config.Project, Name: job.Config.Name); + var oldEntry = await GetJobResult(jobId, taskId, machineId, resultType); - jobResult = UpdateResult(entry, resultType, resultValue); - - var r = await Insert(jobResult); - if (!r.IsOk) { - throw new InvalidOperationException($"failed to insert job result {jobResult.JobId}"); + if (oldEntry == null) { + _logTracer.LogInformation($"attempt to insert new job result {taskId} and taskId+machineId+metricType {taskIdMachineIdMetric}"); + var newEntry = new JobResult(JobId: jobId, TaskIdMachineIdMetric: taskIdMachineIdMetric, TaskId: taskId, MachineId: machineId, CreatedAt: createdAt, Project: job.Config.Project, Name: job.Config.Name, resultType, Version: version, resultValue); + var result = await Insert(newEntry); + if (!result.IsOk) { + throw new InvalidOperationException($"failed to insert job result with taskId {taskId} and taskId+machineId+metricType {taskIdMachineIdMetric}"); } - _logTracer.LogInformation("created job result {JobId}", jobResult.JobId); - } else { - _logTracer.LogInformation("Updating existing JobResult entry for Job {JobId}", jobId); - - jobResult = UpdateResult(jobResult, resultType, resultValue); + return true; + } - var r = await Update(jobResult); - if (!r.IsOk) { - throw new InvalidOperationException($"failed to insert job result {jobResult.JobId}"); - } - _logTracer.LogInformation("updated job result {JobId}", jobResult.JobId); + ResultVoid<(HttpStatusCode Status, string Reason)> r; + switch (resultType) { + case COVERAGE_DATA: + case RUNTIME_STATS: + if (oldEntry.CreatedAt < createdAt) { + oldEntry = oldEntry with { CreatedAt = createdAt, MetricValue = resultValue }; + r = await Update(oldEntry); + if (!r.IsOk) { + throw new InvalidOperationException($"failed to replace job result with taskId {taskId} and machineId+metricType {taskIdMachineIdMetric}"); + } + } else { + _logTracer.LogInformation($"received an out-of-date metric. skipping."); + } + break; + default: + _logTracer.LogInformation($"attempt to update job result {taskId} and taskId+machineId+metricType {taskIdMachineIdMetric}"); + oldEntry.MetricValue["count"]++; + oldEntry = oldEntry with { MetricValue = oldEntry.MetricValue }; + r = await Update(oldEntry); + if (!r.IsOk) { + throw new InvalidOperationException($"failed to update job result with taskId {taskId} and machineId+metricType {taskIdMachineIdMetric}"); + } + break; } + return true; + } - public async Async.Task CreateOrUpdate(Guid jobId, JobResultType resultType, Dictionary resultValue) { + public async Async.Task CreateOrUpdate(Guid jobId, Guid taskId, Guid machineId, DateTime createdAt, double version, string resultType, Dictionary resultValue) { var job = await _context.JobOperations.Get(jobId); if (job == null) { @@ -106,7 +82,7 @@ public async Async.Task CreateOrUpdate(Guid jobId, JobResultT _logTracer.LogInformation("attempt to update job result {JobId}", job.JobId); var policy = Policy.Handle().WaitAndRetryAsync(50, _ => new TimeSpan(0, 0, 5)); await policy.ExecuteAsync(async () => { - success = await TryUpdate(job, resultType, resultValue); + success = await TryUpdate(job, taskId, machineId, createdAt, version, resultType, resultValue); _logTracer.LogInformation("attempt {success}", success); }); return OneFuzzResultVoid.Ok; diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 5393e9b767..e4143473e3 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -463,6 +463,7 @@ dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", + "serde", "winapi 0.3.9", ] @@ -2241,6 +2242,7 @@ version = "0.2.0" dependencies = [ "anyhow", "async-trait", + "chrono", "log", "onefuzz-telemetry", "reqwest", diff --git a/src/agent/onefuzz-result/Cargo.toml b/src/agent/onefuzz-result/Cargo.toml index 7c7de6615c..7e156ac91d 100644 --- a/src/agent/onefuzz-result/Cargo.toml +++ b/src/agent/onefuzz-result/Cargo.toml @@ -9,10 +9,14 @@ license = "MIT" [dependencies] anyhow = { version = "1.0", features = ["backtrace"] } async-trait = "0.1" +chrono = { version = "0.4", default-features = false, features = [ + "clock", + "std", + "serde" +] } reqwest = "0.11" serde = "1.0" storage-queue = { path = "../storage-queue" } uuid = { version = "1.4", features = ["serde", "v4"] } onefuzz-telemetry = { path = "../onefuzz-telemetry" } log = "0.4" - diff --git a/src/agent/onefuzz-result/src/job_result.rs b/src/agent/onefuzz-result/src/job_result.rs index 08f7bbc1ee..e6b4f50377 100644 --- a/src/agent/onefuzz-result/src/job_result.rs +++ b/src/agent/onefuzz-result/src/job_result.rs @@ -3,6 +3,8 @@ use anyhow::Result; use async_trait::async_trait; +use chrono::DateTime; +pub use chrono::Utc; use onefuzz_telemetry::warn; use reqwest::Url; use serde::{self, Deserialize, Serialize}; @@ -32,6 +34,8 @@ struct JobResult { job_id: Uuid, machine_id: Uuid, machine_name: String, + created_at: DateTime, + version: f64, data: JobResultData, value: HashMap, } @@ -103,7 +107,8 @@ impl JobResultSender for TaskJobResultClient { let job_id = self.context.state.job_id; let machine_id = self.context.state.machine_id; let machine_name = self.context.state.machine_name.clone(); - + let created_at = chrono::Utc::now(); + let version = 1.0; let _ = self .context .queue_client @@ -112,6 +117,8 @@ impl JobResultSender for TaskJobResultClient { job_id, machine_id, machine_name, + created_at, + version, data, value, }) From 5cf619ee8fac3062d63f855556015693a632bafa Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:48:18 -0700 Subject: [PATCH 3/6] Move initial coverage reporting to later in startup to capture read-only inputs (#3605) * add * Move intial coverage report farther into task start. * Keeping both calls. --- src/agent/onefuzz-task/src/tasks/coverage/generic.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs index 2ebc748010..8043deee94 100644 --- a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs @@ -146,11 +146,11 @@ impl CoverageTask { bail!("input is not specified on the command line or arguments for the target"); } - context.heartbeat.alive(); - info!("report initial coverage"); context.report_coverage_stats().await; + context.heartbeat.alive(); + for dir in &self.config.readonly_inputs { debug!("recording coverage for {}", dir.local_path.display()); @@ -174,6 +174,9 @@ impl CoverageTask { context.save_and_sync_coverage().await?; } + info!("report coverage"); + context.report_coverage_stats().await; + context.heartbeat.alive(); if let Some(queue) = &self.config.input_queue { From 4a67a4829ae7642f1a5eb902b6f7dea64817d8eb Mon Sep 17 00:00:00 2001 From: Stas Date: Fri, 27 Oct 2023 07:08:49 -0700 Subject: [PATCH 4/6] do not expose secrets as part of deployment output during azure deployment (#3578) * do not expose secrets during deployment * remove commented out code * was passing wrong connection string --------- Co-authored-by: stas --- src/deployment/azuredeploy.bicep | 20 ++++++++++------ .../bicep-templates/function-settings.bicep | 24 ++++++++++++------- src/deployment/bicep-templates/function.bicep | 21 +++++++++++++--- src/deployment/bicep-templates/signalR.bicep | 3 +-- .../bicep-templates/storageAccounts.bicep | 19 --------------- 5 files changed, 48 insertions(+), 39 deletions(-) diff --git a/src/deployment/azuredeploy.bicep b/src/deployment/azuredeploy.bicep index 99c175c6e7..ac735af952 100644 --- a/src/deployment/azuredeploy.bicep +++ b/src/deployment/azuredeploy.bicep @@ -156,7 +156,6 @@ module storage 'bicep-templates/storageAccounts.bicep' = { params: { location: location owner: owner - signedExpiry: signedExpiry } } @@ -172,6 +171,7 @@ module autoscaleSettings 'bicep-templates/autoscale-settings.bicep' = { } } + module eventGrid 'bicep-templates/event-grid.bicep' = { name: 'event-grid' params: { @@ -227,8 +227,8 @@ module function 'bicep-templates/function.bicep' = { params: { name: name linux_fx_version: 'DOTNET-ISOLATED|7.0' - - app_logs_sas_url: storage.outputs.FuncSasUrlBlobAppLogs + signedExpiry: signedExpiry + logs_storage: storage.outputs.FuncName app_func_audiences: app_func_audiences app_func_issuer: app_func_issuer client_id: clientId @@ -241,6 +241,9 @@ module function 'bicep-templates/function.bicep' = { use_windows: true enable_remote_debugging: enable_remote_debugging } + dependsOn:[ + storage + ] } module functionSettings 'bicep-templates/function-settings.bicep' = { @@ -254,8 +257,9 @@ module functionSettings 'bicep-templates/function-settings.bicep' = { app_insights_app_id: operationalInsights.outputs.appInsightsAppId app_insights_key: operationalInsights.outputs.appInsightsInstrumentationKey client_secret: clientSecret - signal_r_connection_string: signalR.outputs.connectionString - func_sas_url: storage.outputs.FuncSasUrl + + signalRName: signalR.outputs.signalRName + funcStorageName: storage.outputs.FuncName func_storage_resource_id: storage.outputs.FuncId fuzz_storage_resource_id: storage.outputs.FuzzId keyvault_name: keyVaultName @@ -269,16 +273,18 @@ module functionSettings 'bicep-templates/function-settings.bicep' = { } dependsOn: [ function + storage + signalR ] } output fuzz_storage string = storage.outputs.FuzzId output fuzz_name string = storage.outputs.FuzzName -output fuzz_key string = storage.outputs.FuzzKey + output func_storage string = storage.outputs.FuncId output func_name string = storage.outputs.FuncName -output func_key string = storage.outputs.FuncKey + output scaleset_identity string = scaleset_identity output tenant_id string = tenantId diff --git a/src/deployment/bicep-templates/function-settings.bicep b/src/deployment/bicep-templates/function-settings.bicep index 742f4f39d5..8235dcbf7a 100644 --- a/src/deployment/bicep-templates/function-settings.bicep +++ b/src/deployment/bicep-templates/function-settings.bicep @@ -5,17 +5,11 @@ param app_insights_app_id string @secure() param app_insights_key string -@secure() -param func_sas_url string - param cli_app_id string param authority string param tenant_domain string param multi_tenant_domain string -@secure() -param signal_r_connection_string string - param app_config_endpoint string param func_storage_resource_id string @@ -33,8 +27,21 @@ param functions_extension_version string param enable_profiler bool +param signalRName string +param funcStorageName string + var telemetry = 'd7a73cf4-5a1a-4030-85e1-e5b25867e45a' + +resource signal_r 'Microsoft.SignalRService/signalR@2021-10-01' existing = { + name: signalRName +} + + +resource funcStorage 'Microsoft.Storage/storageAccounts@2021-08-01' existing = { + name: funcStorageName +} + resource function 'Microsoft.Web/sites@2021-02-01' existing = { name: name } @@ -44,6 +51,7 @@ var enable_profilers = enable_profiler ? { DiagnosticServices_EXTENSION_VERSION: '~3' } : {} +var func_key = funcStorage.listKeys().keys[0].value resource functionSettings 'Microsoft.Web/sites/config@2021-03-01' = { parent: function name: 'appsettings' @@ -54,13 +62,13 @@ resource functionSettings 'Microsoft.Web/sites/config@2021-03-01' = { APPINSIGHTS_INSTRUMENTATIONKEY: app_insights_key APPINSIGHTS_APPID: app_insights_app_id ONEFUZZ_TELEMETRY: telemetry - AzureWebJobsStorage: func_sas_url + AzureWebJobsStorage: 'DefaultEndpointsProtocol=https;AccountName=${funcStorage.name};AccountKey=${func_key};EndpointSuffix=core.windows.net' CLI_APP_ID: cli_app_id AUTHORITY: authority TENANT_DOMAIN: tenant_domain MULTI_TENANT_DOMAIN: multi_tenant_domain AzureWebJobsDisableHomepage: 'true' - AzureSignalRConnectionString: signal_r_connection_string + AzureSignalRConnectionString: signal_r.listKeys().primaryConnectionString AzureSignalRServiceTransportType: 'Transient' APPCONFIGURATION_ENDPOINT: app_config_endpoint ONEFUZZ_INSTANCE_NAME: instance_name diff --git a/src/deployment/bicep-templates/function.bicep b/src/deployment/bicep-templates/function.bicep index a6e695ffbf..bff457f2cd 100644 --- a/src/deployment/bicep-templates/function.bicep +++ b/src/deployment/bicep-templates/function.bicep @@ -9,8 +9,9 @@ param app_func_audiences array param use_windows bool param enable_remote_debugging bool -@secure() -param app_logs_sas_url string +param logs_storage string +param signedExpiry string + @description('The degree of severity for diagnostics logs.') @allowed([ @@ -28,6 +29,14 @@ var siteconfig = (use_windows) ? { linuxFxVersion: linux_fx_version } +var storage_account_sas = { + signedExpiry: signedExpiry + signedPermission: 'rwdlacup' + signedResourceTypes: 'sco' + signedServices: 'bfqt' +} + + var commonSiteConfig = { alwaysOn: true defaultDocuments: [] @@ -45,6 +54,11 @@ var extraProperties = (use_windows && enable_remote_debugging) ? { remoteDebuggingVersion: 'VS2022' } : {} +resource funcStorage 'Microsoft.Storage/storageAccounts@2021-08-01' existing = { + name: logs_storage +} + + resource function 'Microsoft.Web/sites@2021-03-01' = { name: name location: location @@ -97,6 +111,7 @@ resource funcAuthSettings 'Microsoft.Web/sites/config@2021-03-01' = { parent: function } +var sas = funcStorage.listAccountSas('2021-08-01', storage_account_sas) resource funcLogs 'Microsoft.Web/sites/config@2021-03-01' = { name: 'logs' properties: { @@ -104,7 +119,7 @@ resource funcLogs 'Microsoft.Web/sites/config@2021-03-01' = { azureBlobStorage: { level: diagnostics_log_level retentionInDays: log_retention - sasUrl: app_logs_sas_url + sasUrl: '${funcStorage.properties.primaryEndpoints.blob}app-logs?${sas.accountSasToken}' } } } diff --git a/src/deployment/bicep-templates/signalR.bicep b/src/deployment/bicep-templates/signalR.bicep index b054f7ed12..ef393e4427 100644 --- a/src/deployment/bicep-templates/signalR.bicep +++ b/src/deployment/bicep-templates/signalR.bicep @@ -30,5 +30,4 @@ resource signalR 'Microsoft.SignalRService/signalR@2021-10-01' = { } } -var connectionString = signalR.listKeys().primaryConnectionString -output connectionString string = connectionString +output signalRName string = signalr_name diff --git a/src/deployment/bicep-templates/storageAccounts.bicep b/src/deployment/bicep-templates/storageAccounts.bicep index 27f2da21d8..530fcc436c 100644 --- a/src/deployment/bicep-templates/storageAccounts.bicep +++ b/src/deployment/bicep-templates/storageAccounts.bicep @@ -1,18 +1,10 @@ param owner string param location string -param signedExpiry string var suffix = uniqueString(resourceGroup().id) var storageAccountNameFuzz = 'fuzz${suffix}' var storageAccountNameFunc = 'func${suffix}' -var storage_account_sas = { - signedExpiry: signedExpiry - signedPermission: 'rwdlacup' - signedResourceTypes: 'sco' - signedServices: 'bfqt' -} - var storageAccountFuzzContainersParams = [ 'events' ] @@ -119,14 +111,3 @@ output FuzzId string = storageAccountFuzz.id output FuncId string = storageAccountFunc.id output FileChangesQueueName string = storageAccountFuncQueuesParams[fileChangesQueueIndex] - -var sas = storageAccountFunc.listAccountSas('2021-08-01', storage_account_sas) -output FuncSasUrlBlobAppLogs string = '${storageAccountFunc.properties.primaryEndpoints.blob}app-logs?${sas.accountSasToken}' - -var fuzz_key = storageAccountFuzz.listKeys().keys[0].value -output FuzzKey string = fuzz_key - -var func_key = storageAccountFunc.listKeys().keys[0].value -output FuncKey string = func_key - -output FuncSasUrl string = 'DefaultEndpointsProtocol=https;AccountName=${storageAccountFunc.name};AccountKey=${func_key};EndpointSuffix=core.windows.net' From 72d775f4455765c0c42ca779092e4237e8c2d699 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Fri, 27 Oct 2023 13:44:27 -0700 Subject: [PATCH 5/6] Emit `CrashReported` Metric to App insights. (#3571) * add * Emit CrashReported Metric to App insights. * Fixing. * removing? --- src/agent/onefuzz-task/src/tasks/report/crash_report.rs | 4 +++- src/agent/onefuzz-telemetry/src/lib.rs | 2 ++ src/proxy-manager/Cargo.lock | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/agent/onefuzz-task/src/tasks/report/crash_report.rs b/src/agent/onefuzz-task/src/tasks/report/crash_report.rs index 9ae618ce93..407d449c0f 100644 --- a/src/agent/onefuzz-task/src/tasks/report/crash_report.rs +++ b/src/agent/onefuzz-task/src/tasks/report/crash_report.rs @@ -6,7 +6,7 @@ use onefuzz::{blob::BlobUrl, monitor::DirectoryMonitor, syncdir::SyncedDir}; use onefuzz_result::job_result::{JobResultData, JobResultSender, TaskJobResultClient}; use onefuzz_telemetry::{ Event::{ - new_report, new_unable_to_reproduce, new_unique_report, regression_report, + crash_reported, new_report, new_unable_to_reproduce, new_unique_report, regression_report, regression_unable_to_reproduce, }, EventData, @@ -166,6 +166,8 @@ impl CrashTestResult { match self { Self::CrashReport(report) => { // Use SHA-256 of call stack as dedupe key. + event!(crash_reported; EventData::Path = report.unique_blob_name()); + metric!(crash_reported; 1.0; EventData::Path = report.unique_blob_name()); if let Some(jr_client) = jr_client { let _ = jr_client .send_direct( diff --git a/src/agent/onefuzz-telemetry/src/lib.rs b/src/agent/onefuzz-telemetry/src/lib.rs index f08b722695..21e427f06f 100644 --- a/src/agent/onefuzz-telemetry/src/lib.rs +++ b/src/agent/onefuzz-telemetry/src/lib.rs @@ -82,6 +82,7 @@ pub enum Event { runtime_stats, new_report, new_unique_report, + crash_reported, new_unable_to_reproduce, regression_report, regression_unable_to_reproduce, @@ -99,6 +100,7 @@ impl Event { Self::runtime_stats => "runtime_stats", Self::new_report => "new_report", Self::new_unique_report => "new_unique_report", + Self::crash_reported => "crash_reported", Self::new_unable_to_reproduce => "new_unable_to_reproduce", Self::regression_report => "regression_report", Self::regression_unable_to_reproduce => "regression_unable_to_reproduce", diff --git a/src/proxy-manager/Cargo.lock b/src/proxy-manager/Cargo.lock index 98e66a2468..e48d028374 100644 --- a/src/proxy-manager/Cargo.lock +++ b/src/proxy-manager/Cargo.lock @@ -1203,9 +1203,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ "bitflags 2.4.0", "errno", From d50fd48a8b36cd0f902d79457b8dcd3d56832c8b Mon Sep 17 00:00:00 2001 From: Teo Voinea <58236992+tevoinea@users.noreply.github.com> Date: Mon, 30 Oct 2023 13:21:33 -0400 Subject: [PATCH 6/6] Scoped notification pause (#3579) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Laying groundwork * Almost there * CLI updates * Remove unused code * cleanup * Broken test * fmt * . * 🥹 * forgot a file * Move from PUT to PATCH --- .../ApiService/Functions/Containers.cs | 20 +++++++++++- .../ApiService/Functions/QueueFileChanges.cs | 31 +++++++++++++------ .../ApiService/OneFuzzTypes/Enums.cs | 3 ++ .../ApiService/OneFuzzTypes/Requests.cs | 5 +++ .../ApiService/onefuzzlib/Containers.cs | 28 +++++++++++++++++ .../onefuzzlib/NotificationOperations.cs | 8 +++-- .../ApiService/onefuzzlib/Reports.cs | 1 - src/cli/onefuzz/api.py | 11 +++++++ src/pytypes/onefuzztypes/enums.py | 2 ++ src/pytypes/onefuzztypes/requests.py | 5 +++ 10 files changed, 100 insertions(+), 14 deletions(-) diff --git a/src/ApiService/ApiService/Functions/Containers.cs b/src/ApiService/ApiService/Functions/Containers.cs index 96554c880e..5178f5be0f 100644 --- a/src/ApiService/ApiService/Functions/Containers.cs +++ b/src/ApiService/ApiService/Functions/Containers.cs @@ -16,11 +16,12 @@ public ContainersFunction(ILogger logger, IOnefuzzContext co [Function("Containers")] [Authorize(Allow.User)] - public Async.Task Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET", "POST", "DELETE")] HttpRequestData req) + public Async.Task Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET", "POST", "PATCH", "DELETE")] HttpRequestData req) => req.Method switch { "GET" => Get(req), "POST" => Post(req), "DELETE" => Delete(req), + "PATCH" => Patch(req), _ => throw new NotSupportedException(), }; @@ -108,4 +109,21 @@ private async Async.Task Post(HttpRequestData req) { SasUrl: sas, Metadata: post.Metadata)); } + + private async Async.Task Patch(HttpRequestData req) { + var request = await RequestHandling.ParseRequest(req); + if (!request.IsOk) { + return await _context.RequestHandling.NotOk(req, request.ErrorV, context: "container update"); + } + + var toUpdate = request.OkV; + _logger.LogInformation("updating {ContainerName}", toUpdate.Name); + var updated = await _context.Containers.CreateOrUpdateContainerTag(toUpdate.Name, StorageType.Corpus, toUpdate.Metadata.ToDictionary(x => x.Key, x => x.Value)); + + if (!updated.IsOk) { + return await _context.RequestHandling.NotOk(req, updated.ErrorV, "container update"); + } + + return await RequestHandling.Ok(req, new ContainerInfoBase(toUpdate.Name, toUpdate.Metadata)); + } } diff --git a/src/ApiService/ApiService/Functions/QueueFileChanges.cs b/src/ApiService/ApiService/Functions/QueueFileChanges.cs index 8ef77bd2a5..f2aa08b306 100644 --- a/src/ApiService/ApiService/Functions/QueueFileChanges.cs +++ b/src/ApiService/ApiService/Functions/QueueFileChanges.cs @@ -2,6 +2,8 @@ using System.Text.Json.Nodes; using System.Threading.Tasks; using Azure.Core; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; @@ -60,7 +62,11 @@ public async Async.Task Run( try { var result = await FileAdded(storageAccount, fileChangeEvent); if (!result.IsOk) { - await RequeueMessage(msg, result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED ? TimeSpan.FromDays(1) : null); + if (result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) { + await RequeueMessage(msg, TimeSpan.FromDays(1), incrementDequeueCount: false); + } else { + await RequeueMessage(msg); + } } } catch (Exception e) { _log.LogError(e, "File Added failed"); @@ -83,21 +89,26 @@ private async Async.Task FileAdded(ResourceIdentifier storage _log.LogInformation("file added : {Container} - {Path}", container.String, path); + var account = await _storage.GetBlobServiceClientForAccount(storageAccount); + var containerClient = account.GetBlobContainerClient(container.String); + var containerProps = await containerClient.GetPropertiesAsync(); + + if (_context.NotificationOperations.ShouldPauseNotificationsForContainer(containerProps.Value.Metadata)) { + return Error.Create(ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED, $"container {container} has a metadata tag set to pause notifications processing"); + } + var (_, result) = await ( - ApplyRetentionPolicy(storageAccount, container, path), + ApplyRetentionPolicy(containerClient, containerProps, path), _notificationOperations.NewFiles(container, path)); return result; } - private async Async.Task ApplyRetentionPolicy(ResourceIdentifier storageAccount, Container container, string path) { + private async Async.Task ApplyRetentionPolicy(BlobContainerClient containerClient, BlobContainerProperties containerProps, string path) { if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableContainerRetentionPolicies)) { // default retention period can be applied to the container // if one exists, we will set the expiry date on the newly-created blob, if it doesn't already have one - var account = await _storage.GetBlobServiceClientForAccount(storageAccount); - var containerClient = account.GetBlobContainerClient(container.String); - var containerProps = await containerClient.GetPropertiesAsync(); - var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Value.Metadata); + var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Metadata); if (!retentionPeriod.IsOk) { _log.LogError("invalid retention period: {Error}", retentionPeriod.ErrorV); } else if (retentionPeriod.OkV is TimeSpan period) { @@ -116,7 +127,7 @@ private async Async.Task ApplyRetentionPolicy(ResourceIdentifier storageAc return false; } - private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) { + private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null, bool incrementDequeueCount = true) { var json = JsonNode.Parse(msg); // Messages that are 'manually' requeued by us as opposed to being requeued by the azure functions runtime @@ -135,7 +146,9 @@ await _context.Queue.QueueObject( StorageType.Config) .IgnoreResult(); } else { - json!["data"]!["customDequeueCount"] = newCustomDequeueCount + 1; + if (incrementDequeueCount) { + json!["data"]!["customDequeueCount"] = newCustomDequeueCount + 1; + } await _context.Queue.QueueObject( QueueFileChangesQueueName, json, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index 5a8b22527d..c0e3c68eba 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -53,6 +53,9 @@ public enum ErrorCode { INVALID_RETENTION_PERIOD = 497, INVALID_CLI_VERSION = 498, TRANSIENT_NOTIFICATION_FAILURE = 499, + + FAILED_CONTAINER_PROPERTIES_ACCESS = 500, + FAILED_SAVING_CONTAINER_METADATA = 501, // NB: if you update this enum, also update enums.py } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index db63499d30..f3a4c32965 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -128,6 +128,11 @@ public record ContainerDelete( IDictionary? Metadata = null ) : BaseRequest; +public record ContainerUpdate( + [property: Required] Container Name, + [property: Required] IDictionary Metadata +) : BaseRequest; + public record NotificationCreate( [property: Required] Container Container, [property: Required] bool ReplaceExisting, diff --git a/src/ApiService/ApiService/onefuzzlib/Containers.cs b/src/ApiService/ApiService/onefuzzlib/Containers.cs index f7bb3086a0..e004815abc 100644 --- a/src/ApiService/ApiService/onefuzzlib/Containers.cs +++ b/src/ApiService/ApiService/onefuzzlib/Containers.cs @@ -1,5 +1,6 @@ using System.IO; using System.IO.Compression; +using System.Text.Json; using System.Threading.Tasks; using ApiService.OneFuzzLib.Orm; using Azure; @@ -41,6 +42,8 @@ public interface IContainers { public Async.Task DownloadAsZip(Container container, StorageType storageType, Stream stream, string? prefix = null); public Async.Task DeleteAllExpiredBlobs(); + + public Async.Task CreateOrUpdateContainerTag(Container container, StorageType storageType, Dictionary tags); } public class Containers : Orm, IContainers { @@ -448,4 +451,29 @@ private async Async.Task DeleteExpiredBlobsForAccount(ResourceIdentifier storage } } } + + public async Task CreateOrUpdateContainerTag(Container container, StorageType storageType, Dictionary tags) { + var client = await FindContainer(container, storageType); + if (client is null || !await client.ExistsAsync()) { + return Error.Create(ErrorCode.INVALID_CONTAINER, $"Could not find container {container} in {storageType}"); + } + + var metadataRequest = await client.GetPropertiesAsync(); + if (metadataRequest is null || metadataRequest.GetRawResponse().IsError) { + return Error.Create(ErrorCode.FAILED_CONTAINER_PROPERTIES_ACCESS, $"Could not access container properties for container: {container} in {storageType}"); + } + + var metadata = metadataRequest.Value.Metadata ?? new Dictionary(); + + foreach (var kvp in tags) { + metadata[kvp.Key] = kvp.Value; + } + + var saveMetadataRequest = await client.SetMetadataAsync(metadata); + if (saveMetadataRequest is null || saveMetadataRequest.GetRawResponse().IsError) { + return Error.Create(ErrorCode.FAILED_SAVING_CONTAINER_METADATA, $"Could not save metadata to container: {container} in {storageType}. Metadata: {JsonSerializer.Serialize(metadata)}"); + } + + return OneFuzzResultVoid.Ok; + } } diff --git a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs index 5744d6d2f5..67369989e1 100644 --- a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs @@ -11,12 +11,11 @@ public interface INotificationOperations : IOrm { Async.Task> Create(Container container, NotificationTemplate config, bool replaceExisting); Async.Task GetNotification(Guid notifificationId); - System.Threading.Tasks.Task TriggerNotification(Container container, - Notification notification, IReport? reportOrRegression); + System.Threading.Tasks.Task TriggerNotification(Container container, Notification notification, IReport? reportOrRegression); + bool ShouldPauseNotificationsForContainer(IDictionary containerMetadata); } public class NotificationOperations : Orm, INotificationOperations { - public NotificationOperations(ILogger log, IOnefuzzContext context) : base(log, context) { @@ -190,4 +189,7 @@ private async Async.Task HideSecrets(NotificationTemplate public async Async.Task GetNotification(Guid notifificationId) { return await SearchByPartitionKeys(new[] { notifificationId.ToString() }).SingleOrDefaultAsync(); } + + private const string PAUSE_NOTIFICATIONS_TAG = "pauseNotifications"; + public bool ShouldPauseNotificationsForContainer(IDictionary containerMetadata) => containerMetadata.ContainsKey(PAUSE_NOTIFICATIONS_TAG) && containerMetadata[PAUSE_NOTIFICATIONS_TAG] == "true"; } diff --git a/src/ApiService/ApiService/onefuzzlib/Reports.cs b/src/ApiService/ApiService/onefuzzlib/Reports.cs index fdda7259e9..ac2e1029b2 100644 --- a/src/ApiService/ApiService/onefuzzlib/Reports.cs +++ b/src/ApiService/ApiService/onefuzzlib/Reports.cs @@ -67,7 +67,6 @@ public Reports(ILogger log, IContainers containers) { } private static T? TryDeserialize(string content) where T : class { - try { return JsonSerializer.Deserialize(content, EntityConverter.GetJsonSerializerOptions()); } catch (JsonException) { diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 64cad8c368..4f4e152484 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -486,6 +486,17 @@ def delete(self, name: str) -> responses.BoolResult: "DELETE", responses.BoolResult, data=requests.ContainerDelete(name=name) ) + def update( + self, name: str, metadata: Dict[str, str] + ) -> responses.ContainerInfoBase: + """Update a container's metadata""" + self.logger.debug("update container: %s", name) + return self._req_model( + "PATCH", + responses.ContainerInfoBase, + data=requests.ContainerUpdate(name=name, metadata=metadata), + ) + def list(self) -> List[responses.ContainerInfoBase]: """Get a list of containers""" self.logger.debug("list containers") diff --git a/src/pytypes/onefuzztypes/enums.py b/src/pytypes/onefuzztypes/enums.py index 446193f1d2..14315ddb5d 100644 --- a/src/pytypes/onefuzztypes/enums.py +++ b/src/pytypes/onefuzztypes/enums.py @@ -307,6 +307,8 @@ class ErrorCode(Enum): INVALID_RETENTION_PERIOD = 497 INVALID_CLI_VERSION = 498 TRANSIENT_NOTIFICATION_FAILURE = 499 + FAILED_CONTAINER_PROPERTIES_ACCESS = 500 + FAILED_SAVING_CONTAINER_METADATA = 501 # NB: if you update this enum, also update Enums.cs diff --git a/src/pytypes/onefuzztypes/requests.py b/src/pytypes/onefuzztypes/requests.py index d284fb416d..df9fb3e1f3 100644 --- a/src/pytypes/onefuzztypes/requests.py +++ b/src/pytypes/onefuzztypes/requests.py @@ -220,6 +220,11 @@ class ContainerDelete(BaseRequest): metadata: Optional[Dict[str, str]] +class ContainerUpdate(BaseRequest): + name: Container + metadata: Dict[str, str] + + class ReproGet(BaseRequest): vm_id: Optional[UUID]