Skip to content
This repository has been archived by the owner on Dec 11, 2023. It is now read-only.

Commit

Permalink
Merge pull request #4 from DeterminateSystems/runcommand/dynamic-guar…
Browse files Browse the repository at this point in the history
…ding

Dynamic RunCommand guarding
  • Loading branch information
grahamc authored Jan 25, 2022
2 parents 4b3d3d6 + ca44bb7 commit 64952e7
Show file tree
Hide file tree
Showing 14 changed files with 464 additions and 18 deletions.
5 changes: 3 additions & 2 deletions doc/manual/src/plugins/RunCommand.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@ Command to run. Can use the `$HYDRA_JSON` environment variable to access informa

### Dynamic Commands

Hydra can optionally run RunCommand hooks defined dynamically by the jobset.
This must be turned on explicitly in the `hydra.conf` and per jobset.
Hydra can optionally run RunCommand hooks defined dynamically by the jobset. In
order to enable dynamic commands, you must enable this feature in your
`hydra.conf`, *as well as* in the parent project and jobset configuration.

#### Behavior

Expand Down
3 changes: 3 additions & 0 deletions hydra-api.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,9 @@ paths:
enabled:
description: when set to true the project gets scheduled for evaluation
type: boolean
enable_dynamic_run_command:
description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
type: boolean
visible:
description: when set to true the project is displayed in the web interface
type: boolean
Expand Down
10 changes: 9 additions & 1 deletion src/lib/Hydra/Controller/Jobset.pm
Original file line number Diff line number Diff line change
Expand Up @@ -261,14 +261,22 @@ sub updateJobset {

my $checkinterval = int(trim($c->stash->{params}->{checkinterval}));

my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($c->config->{dynamicruncommand}->{enable}
&& $jobset->project->enable_dynamic_run_command))
{
badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project.");
}

$jobset->update(
{ name => $jobsetName
, description => trim($c->stash->{params}->{"description"})
, nixexprpath => $nixExprPath
, nixexprinput => $nixExprInput
, enabled => $enabled
, enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
, enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
, enable_dynamic_run_command => $enable_dynamic_run_command
, emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))
Expand Down
7 changes: 6 additions & 1 deletion src/lib/Hydra/Controller/Project.pm
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,11 @@ sub updateProject {
my $displayName = trim $c->stash->{params}->{displayname};
error($c, "You must specify a display name.") if $displayName eq "";

my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) {
badRequest($c, "Dynamic RunCommand is not enabled by the server.");
}

$project->update(
{ name => $projectName
, displayname => $displayName
Expand All @@ -157,7 +162,7 @@ sub updateProject {
, enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, owner => $owner
, enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
, enable_dynamic_run_command => $enable_dynamic_run_command
, declfile => trim($c->stash->{params}->{declarative}->{file})
, decltype => trim($c->stash->{params}->{declarative}->{type})
, declvalue => trim($c->stash->{params}->{declarative}->{value})
Expand Down
44 changes: 35 additions & 9 deletions src/lib/Hydra/Helper/AddBuilds.pm
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils;

our @ISA = qw(Exporter);
our @EXPORT = qw(
validateDeclarativeJobset
createJobsetInputsRowAndData
updateDeclarativeJobset
handleDeclarativeJobsetBuild
handleDeclarativeJobsetJson
);


sub updateDeclarativeJobset {
my ($db, $project, $jobsetName, $declSpec) = @_;
sub validateDeclarativeJobset {
my ($config, $project, $jobsetName, $declSpec) = @_;

my @allowed_keys = qw(
enabled
Expand Down Expand Up @@ -62,16 +64,39 @@ sub updateDeclarativeJobset {
}
}

my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($config->{dynamicruncommand}->{enable}
&& $project->{enable_dynamic_run_command}))
{
die "Dynamic RunCommand is not enabled by the server or the parent project.";
}

return %update;
}

sub createJobsetInputsRowAndData {
my ($name, $declSpec) = @_;
my $data = $declSpec->{"inputs"}->{$name};
my $row = {
name => $name,
type => $data->{type}
};
$row->{emailresponsible} = $data->{emailresponsible} // 0;

return ($row, $data);
}

sub updateDeclarativeJobset {
my ($config, $db, $project, $jobsetName, $declSpec) = @_;

my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec);

$db->txn_do(sub {
my $jobset = $project->jobsets->update_or_create(\%update);
$jobset->jobsetinputs->delete;
foreach my $name (keys %{$declSpec->{"inputs"}}) {
my $data = $declSpec->{"inputs"}->{$name};
my $row = {
name => $name,
type => $data->{type}
};
$row->{emailresponsible} = $data->{emailresponsible} // 0;
my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec);
my $input = $jobset->jobsetinputs->create($row);
$input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
}
Expand All @@ -82,14 +107,15 @@ sub updateDeclarativeJobset {

sub handleDeclarativeJobsetJson {
my ($db, $project, $declSpec) = @_;
my $config = getHydraConfig();
$db->txn_do(sub {
my @kept = keys %$declSpec;
push @kept, ".jobsets";
$project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 });
foreach my $jobsetName (keys %$declSpec) {
my $spec = $declSpec->{$jobsetName};
eval {
updateDeclarativeJobset($db, $project, $jobsetName, $spec);
updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec);
1;
} or do {
print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";
Expand Down
10 changes: 9 additions & 1 deletion src/root/edit-jobset.tt
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,15 @@
<div class="form-group row">
<label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label>
<div class="col-sm-9">
<input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
<input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command"
[% IF !c.config.dynamicruncommand.enable %]
title="The server has not enabled dynamic RunCommands" disabled
[% ELSIF !project.enable_dynamic_run_command %]
title="The parent project has not enabled dynamic RunCommands" disabled
[% ELSIF jobset.enable_dynamic_run_command %]
checked
[% END %]
/>
</div>
</div>

Expand Down
8 changes: 7 additions & 1 deletion src/root/edit-project.tt
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,13 @@
<div class="form-group row">
<label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label>
<div class="col-sm-9">
<input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
<input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command"
[% IF !c.config.dynamicruncommand.enable %]
title="The server has not enabled dynamic RunCommands" disabled
[% ELSIF project.enable_dynamic_run_command %]
checked
[% END %]
/>
</div>
</div>

Expand Down
2 changes: 1 addition & 1 deletion src/root/jobset.tt
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@
</tr>
<tr>
<th>Enable Dynamic RunCommand Hooks:</th>
<td>[% jobset.enable_dynamic_run_command ? "Yes" : "No" %]</td>
<td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? jobset.enable_dynamic_run_command ? "Yes" : "No (not enabled by jobset)" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
</tr>
[% IF emailNotification %]
<tr>
Expand Down
2 changes: 1 addition & 1 deletion src/root/project.tt
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@
</tr>
<tr>
<th>Enable Dynamic RunCommand Hooks:</th>
<td>[% project.enable_dynamic_run_command ? "Yes" : "No" %]</td>
<td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? "Yes" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
</tr>
</table>
</div>
Expand Down
2 changes: 1 addition & 1 deletion src/script/hydra-eval-jobset
Original file line number Diff line number Diff line change
Expand Up @@ -617,7 +617,7 @@ sub checkJobsetWrapped {
} else {
# Update the jobset with the spec's inputs, and the continue
# evaluating the .jobsets jobset.
updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
updateDeclarativeJobset($config, $db, $project, ".jobsets", $declSpec);
$jobset->discard_changes;
$inputInfo->{"declInput"} = [ $declInput ];
$inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ];
Expand Down
85 changes: 85 additions & 0 deletions t/Helper/AddBuilds/dynamic-disabled.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
use strict;
use warnings;
use Setup;
use Test2::V0;

require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
use Hydra::Helper::Nix qw(getHydraConfig);

my $ctx = test_context();

sub makeJobsetSpec {
my ($dynamic) = @_;

return {
enabled => 2,
enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
visible => JSON::MaybeXS::true,
name => "job",
type => 1,
description => "test jobset",
flake => "github:nixos/nix",
checkinterval => 0,
schedulingshares => 100,
keepnr => 3
};
};

subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub {
my $config = getHydraConfig();

subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
like(
dies {
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true),
),
},
qr/Dynamic RunCommand is not enabled/,
);
};

subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};

subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
like(
dies {
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true),
),
},
qr/Dynamic RunCommand is not enabled/,
);
};

subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
};

done_testing;
88 changes: 88 additions & 0 deletions t/Helper/AddBuilds/dynamic-enabled.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
use strict;
use warnings;
use Setup;
use Test2::V0;

require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
use Hydra::Helper::Nix qw(getHydraConfig);

my $ctx = test_context(
hydra_config => q|
<dynamicruncommand>
enable = 1
</dynamicruncommand>
|
);

sub makeJobsetSpec {
my ($dynamic) = @_;

return {
enabled => 2,
enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
visible => JSON::MaybeXS::true,
name => "job",
type => 1,
description => "test jobset",
flake => "github:nixos/nix",
checkinterval => 0,
schedulingshares => 100,
keepnr => 3
};
};

subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub {
my $config = getHydraConfig();

subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true)
),
);
};

subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};

subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
like(
dies {
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true),
),
},
qr/Dynamic RunCommand is not enabled/,
);
};

subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
};

done_testing;
Loading

0 comments on commit 64952e7

Please sign in to comment.