Skip to content
This repository has been archived by the owner on Dec 11, 2023. It is now read-only.
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: DeterminateSystems/hydra
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 34be56db67b1340e20a56bc3a1bfc87d113abb8b
Choose a base ref
..
head repository: DeterminateSystems/hydra
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 38fedea946d3c89fb6df021163a2b647317d82b1
Choose a head ref
Showing with 1,890 additions and 348 deletions.
  1. +1 −0 .gitignore
  2. +1 −1 README.md
  3. +5 −1 configure.ac
  4. +1 −1 doc/dev-notes.txt
  5. +8 −1 doc/manual/src/configuration.md
  6. +1 −1 doc/manual/src/installation.md
  7. +13 −0 doc/manual/src/notifications.md
  8. +47 −3 flake.nix
  9. +35 −6 src/hydra-queue-runner/build-remote.cc
  10. +12 −10 src/hydra-queue-runner/queue-monitor.cc
  11. +4 −1 src/hydra-queue-runner/state.hh
  12. +28 −11 src/lib/Hydra/Controller/API.pm
  13. +3 −1 src/lib/Hydra/Controller/Project.pm
  14. +11 −6 src/lib/Hydra/Controller/Root.pm
  15. +1 −1 src/lib/Hydra/Controller/User.pm
  16. +4 −0 src/lib/Hydra/Event.pm
  17. +59 −0 src/lib/Hydra/Event/CachedBuildFinished.pm
  18. +59 −0 src/lib/Hydra/Event/CachedBuildQueued.pm
  19. +4 −4 src/lib/Hydra/Helper/CatalystUtils.pm
  20. +31 −14 src/lib/Hydra/Helper/Nix.pm
  21. +12 −0 src/lib/Hydra/Plugin.pm
  22. +1 −1 src/lib/Hydra/Plugin/DeclarativeJobsets.pm
  23. +2 −2 src/lib/Hydra/Plugin/EmailNotification.pm
  24. +1 −1 src/lib/Hydra/Plugin/HipChatNotification.pm
  25. +3 −3 src/lib/Hydra/Plugin/InfluxDBNotification.pm
  26. +2 −2 src/lib/Hydra/Plugin/RunCommand.pm
  27. +1 −1 src/lib/Hydra/Plugin/SlackNotification.pm
  28. +39 −80 src/lib/Hydra/Schema/Result/Builds.pm
  29. +4 −22 src/lib/Hydra/Schema/Result/Jobsets.pm
  30. +7 −17 src/lib/Hydra/Schema/Result/Projects.pm
  31. +2 −2 src/root/common.tt
  32. +1 −1 src/root/search.tt
  33. +67 −13 src/script/hydra-create-user
  34. +36 −19 src/script/hydra-eval-jobset
  35. +4 −2 src/script/hydra-notify
  36. +11 −3 src/script/hydra-update-gc-roots
  37. +1 −10 src/sql/hydra.sql
  38. +9 −4 src/sql/upgrade-80.sql
  39. +4 −0 src/sql/upgrade-81.sql
  40. +111 −0 t/Event/CachedBuildFinished.t
  41. +111 −0 t/Event/CachedBuildQueued.t
  42. +215 −0 t/Hydra/Controller/API/checks.t
  43. +105 −0 t/Hydra/Controller/Build/api.t
  44. +0 −47 t/Hydra/Controller/Build/constituents.t
  45. +60 −0 t/Hydra/Controller/Job/builds.t
  46. +31 −0 t/Hydra/Controller/Jobset/builds.t
  47. +14 −2 t/Hydra/Controller/JobsetEval/fetch.t
  48. +30 −0 t/Hydra/Controller/Project/builds.t
  49. +75 −0 t/Hydra/Controller/Project/delete.t
  50. +0 −30 t/Hydra/Controller/Root/queue-runner-status.t
  51. +103 −0 t/Hydra/Controller/Root/status.t
  52. +11 −0 t/Hydra/Helper/CatalystUtils.t
  53. +63 −0 t/Hydra/Plugin/DeclarativeJobsets/basic.t
  54. +29 −0 t/jobs/declarative/generator.nix
  55. +3 −0 t/jobs/declarative/generator.sh
  56. +24 −0 t/jobs/declarative/project.json.in
  57. +17 −0 t/jobs/dependencies/dependency.nix
  58. +4 −0 t/jobs/dependencies/dependentOnly.nix
  59. +4 −0 t/jobs/dependencies/underlyingOnly.nix
  60. +30 −0 t/jobs/hydra-eval-notifications.nix
  61. +8 −0 t/jobs/one-job.nix
  62. +43 −6 t/lib/CliRunners.pm
  63. +10 −1 t/lib/HydraTestContext.pm
  64. +18 −4 t/lib/Setup.pm
  65. +51 −0 t/queue-runner/build-locally-with-substitutable-path.t
  66. +48 −13 t/scripts/hydra-create-user.t
  67. +115 −0 t/scripts/hydra-eval-jobset/notifications.t
  68. +22 −0 t/scripts/hydra-update-gc-roots/update-gc-roots.t
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@ Makefile.in
/t/nix
/t/data
/t/jobs/config.nix
t/jobs/declarative/project.json
/inst
hydra-config.h
hydra-config.h.in
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -28,7 +28,7 @@ Once the Hydra service has been configured as above and activate you should alre
```
$ su - hydra
$ hydra-create-user <USER> --full-name '<NAME>' \
--email-address '<EMAIL>' --password <PASSWORD> --role admin
--email-address '<EMAIL>' --password-prompt --role admin
```

Afterwards you should be able to log by clicking on "_Sign In_" on the top right of the web interface using the credentials specified by `hydra-create-user`. Once you are logged in you can click "_Admin -> Create Project_" to configure your first project.
6 changes: 5 additions & 1 deletion configure.ac
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ then
NIX_STATE_DIR="$TMPDIR"
export NIX_STATE_DIR
fi
if NIX_REMOTE=daemon PAGER=cat "$NIX_STORE_PROGRAM" --timeout 123 -q; then
if NIX_REMOTE=daemon PAGER=cat "$NIX_STORE_PROGRAM" --timeout 123 -q; then
AC_MSG_RESULT([yes])
else
AC_MSG_RESULT([no])
@@ -55,6 +55,9 @@ PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store])
testPath="$(dirname $(type -p expr))"
AC_SUBST(testPath)

jobsPath="$(realpath ./t/jobs)"
AC_SUBST(jobsPath)

CXXFLAGS+=" -include nix/config.h"

AC_CONFIG_FILES([
@@ -72,6 +75,7 @@ AC_CONFIG_FILES([
src/script/Makefile
t/Makefile
t/jobs/config.nix
t/jobs/declarative/project.json
])

AC_CONFIG_COMMANDS([executable-scripts], [])
2 changes: 1 addition & 1 deletion doc/dev-notes.txt
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@
* Creating a user:

$ hydra-create-user root --email-address 'e.dolstra@tudelft.nl' \
--password-hash "$(echo -n foobar | sha1sum | cut -c1-40)"
--password-prompt

(Replace "foobar" with the desired password.)

9 changes: 8 additions & 1 deletion doc/manual/src/configuration.md
Original file line number Diff line number Diff line change
@@ -4,6 +4,13 @@ Configuration
This chapter is a collection of configuration snippets for different
scenarios.

The configuration is parsed by `Config::General` which has [a pretty
thorough documentation on their file format](https://metacpan.org/pod/Config::General#CONFIG-FILE-FORMAT).
Hydra calls the parser with the following options:
- `-UseApacheInclude => 1`
- `-IncludeAgain => 1`
- `-IncludeRelative => 1`

Including files
---------------

@@ -126,7 +133,7 @@ general any LDAP group of the form *hydra\_some\_role* (notice the
binddn: "cn=root,dc=example"
bindpw: notapassword
start_tls: 0
start_tls_options
start_tls_options:
verify: none
user_basedn: "ou=users,dc=example"
user_filter: "(&(objectClass=inetOrgPerson)(cn=%s))"
2 changes: 1 addition & 1 deletion doc/manual/src/installation.md
Original file line number Diff line number Diff line change
@@ -114,7 +114,7 @@ This can be done using the command `hydra-create-user`:

```console
$ hydra-create-user alice --full-name 'Alice Q. User' \
--email-address 'alice@example.org' --password foobar --role admin
--email-address 'alice@example.org' --password-prompt --role admin
```

Additional users can be created through the web interface.
13 changes: 13 additions & 0 deletions doc/manual/src/notifications.md
Original file line number Diff line number Diff line change
@@ -8,6 +8,19 @@ Notifications are passed from `hydra-queue-runner` to `hydra-notify` through Pos

Note that the notification format is subject to change and should not be considered an API. Integrate with `hydra-notify` instead of listening directly.

### `cached_build_finished`

* **Payload:** Exactly two values, tab separated: The ID of the evaluation which contains the finished build, followed by the ID of the finished build.
* **When:** Issued directly after an evaluation completes, when that evaluation includes this finished build.
* **Delivery Semantics:** At most once per evaluation.


### `cached_build_queued`

* **Payload:** Exactly two values, tab separated: The ID of the evaluation which contains the finished build, followed by the ID of the queued build.
* **When:** Issued directly after an evaluation completes, when that evaluation includes this queued build.
* **Delivery Semantics:** At most once per evaluation.

### `build_queued`

* **Payload:** Exactly one value, the ID of the build.
50 changes: 47 additions & 3 deletions flake.nix
Original file line number Diff line number Diff line change
@@ -495,6 +495,7 @@
StringCompareConstantTime
SysHostnameLong
TermSizeAny
TermReadKey
Test2Harness
TestMore
TestPostgreSQL
@@ -868,9 +869,16 @@

services.openldap.enable = true;
services.openldap.settings.children = {
"cn=schema".includes = [
"${pkgs.openldap}/etc/schema/core.ldif"
"${pkgs.openldap}/etc/schema/cosine.ldif"
"${pkgs.openldap}/etc/schema/inetorgperson.ldif"
"${pkgs.openldap}/etc/schema/nis.ldif"
];

"olcDatabase={1}mdb".attrs = {
objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ];
database = "{1}mdbg";
olcDatabase = "{1}mdb";
olcSuffix = "dc=example";
olcRootDN = "cn=root,dc=example";
olcRootPW = "notapassword";
@@ -906,6 +914,12 @@
objectClass: groupOfNames
member: cn=admin,ou=users,dc=example
dn: cn=hydra-admin,ou=groups,dc=example
cn: hydra-admin
description: Users who are NOT Hydra Admins because the prefix needs to be a _
objectClass: groupOfNames
member: cn=notadmin,ou=users,dc=example
dn: cn=user,ou=users,dc=example
objectClass: organizationalPerson
objectClass: inetOrgPerson
@@ -921,6 +935,15 @@
cn: admin
mail: admin@example
userPassword: {SSHA}BsgOQcRnoiULzwLrGmuzVGH6EC5Dkwmf
dn: cn=notadmin,ou=users,dc=example
objectClass: organizationalPerson
objectClass: inetOrgPerson
sn: notadmin
cn: notadmin
mail: notadmin@example
userPassword: {SSHA}BsgOQcRnoiULzwLrGmuzVGH6EC5Dkwmf
'';
systemd.services.hydra-server.environment.CATALYST_DEBUG = "1";
systemd.services.hydra-server.environment.HYDRA_LDAP_CONFIG = pkgs.writeText "config.yaml"
@@ -933,7 +956,9 @@
store:
class: LDAP
ldap_server: localhost
ldap_server_options.timeout: 30
ldap_server_options:
timeout: 30
debug: 2
binddn: "cn=root,dc=example"
bindpw: notapassword
start_tls: 0
@@ -953,38 +978,57 @@
role_value: dn
role_search_options:
deref: always
'';
'';
networking.firewall.enable = false;
};
testScript = ''
import json
from pprint import pprint
machine.wait_for_unit("openldap.service")
machine.wait_for_job("hydra-init")
machine.wait_for_open_port("3000")
print("Logging in as a regular user:")
response = machine.succeed(
"curl --fail http://localhost:3000/login -H 'Accept: application/json' -H 'Referer: http://localhost:3000' --data 'username=user&password=foobar'"
)
response_json = json.loads(response)
pprint(response_json)
assert "user" == response_json["username"]
assert "user@example" == response_json["emailaddress"]
assert len(response_json["userroles"]) == 0
# logging on with wrong credentials shouldn't work
print("Logging in with bad creds:")
machine.fail(
"curl --fail http://localhost:3000/login -H 'Accept: application/json' -H 'Referer: http://localhost:3000' --data 'username=user&password=wrongpassword'"
)
# the admin user should get the admin role from his group membership in `hydra_admin`
print("Logging in as an admin user:")
response = machine.succeed(
"curl --fail http://localhost:3000/login -H 'Accept: application/json' -H 'Referer: http://localhost:3000' --data 'username=admin&password=password'"
)
response_json = json.loads(response)
pprint(response_json)
assert "admin" == response_json["username"]
assert "admin@example" == response_json["emailaddress"]
assert "admin" in response_json["userroles"]
# the notadmin user should NOT get the admin role from their group membership in `hydra-admin`
response = machine.succeed(
"curl --fail http://localhost:3000/login -H 'Accept: application/json' -H 'Referer: http://localhost:3000' --data 'username=notadmin&password=password'"
)
response_json = json.loads(response)
pprint(response_json)
assert "notadmin" == response_json["username"]
assert "notadmin@example" == response_json["emailaddress"]
assert "admin" not in response_json["userroles"]
'';
};

41 changes: 35 additions & 6 deletions src/hydra-queue-runner/build-remote.cc
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@
#include "util.hh"
#include "worker-protocol.hh"
#include "finally.hh"
#include "url.hh"

using namespace nix;

@@ -26,6 +27,25 @@ static void append(Strings & dst, const Strings & src)
dst.insert(dst.end(), src.begin(), src.end());
}

static Strings extraStoreArgs(std::string & machine)
{
Strings result;
try {
auto parsed = parseURL(machine);
if (parsed.scheme != "ssh") {
throw SysError("Currently, only (legacy-)ssh stores are supported!");
}
machine = parsed.authority.value_or("");
auto remoteStore = parsed.query.find("remote-store");
if (remoteStore != parsed.query.end()) {
result = {"--store", shellEscape(remoteStore->second)};
}
} catch (BadURL &) {
// We just try to continue with `machine->sshName` here for backwards compat.
}

return result;
}

static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child)
{
@@ -54,7 +74,9 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
}
else {
pgmName = "ssh";
argv = {"ssh", machine->sshName};
auto sshName = machine->sshName;
Strings extraArgs = extraStoreArgs(sshName);
argv = {"ssh", sshName};
if (machine->sshKey != "") append(argv, {"-i", machine->sshKey});
if (machine->sshPublicHostKey != "") {
Path fileName = tmpDir + "/host-key";
@@ -66,6 +88,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
append(argv,
{ "-x", "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
, "--", "nix-store", "--serve", "--write" });
append(argv, extraArgs);
}

execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast
@@ -86,8 +109,7 @@ static void copyClosureTo(std::timed_mutex & sendMutex, ref<Store> destStore,
bool useSubstitutes = false)
{
StorePathSet closure;
for (auto & path : paths)
destStore->computeFSClosure(path, closure);
destStore->computeFSClosure(paths, closure);

/* Send the "query valid paths" command with the "lock" option
enabled. This prevents a race where the remote host
@@ -270,17 +292,24 @@ void State::buildRemote(ref<Store> destStore,
copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
}

/* Copy the input closure. */
if (!machine->isLocalhost()) {
{
auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting);
mc1.reset();
MaintainCount<counter> mc2(nrStepsCopyingTo);

printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
localStore->printStorePath(step->drvPath), machine->sshName);

auto now1 = std::chrono::steady_clock::now();

copyClosureTo(machine->state->sendLock, destStore, from, to, inputs, true);
/* Copy the input closure. */
if (machine->isLocalhost()) {
StorePathSet closure;
destStore->computeFSClosure(inputs, closure);
copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
} else {
copyClosureTo(machine->state->sendLock, destStore, from, to, inputs, true);
}

auto now2 = std::chrono::steady_clock::now();

22 changes: 12 additions & 10 deletions src/hydra-queue-runner/queue-monitor.cc
Original file line number Diff line number Diff line change
@@ -96,8 +96,11 @@ bool State::getQueuedBuilds(Connection & conn,
pqxx::work txn(conn);

auto res = txn.exec_params
("select id, project, jobset, job, drvPath, maxsilent, timeout, timestamp, globalPriority, priority from Builds "
"where id > $1 and finished = 0 order by globalPriority desc, id",
("select builds.id, builds.jobset_id, jobsets.project as project, "
"jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, "
"globalPriority, priority from Builds "
"inner join jobsets on builds.jobset_id = jobsets.id "
"where builds.id > $1 and finished = 0 order by globalPriority desc, builds.id",
lastBuildId);

for (auto const & row : res) {
@@ -110,6 +113,7 @@ bool State::getQueuedBuilds(Connection & conn,
auto build = std::make_shared<Build>(
localStore->parseStorePath(row["drvPath"].as<string>()));
build->id = id;
build->jobsetId = row["jobset_id"].as<JobsetID>();
build->projectName = row["project"].as<string>();
build->jobsetName = row["jobset"].as<string>();
build->jobName = row["job"].as<string>();
@@ -118,7 +122,7 @@ bool State::getQueuedBuilds(Connection & conn,
build->timestamp = row["timestamp"].as<time_t>();
build->globalPriority = row["globalPriority"].as<int>();
build->localPriority = row["priority"].as<int>();
build->jobset = createJobset(txn, build->projectName, build->jobsetName);
build->jobset = createJobset(txn, build->projectName, build->jobsetName, build->jobsetId);

newIDs.push_back(id);
newBuildsByID[id] = build;
@@ -569,7 +573,7 @@ Step::ptr State::createStep(ref<Store> destStore,


Jobset::ptr State::createJobset(pqxx::work & txn,
const std::string & projectName, const std::string & jobsetName)
const std::string & projectName, const std::string & jobsetName, const JobsetID jobsetID)
{
auto p = std::make_pair(projectName, jobsetName);

@@ -580,9 +584,8 @@ Jobset::ptr State::createJobset(pqxx::work & txn,
}

auto res = txn.exec_params1
("select schedulingShares from Jobsets where project = $1 and name = $2",
projectName,
jobsetName);
("select schedulingShares from Jobsets where id = $1",
jobsetID);
if (res.empty()) throw Error("missing jobset - can't happen");

auto shares = res["schedulingShares"].as<unsigned int>();
@@ -593,10 +596,9 @@ Jobset::ptr State::createJobset(pqxx::work & txn,
/* Load the build steps from the last 24 hours. */
auto res2 = txn.exec_params
("select s.startTime, s.stopTime from BuildSteps s join Builds b on build = id "
"where s.startTime is not null and s.stopTime > $1 and project = $2 and jobset = $3",
"where s.startTime is not null and s.stopTime > $1 and jobset_id = $2",
time(0) - Jobset::schedulingWindow * 10,
projectName,
jobsetName);
jobsetID);
for (auto const & row : res2) {
time_t startTime = row["startTime"].as<time_t>();
time_t stopTime = row["stopTime"].as<time_t>();
Loading