From 8030c0d4cce979729b1527eec5a85873cd9de6b2 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Mon, 11 Dec 2023 13:36:10 +0200 Subject: [PATCH] Switch to the authentication token-based runner creation workflow Starting in GitLab v15.10 (and enabled by default in v16.0), the runner creation uses a workflow based on authentication tokens instead of registration tokens. This introduces a new runner property named system_id that needs to be passed to the /jobs/request API. Extend the Runner::new() and Runner::new_with_layout() functions with a new system_id parameter, and plumb it through the implementation. Additionall update the documentation in README.md to explain the new workflow. Signed-off-by: Laurent Pinchart --- README.md | 63 ++++++++++++++++++--------- gitlab-runner-mock/src/lib.rs | 6 +++ gitlab-runner/examples/demo-runner.rs | 9 +++- gitlab-runner/src/client.rs | 18 ++++++-- gitlab-runner/src/lib.rs | 20 ++++++--- gitlab-runner/tests/artifacts.rs | 2 + gitlab-runner/tests/integration.rs | 12 +++++ gitlab-runner/tests/runhandler.rs | 1 + 8 files changed, 100 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index abb7e98..a6330d4 100644 --- a/README.md +++ b/README.md @@ -35,33 +35,56 @@ async fn main() { let mut runner = Runner::new( "https://gitlab.example.com".try_into().unwrap(), "runner token".to_owned(), + "system id".to_owned(), PathBuf::from("/tmp")); runner.run(move | _job | async move { Ok(Run{}) }, 16).await.unwrap(); } ``` -## Gitlab runner registration +## Gitlab runner creation -This crate does not support registering new runners with the gitlab server, so this has to be -done by hand using the gitlab -[runner registration API](https://docs.gitlab.com/ee/api/runners.html#register-a-new-runner). +This crate does not support creating new runners on the GitLab server. This can +be done using the +[runner creation API](https://docs.gitlab.com/ee/api/users.html#create-a-runner-linked-to-a-user), +or manually in the GitLab +[runner management web interface](https://docs.gitlab.com/ee/ci/runners/runners_scope.html). +Make sure to follow the runner creation with an authentication token workflow, +as the registration token workflow is deprecated. + +One key parameter provided when creating the runner is `run_untagged=false` (or +leaving the `Run untagged jobs` box unchecked in the web interface), which will +make the runner *only* pickup jobs which matches its tags. This is important to +prevent the runner from picking up "normal" jobs which it will not be able to +process. + +When the runner is created GitLab provides an authentication token starting +with `glrt-`. This token should be provided to the runner for its GitLab +connection, along with a system ID that identifies the machine on which the +runner is executed. + +The system ID should be a unique string. GitLab doesn't currently require any +particular formatting, but it is recommended to follow the way the official +`gitlab-runner` creates system IDs: + +- Deriving it from the machine ID, found in `/var/lib/dbus/machine-id` or + `/etc/machine-id`, but concatenating the machine ID with the string + "gitlab-runner", taking the first 12 characters of its SHA256 hash in hex + form, and prepending it with `s_`. + +- Generating a random 12-character string using letters and digits + (`[a-z][A-Z][0-9]`), and prepending it with `r_`. + +In either case the system ID should be recorded in a persistent storage, along +with the authentication token, and be passed to the `Runner::new()` function. + +The token can be verified using a curl command like: -The registration token can be retrieved from the runners section in the Gitlab -administration area. With that token the runner can be register using a curl -command like: ```shell -curl --request POST "https://GITLAB_URL/api/v4/runners" \ - --form "description=My custom runner" \ - --form "run_untagged=false" \ - --form "tag_list=custom-gitlab-runner" \ - --form "token=REGISTRATION_TOKEN" +curl --request POST "https://GITLAB_URL/api/v4/runners/verify" \ + --form "token=AUTHENTICATION_TOKEN" \ + --form "system_id=SYSTEM_ID" ``` -As a response to this command a new token for the registered runner will be -provided, this token should be provided to the runner for it's gitlab -connection. - -One thing to key parameter provided here is `run_untagged=false`, which will -make the runner *only* pickup jobs which matches its tag. This is important to -prevent the runner from picking up "normal" jobs which it will not be able to -process. +This step is optional. If performed, it will pre-register the system ID with +the GitLab server. Otherwise the system ID will be registered the first time +the runner pings for jobs. diff --git a/gitlab-runner-mock/src/lib.rs b/gitlab-runner-mock/src/lib.rs index e75b00f..db5ecd5 100644 --- a/gitlab-runner-mock/src/lib.rs +++ b/gitlab-runner-mock/src/lib.rs @@ -29,6 +29,7 @@ struct JobData { struct Inner { server: MockServer, runner_token: String, + system_id: String, jobs: Mutex, update_interval: Mutex, } @@ -48,6 +49,7 @@ impl GitlabRunnerMock { let inner = Inner { server: m, runner_token: "fakerunnertoken".to_string(), + system_id: "r_0123456789ab".to_string(), jobs: Mutex::new(jobs), update_interval: Mutex::new(3), }; @@ -98,6 +100,10 @@ impl GitlabRunnerMock { &self.inner.runner_token } + pub fn runner_system_id(&self) -> &str { + &self.inner.system_id + } + pub fn add_dummy_job(&self, name: String) -> MockJob { let mut jobs = self.inner.jobs.lock().unwrap(); jobs.last_id += 1; diff --git a/gitlab-runner/examples/demo-runner.rs b/gitlab-runner/examples/demo-runner.rs index 1747086..854430a 100644 --- a/gitlab-runner/examples/demo-runner.rs +++ b/gitlab-runner/examples/demo-runner.rs @@ -16,6 +16,7 @@ use url::Url; struct Opts { server: Url, token: String, + system_id: String, } #[derive(Deserialize)] @@ -189,8 +190,12 @@ async fn main() { let opts = Opts::from_args(); let dir = tempfile::tempdir().unwrap(); - let (mut runner, layer) = - Runner::new_with_layer(opts.server, opts.token, dir.path().to_path_buf()); + let (mut runner, layer) = Runner::new_with_layer( + opts.server, + opts.token, + opts.system_id, + dir.path().to_path_buf(), + ); tracing_subscriber::Registry::default() .with( diff --git a/gitlab-runner/src/client.rs b/gitlab-runner/src/client.rs index 02724a1..fc23942 100644 --- a/gitlab-runner/src/client.rs +++ b/gitlab-runner/src/client.rs @@ -81,6 +81,7 @@ struct VersionInfo { #[derive(Debug, Clone, Serialize)] struct JobRequest<'a> { token: &'a str, + system_id: &'a str, info: VersionInfo, } @@ -275,20 +276,23 @@ pub(crate) struct Client { client: reqwest::Client, url: Url, token: String, + system_id: String, } impl Client { - pub fn new(url: Url, token: String) -> Self { + pub fn new(url: Url, token: String, system_id: String) -> Self { Self { client: reqwest::Client::new(), url, token, + system_id, } } pub async fn request_job(&self) -> Result, Error> { let request = JobRequest { token: &self.token, + system_id: &self.system_id, info: VersionInfo { // Setting `refspecs` is required to run detached MR pipelines. features: FeaturesInfo { @@ -525,7 +529,11 @@ mod test { async fn no_job() { let mock = GitlabRunnerMock::start().await; - let client = Client::new(mock.uri(), mock.runner_token().to_string()); + let client = Client::new( + mock.uri(), + mock.runner_token().to_string(), + mock.runner_system_id().to_string(), + ); let job = client.request_job().await.unwrap(); @@ -537,7 +545,11 @@ mod test { let mock = GitlabRunnerMock::start().await; mock.add_dummy_job("process job".to_string()); - let client = Client::new(mock.uri(), mock.runner_token().to_string()); + let client = Client::new( + mock.uri(), + mock.runner_token().to_string(), + mock.runner_system_id().to_string(), + ); if let Some(job) = client.request_job().await.unwrap() { client diff --git a/gitlab-runner/src/lib.rs b/gitlab-runner/src/lib.rs index 8047838..5a7fc4b 100644 --- a/gitlab-runner/src/lib.rs +++ b/gitlab-runner/src/lib.rs @@ -34,6 +34,7 @@ //! let mut runner = Runner::new( //! "https://gitlab.example.com".try_into().unwrap(), //! "runner token".to_owned(), +//! "runner system id".to_owned(), //! PathBuf::from("/tmp")); //! runner.run(move | _job | async move { Ok(Run{}) }, 16).await.unwrap(); //! } @@ -275,8 +276,8 @@ pub struct Runner { } impl Runner { - /// Create a new Runner for the given server url and runner token, storing (temporary job - /// files) in build_dir + /// Create a new Runner for the given server url, runner token and system ID, storing + /// (temporary job files) in build_dir /// /// The build_dir is used to store temporary files during a job run. This will also configure a /// default tracing subscriber if that's not wanted use [`Runner::new_with_layer`] instead. @@ -288,14 +289,15 @@ impl Runner { /// let dir = tempfile::tempdir().unwrap(); /// let runner = Runner::new(Url::parse("https://gitlab.com/").unwrap(), /// "RunnerToken".to_string(), + /// "RunnerSystemID".to_string(), /// dir.path().to_path_buf()); /// ``` /// /// # Panics /// /// Panics if a default subscriber is already setup - pub fn new(server: Url, token: String, build_dir: PathBuf) -> Self { - let (runner, layer) = Self::new_with_layer(server, token, build_dir); + pub fn new(server: Url, token: String, system_id: String, build_dir: PathBuf) -> Self { + let (runner, layer) = Self::new_with_layer(server, token, system_id, build_dir); Registry::default().with(layer).init(); runner @@ -314,11 +316,17 @@ impl Runner { /// let dir = tempfile::tempdir().unwrap(); /// let (runner, layer) = Runner::new_with_layer(Url::parse("https://gitlab.com/").unwrap(), /// "RunnerToken".to_string(), + /// "RunnerSystemID".to_string(), /// dir.path().to_path_buf()); /// let subscriber = Registry::default().with(layer).init(); /// ``` - pub fn new_with_layer(server: Url, token: String, build_dir: PathBuf) -> (Self, GitlabLayer) { - let client = Client::new(server, token); + pub fn new_with_layer( + server: Url, + token: String, + system_id: String, + build_dir: PathBuf, + ) -> (Self, GitlabLayer) { + let client = Client::new(server, token, system_id); let run_list = RunList::new(); ( Self { diff --git a/gitlab-runner/tests/artifacts.rs b/gitlab-runner/tests/artifacts.rs index cd6b318..3d8aa38 100644 --- a/gitlab-runner/tests/artifacts.rs +++ b/gitlab-runner/tests/artifacts.rs @@ -133,6 +133,7 @@ async fn upload_download() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -214,6 +215,7 @@ async fn multiple_upload() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); diff --git a/gitlab-runner/tests/integration.rs b/gitlab-runner/tests/integration.rs index 5ffd1a6..7a3a77f 100644 --- a/gitlab-runner/tests/integration.rs +++ b/gitlab-runner/tests/integration.rs @@ -274,6 +274,7 @@ async fn job_success() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -300,6 +301,7 @@ async fn job_fail() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -326,6 +328,7 @@ async fn job_panic() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -356,6 +359,7 @@ async fn job_cancel_step() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_owned(), + mock.runner_system_id().to_owned(), dir.path().to_path_buf(), ); @@ -386,6 +390,7 @@ async fn job_log() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -432,6 +437,7 @@ async fn job_steps() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -466,6 +472,7 @@ async fn job_parallel() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -523,6 +530,7 @@ async fn runner_run() { let (mut r, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -582,6 +590,7 @@ async fn runner_limit() { let (mut r, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -642,6 +651,7 @@ async fn runner_delay() { let (mut r, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -707,6 +717,7 @@ async fn job_variables() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); @@ -748,6 +759,7 @@ async fn job_drain() { let (mut runner, layer) = Runner::new_with_layer( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), ); diff --git a/gitlab-runner/tests/runhandler.rs b/gitlab-runner/tests/runhandler.rs index 2f600f9..97c9dc0 100644 --- a/gitlab-runner/tests/runhandler.rs +++ b/gitlab-runner/tests/runhandler.rs @@ -186,6 +186,7 @@ async fn update_interval() { let mut runner = Runner::new( mock.uri(), mock.runner_token().to_string(), + mock.runner_system_id().to_string(), dir.path().to_path_buf(), );