Switch to the authentication token-based runner creation workflow

README.md

@@ -35,33 +35,56 @@ async fn main() {
     let mut runner = Runner::new(
         "https://gitlab.example.com".try_into().unwrap(),
         "runner token".to_owned(),
+        "system id".to_owned(),
         PathBuf::from("/tmp"));
     runner.run(move | _job | async move { Ok(Run{})  }, 16).await.unwrap();
 }
 ```
 
-## Gitlab runner registration
+## Gitlab runner creation
 
-This crate does not support registering new runners with the gitlab server, so this has to be
-done by hand using the gitlab
-[runner registration API](https://docs.gitlab.com/ee/api/runners.html#register-a-new-runner).
+This crate does not support creating new runners on the GitLab server. This can
+be done using the
+[runner creation API](https://docs.gitlab.com/ee/api/users.html#create-a-runner-linked-to-a-user),
+or manually in the GitLab
+[runner management web interface](https://docs.gitlab.com/ee/ci/runners/runners_scope.html).
+Make sure to follow the runner creation with an authentication token workflow,
+as the registration token workflow is deprecated.
+
+One key parameter provided when creating the runner is `run_untagged=false` (or
+leaving the `Run untagged jobs` box unchecked in the web interface), which will
+make the runner *only* pickup jobs which matches its tags. This is important to
+prevent the runner from picking up "normal" jobs which it will not be able to
+process.
+
+When the runner is created GitLab provides an authentication token starting
+with `glrt-`. This token should be provided to the runner for its GitLab
+connection, along with a system ID that identifies the machine on which the
+runner is executed.
+
+The system ID should be a unique string. GitLab doesn't currently require any
+particular formatting, but it is recommended to follow the way the official
+`gitlab-runner` creates system IDs:
+
+- Deriving it from the machine ID, found in `/var/lib/dbus/machine-id` or
+  `/etc/machine-id`, but concatenating the machine ID with the string
+  "gitlab-runner", taking the first 12 characters of its SHA256 hash in hex
+  form, and prepending it with `s_`.
+
+- Generating a random 12-character string using letters and digits
+  (`[a-z][A-Z][0-9]`), and prepending it with `r_`.
+
+In either case the system ID should be recorded in a persistent storage, along
+with the authentication token, and be passed to the `Runner::new()` function.
+
+The token can be verified using a curl command like:
 
-The registration token can be retrieved from the runners section in the Gitlab
-administration area. With that token the runner can be register using a curl
-command like:
 ```shell
-curl --request POST "https://GITLAB_URL/api/v4/runners"  \
-  --form "description=My custom runner" \
-  --form "run_untagged=false" \
-  --form "tag_list=custom-gitlab-runner" \
-  --form "token=REGISTRATION_TOKEN"
+curl --request POST "https://GITLAB_URL/api/v4/runners/verify"  \
+  --form "token=AUTHENTICATION_TOKEN" \
+  --form "system_id=SYSTEM_ID"
 ```
 
-As a response to this command a new token for the registered runner will be
-provided, this token should be provided to the runner for it's gitlab
-connection.
-
-One thing to key parameter provided here is `run_untagged=false`, which will
-make the runner *only* pickup jobs which matches its tag. This is important to
-prevent the runner from picking up "normal" jobs which it will not be able to
-process.
+This step is optional. If performed, it will pre-register the system ID with
+the GitLab server. Otherwise the system ID will be registered the first time
+the runner pings for jobs.

gitlab-runner-mock/src/lib.rs

@@ -29,6 +29,7 @@ struct JobData {
 struct Inner {
     server: MockServer,
     runner_token: String,
+    system_id: String,
     jobs: Mutex<JobData>,
     update_interval: Mutex<u32>,
 }
@@ -48,6 +49,7 @@ impl GitlabRunnerMock {
         let inner = Inner {
             server: m,
             runner_token: "fakerunnertoken".to_string(),
+            system_id: "r_0123456789ab".to_string(),
             jobs: Mutex::new(jobs),
             update_interval: Mutex::new(3),
         };
@@ -98,6 +100,10 @@ impl GitlabRunnerMock {
         &self.inner.runner_token
     }
 
+    pub fn runner_system_id(&self) -> &str {
+        &self.inner.system_id
+    }
+
     pub fn add_dummy_job(&self, name: String) -> MockJob {
         let mut jobs = self.inner.jobs.lock().unwrap();
         jobs.last_id += 1;

gitlab-runner/examples/demo-runner.rs

@@ -16,6 +16,7 @@ use url::Url;
 struct Opts {
     server: Url,
     token: String,
+    system_id: String,
 }
 
 #[derive(Deserialize)]
@@ -189,8 +190,12 @@ async fn main() {
     let opts = Opts::from_args();
     let dir = tempfile::tempdir().unwrap();
 
-    let (mut runner, layer) =
-        Runner::new_with_layer(opts.server, opts.token, dir.path().to_path_buf());
+    let (mut runner, layer) = Runner::new_with_layer(
+        opts.server,
+        opts.token,
+        opts.system_id,
+        dir.path().to_path_buf(),
+    );
 
     tracing_subscriber::Registry::default()
         .with(

gitlab-runner/src/client.rs

@@ -81,6 +81,7 @@ struct VersionInfo {
 #[derive(Debug, Clone, Serialize)]
 struct JobRequest<'a> {
     token: &'a str,
+    system_id: &'a str,
     info: VersionInfo,
 }
 
@@ -275,20 +276,23 @@ pub(crate) struct Client {
     client: reqwest::Client,
     url: Url,
     token: String,
+    system_id: String,
 }
 
 impl Client {
-    pub fn new(url: Url, token: String) -> Self {
+    pub fn new(url: Url, token: String, system_id: String) -> Self {
         Self {
             client: reqwest::Client::new(),
             url,
             token,
+            system_id,
         }
     }
 
     pub async fn request_job(&self) -> Result<Option<JobResponse>, Error> {
         let request = JobRequest {
             token: &self.token,
+            system_id: &self.system_id,
             info: VersionInfo {
                 // Setting `refspecs` is required to run detached MR pipelines.
                 features: FeaturesInfo {
@@ -525,7 +529,11 @@ mod test {
     async fn no_job() {
         let mock = GitlabRunnerMock::start().await;
 
-        let client = Client::new(mock.uri(), mock.runner_token().to_string());
+        let client = Client::new(
+            mock.uri(),
+            mock.runner_token().to_string(),
+            mock.runner_system_id().to_string(),
+        );
 
         let job = client.request_job().await.unwrap();
 
@@ -537,7 +545,11 @@ mod test {
         let mock = GitlabRunnerMock::start().await;
         mock.add_dummy_job("process job".to_string());
 
-        let client = Client::new(mock.uri(), mock.runner_token().to_string());
+        let client = Client::new(
+            mock.uri(),
+            mock.runner_token().to_string(),
+            mock.runner_system_id().to_string(),
+        );
 
         if let Some(job) = client.request_job().await.unwrap() {
             client

gitlab-runner/src/lib.rs

@@ -34,6 +34,7 @@
 //!     let mut runner = Runner::new(
 //!         "https://gitlab.example.com".try_into().unwrap(),
 //!         "runner token".to_owned(),
+//!         "runner system id".to_owned(),
 //!         PathBuf::from("/tmp"));
 //!     runner.run(move | _job | async move { Ok(Run{})  }, 16).await.unwrap();
 //! }
@@ -275,8 +276,8 @@ pub struct Runner {
 }
 
 impl Runner {
-    /// Create a new Runner for the given server url and runner token, storing (temporary job
-    /// files) in build_dir
+    /// Create a new Runner for the given server url, runner token and system ID, storing
+    /// (temporary job files) in build_dir
     ///
     /// The build_dir is used to store temporary files during a job run. This will also configure a
     /// default tracing subscriber if that's not wanted use [`Runner::new_with_layer`] instead.
@@ -288,14 +289,15 @@ impl Runner {
     /// let dir = tempfile::tempdir().unwrap();
     /// let runner = Runner::new(Url::parse("https://gitlab.com/").unwrap(),
     ///     "RunnerToken".to_string(),
+    ///     "RunnerSystemID".to_string(),
     ///     dir.path().to_path_buf());
     /// ```
     ///
     /// # Panics
     ///
     /// Panics if a default subscriber is already setup
-    pub fn new(server: Url, token: String, build_dir: PathBuf) -> Self {
-        let (runner, layer) = Self::new_with_layer(server, token, build_dir);
+    pub fn new(server: Url, token: String, system_id: String, build_dir: PathBuf) -> Self {
+        let (runner, layer) = Self::new_with_layer(server, token, system_id, build_dir);
         Registry::default().with(layer).init();
 
         runner
@@ -314,11 +316,17 @@ impl Runner {
     /// let dir = tempfile::tempdir().unwrap();
     /// let (runner, layer) = Runner::new_with_layer(Url::parse("https://gitlab.com/").unwrap(),
     ///     "RunnerToken".to_string(),
+    ///     "RunnerSystemID".to_string(),
     ///     dir.path().to_path_buf());
     /// let subscriber = Registry::default().with(layer).init();
     /// ```
-    pub fn new_with_layer(server: Url, token: String, build_dir: PathBuf) -> (Self, GitlabLayer) {
-        let client = Client::new(server, token);
+    pub fn new_with_layer(
+        server: Url,
+        token: String,
+        system_id: String,
+        build_dir: PathBuf,
+    ) -> (Self, GitlabLayer) {
+        let client = Client::new(server, token, system_id);
         let run_list = RunList::new();
         (
             Self {

gitlab-runner/tests/artifacts.rs

@@ -133,6 +133,7 @@ async fn upload_download() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -214,6 +215,7 @@ async fn multiple_upload() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 

gitlab-runner/tests/integration.rs

@@ -274,6 +274,7 @@ async fn job_success() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -300,6 +301,7 @@ async fn job_fail() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -326,6 +328,7 @@ async fn job_panic() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -356,6 +359,7 @@ async fn job_cancel_step() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_owned(),
+        mock.runner_system_id().to_owned(),
         dir.path().to_path_buf(),
     );
 
@@ -386,6 +390,7 @@ async fn job_log() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -432,6 +437,7 @@ async fn job_steps() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -466,6 +472,7 @@ async fn job_parallel() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -523,6 +530,7 @@ async fn runner_run() {
     let (mut r, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -582,6 +590,7 @@ async fn runner_limit() {
     let (mut r, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -642,6 +651,7 @@ async fn runner_delay() {
     let (mut r, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -707,6 +717,7 @@ async fn job_variables() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 
@@ -748,6 +759,7 @@ async fn job_drain() {
     let (mut runner, layer) = Runner::new_with_layer(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );
 

gitlab-runner/tests/runhandler.rs

@@ -186,6 +186,7 @@ async fn update_interval() {
     let mut runner = Runner::new(
         mock.uri(),
         mock.runner_token().to_string(),
+        mock.runner_system_id().to_string(),
         dir.path().to_path_buf(),
     );