Merge pull request #27 from Cyberboss/PrivateRepoSupport

Support for private repos

crates/diffbot_lib/Cargo.toml

@@ -26,3 +26,4 @@ flume = "0.11.0"
 actix-web = "4.4.0"
 
 async-fs = "1.6.0"
+base64 = "0.21.4"

crates/diffbot_lib/src/github/github_api.rs

@@ -1,14 +1,12 @@
 use crate::github::github_types::{
     CreateCheckRun, Output, RawCheckRun, Repository, UpdateCheckRunBuilder,
 };
-use async_fs::File;
 use eyre::{format_err, Context, Result};
-use futures_lite::io::AsyncWriteExt;
 use octocrab::models::repos::Content;
 use octocrab::models::InstallationId;
 use serde::{Deserialize, Serialize};
-use std::path::PathBuf;
 use std::{future::Future, pin::Pin};
+use base64::{Engine as _, engine::general_purpose};
 
 pub struct GithubEvent(pub String, pub Option<Vec<u8>>);
 
@@ -207,8 +205,6 @@ impl CheckRun {
     }
 }
 
-static DOWNLOAD_DIR: &str = "download";
-
 async fn find_content<S: AsRef<str>>(
     installation: &InstallationId,
     repo: &Repository,
@@ -244,34 +240,57 @@ pub async fn download_url<S: AsRef<str>>(
 ) -> Result<Vec<u8>> {
     let target = find_content(installation, repo, filename, commit).await?;
 
-    let download_url = target
-        .download_url
-        .as_ref()
-        .ok_or_else(|| format_err!("No download URL given by GitHub"))?;
-
-    let response = reqwest::get(download_url).await?;
+    let content = target.content
+        .ok_or_else(|| format_err!("File had no content!"))?
+        .replace("\n", "");
 
-    Ok(response.bytes().await?.to_vec())
+    general_purpose::STANDARD
+        .decode(content)
+        .map_err(|decode_error| format_err!("DecodeError: {}", decode_error))
 }
 
-pub async fn download_file<S: AsRef<str>>(
-    installation: &InstallationId,
-    repo: &Repository,
-    filename: S,
-    commit: S,
-) -> Result<PathBuf> {
-    let target = find_content(installation, repo, &filename, &commit).await?;
-
-    let mut path = PathBuf::new();
-    path.push(".");
-    path.push(DOWNLOAD_DIR);
-    path.push(&target.sha);
-    path.set_extension("dmi"); // Method should have an IDB qualifier due to being a shared crate
-
-    async_fs::create_dir_all(path.parent().unwrap()).await?;
-    let mut file = File::create(&path).await?;
-
-    let data = download_url(installation, repo, &filename, &commit).await?;
-    file.write_all(&data).await?;
-    Ok(path)
+/* local test requires commenting out the .installation(...) call in find_content(), a valid github token with access, and the following dep: actix-rt = "2.9.0"
+
+#[actix_web::rt::test]
+async fn test_private_repo_file_download() {
+    octocrab::initialise(octocrab::OctocrabBuilder::new()
+        .personal_token("lol".to_owned())
+        .build()
+        .unwrap());
+
+    let bytes = download_url(
+        &InstallationId(0),
+        &Repository{
+        url: "https://api.github.com/repos/Cyberboss/tgstation-private-test".to_owned(),
+        id:0,
+    }, ".tgs.yml", "140c79189849ea616f09b3484f8930211d3705cd").await.unwrap();
+
+    let text = std::str::from_utf8(bytes.as_slice()).unwrap();
+    assert_eq!(r#"# This file is used by TGS (https://github.com/tgstation/tgstation-server) clients to quickly initialize a server instance for the codebase
+# The format isn't documented anywhere but hopefully we never have to change it. If there are questions, contact the TGS maintainer Cyberboss/@Dominion#0444
+version: 1
+# The BYOND version to use (kept in sync with dependencies.sh by the "TGS Test Suite" CI job)
+# Must be interpreted as a string, keep quoted
+byond: "514.1588"
+# Folders to create in "<instance_path>/Configuration/GameStaticFiles/"
+static_files:
+  # Config directory should be static
+  - name: config
+    # This implies the folder should be pre-populated with contents from the repo
+    populate: true
+  # Data directory must be static
+  - name: data
+# String dictionary. The value is the location of the file in the repo to upload to TGS. The key is the name of the file to upload to "<instance_path>/Configuration/EventScripts/"
+# This one is for Linux hosted servers
+linux_scripts:
+  PreCompile.sh: tools/tgs_scripts/PreCompile.sh
+  WatchdogLaunch.sh: tools/tgs_scripts/WatchdogLaunch.sh
+  InstallDeps.sh: tools/tgs_scripts/InstallDeps.sh
+# Same as above for Windows hosted servers
+windows_scripts:
+  PreCompile.bat: tools/tgs_scripts/PreCompile.bat
+# The security level the game should be run at
+security: Trusted
+"#, text)
 }
+*/

crates/mapdiffbot2/Cargo.toml

@@ -39,6 +39,7 @@ tokio = { version = "1.32.0", features = ["io-util", "rt"] }
 
 mysql_async = "0.32.2"
 time = "0.3.28"
+secrecy = "0.8.0"
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemallocator = "0.5.4"

crates/mapdiffbot2/src/git_operations.rs

@@ -21,11 +21,11 @@ pub fn fetch_and_get_branches<'a>(
 
     remote
         .fetch(
-            &[base_branch_name],
+            &[base_branch_name, head_branch_name],
             Some(FetchOptions::new().prune(git2::FetchPrune::On)),
             None,
         )
-        .wrap_err("Fetching base")?;
+        .wrap_err("Fetching base and head")?;
     let fetch_head = repo
         .find_reference("FETCH_HEAD")
         .wrap_err("Getting FETCH_HEAD")?;
@@ -67,14 +67,6 @@ pub fn fetch_and_get_branches<'a>(
         .resolve_reference_from_short_name(base_branch_name)
         .wrap_err("Getting the base reference")?;
 
-    remote
-        .fetch(
-            &[head_branch_name],
-            Some(FetchOptions::new().prune(git2::FetchPrune::On)),
-            None,
-        )
-        .wrap_err("Fetching head")?;
-
     let fetch_head = repo
         .find_reference("FETCH_HEAD")
         .wrap_err("Getting FETCH_HEAD")?;

crates/mapdiffbot2/src/job_processor.rs

@@ -1,5 +1,6 @@
 use eyre::{Context, Result};
 use path_absolutize::Absolutize;
+use secrecy::ExposeSecret;
 use std::path::Path;
 use std::path::PathBuf;
 
@@ -342,12 +343,16 @@ pub fn do_job(job: Job, blob_client: Azure) -> Result<CheckOutputs> {
 
     let base = &job.base;
     let head = &job.head;
-    let repo = format!("https://github.com/{}", job.repo.full_name());
-    let repo_dir: PathBuf = ["./repos/", &job.repo.full_name()].iter().collect();
 
     let handle = actix_web::rt::Runtime::new()?;
+    let (_, secret_token) = handle.block_on(octocrab::instance()
+        .installation_and_token(job.installation))?;
+
+    let repo_dir: PathBuf = ["./repos/", &job.repo.full_name()].iter().collect();
 
-    if !repo_dir.exists() {
+    let url = format!("https://x-access-token:{}@github.com/{}", secret_token.expose_secret(), job.repo.full_name());
+    let clone_required = !repo_dir.exists();
+    if clone_required {
         tracing::debug!("Directory {:?} doesn't exist, creating dir", repo_dir);
         std::fs::create_dir_all(&repo_dir)?;
         handle.block_on(async {
@@ -358,7 +363,7 @@ pub fn do_job(job: Job, blob_client: Azure) -> Result<CheckOutputs> {
                 };
                 let _ = job.check_run.set_output(output).await; // we don't really care if updating the job fails, just continue
             });
-        clone_repo(&repo, &repo_dir).wrap_err("Cloning repo")?;
+        clone_repo(&url, &repo_dir).wrap_err("Cloning repo")?;
     }
 
     let non_abs_directory: PathBuf = [
@@ -392,6 +397,10 @@ pub fn do_job(job: Job, blob_client: Azure) -> Result<CheckOutputs> {
 
     let repository = git2::Repository::open(&repo_dir).wrap_err("Opening repository")?;
 
+    if !clone_required {
+        repository.remote_set_url("origin", &url)?;
+    }
+
     let mut remote = repository.find_remote("origin")?;
 
     remote
@@ -412,7 +421,7 @@ pub fn do_job(job: Job, blob_client: Azure) -> Result<CheckOutputs> {
         (&added_files, &modified_files, &removed_files),
         (&repository, &job.base.r#ref),
         (&repo_dir, output_directory, blob_client),
-        job.pull_request,
+        job.pull_request
     )
     .wrap_err("")
     {