From f78b770a2fd330a4b58bcf6636fd799b58bd4622 Mon Sep 17 00:00:00 2001 From: tavo Date: Sat, 21 Nov 2020 14:38:42 +0200 Subject: [PATCH 01/33] Add initial version of config --- Cargo.toml | 6 ++- src/config/config.rs | 27 +++++++++++ src/git/git.rs | 112 +++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 14 ++++++ 4 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 src/config/config.rs create mode 100644 src/git/git.rs diff --git a/Cargo.toml b/Cargo.toml index 0a3e4f6..61cef59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,4 +7,8 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -rocket = "0.4.6" \ No newline at end of file +rocket = "0.4.6" +git2 = "0.13" +serde = "1.0" +serde_derive = "1.0" +toml = "0.5.7" \ No newline at end of file diff --git a/src/config/config.rs b/src/config/config.rs new file mode 100644 index 0000000..0c7951e --- /dev/null +++ b/src/config/config.rs @@ -0,0 +1,27 @@ + +use serde_derive::{Serialize, Deserialize}; +use std::fs; + +#[derive(Serialize, Deserialize)] +pub struct SyncConfig { + pub target_host: String, + pub target_port: Option, + pub port: Option, + pub repositories: Vec, +} + +#[derive(Serialize, Deserialize)] +pub struct Repository { + pub url: String, +} + +pub fn load(config_file: &String) -> SyncConfig { + let content = fs::read_to_string(config_file).expect("Unable to read config!"); + let config: SyncConfig = toml::from_str(&content).expect("Unable to deserialize config!"); + return config; +} + +pub fn save(config_file: &String, config: &SyncConfig) { + let content = toml::to_string(config).expect("Unable to serialize config!"); + fs::write(config_file, content).expect("Unable to save config!"); +} \ No newline at end of file diff --git a/src/git/git.rs b/src/git/git.rs new file mode 100644 index 0000000..f1fc4c7 --- /dev/null +++ b/src/git/git.rs @@ -0,0 +1,112 @@ +#![deny(warnings)] + +use git2::build::{CheckoutBuilder, RepoBuilder}; +use git2::{FetchOptions, Progress, RemoteCallbacks}; +use std::cell::RefCell; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +struct Args { + arg_url: String, + arg_path: String, +} + +struct State { + progress: Option>, + total: usize, + current: usize, + path: Option, + newline: bool, +} + +fn print(state: &mut State) { + let stats = state.progress.as_ref().unwrap(); + let network_pct = (100 * stats.received_objects()) / stats.total_objects(); + let index_pct = (100 * stats.indexed_objects()) / stats.total_objects(); + let co_pct = if state.total > 0 { + (100 * state.current) / state.total + } else { + 0 + }; + let kbytes = stats.received_bytes() / 1024; + if stats.received_objects() == stats.total_objects() { + if !state.newline { + println!(); + state.newline = true; + } + print!( + "Resolving deltas {}/{}\r", + stats.indexed_deltas(), + stats.total_deltas() + ); + } else { + print!( + "net {:3}% ({:4} kb, {:5}/{:5}) / idx {:3}% ({:5}/{:5}) \ + / chk {:3}% ({:4}/{:4}) {}\r", + network_pct, + kbytes, + stats.received_objects(), + stats.total_objects(), + index_pct, + stats.indexed_objects(), + stats.total_objects(), + co_pct, + state.current, + state.total, + state + .path + .as_ref() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_default() + ) + } + io::stdout().flush().unwrap(); +} + +fn run(args: &Args) -> Result<(), git2::Error> { + let state = RefCell::new(State { + progress: None, + total: 0, + current: 0, + path: None, + newline: false, + }); + let mut cb = RemoteCallbacks::new(); + cb.transfer_progress(|stats| { + let mut state = state.borrow_mut(); + state.progress = Some(stats.to_owned()); + print(&mut *state); + true + }); + + let mut co = CheckoutBuilder::new(); + co.progress(|path, cur, total| { + let mut state = state.borrow_mut(); + state.path = path.map(|p| p.to_path_buf()); + state.current = cur; + state.total = total; + print(&mut *state); + }); + + let mut fo = FetchOptions::new(); + fo.remote_callbacks(cb); + RepoBuilder::new() + .fetch_options(fo) + .with_checkout(co) + .clone(&args.arg_url, Path::new(&args.arg_path))?; + println!(); + + Ok(()) +} + +pub fn main() { + let args = Args { + arg_url: "https://github.com/DEVELOPEST/gtm.git".parse().unwrap(), + arg_path: "./test".parse().unwrap() + }; + + match run(&args) { + Ok(()) => {} + Err(e) => println!("error: {}", e), + } +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 7f88e41..6587b6c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,11 +2,25 @@ #[macro_use] extern crate rocket; +use crate::config::{SyncConfig, Repository}; + +#[path = "git/git.rs"] mod git; +#[path = "config/config.rs"] mod config; + #[get("/")] fn index() -> &'static str { "Hello, world!" } fn main() { + // git::main(); + let cfg = SyncConfig { + target_host: "http://our.server.ee".to_string(), + target_port: None, + port: Some(8765), + repositories: vec![Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2018".to_string() }] + }; + let loc = "./test.toml".to_string(); + config::save(&loc, &cfg); rocket::ignite().mount("/", routes![index]).launch(); } \ No newline at end of file From d7fdab95defc05c56ad9267c521002630454eb51 Mon Sep 17 00:00:00 2001 From: tavo Date: Sat, 21 Nov 2020 17:43:12 +0200 Subject: [PATCH 02/33] Something works with fetch --- src/config/config.rs | 5 +- src/git/git.rs | 125 +++++++++---------------------------------- src/main.rs | 14 +++-- 3 files changed, 40 insertions(+), 104 deletions(-) diff --git a/src/config/config.rs b/src/config/config.rs index 0c7951e..a49eb45 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -7,18 +7,19 @@ pub struct SyncConfig { pub target_host: String, pub target_port: Option, pub port: Option, + pub repositories_base_path: String, pub repositories: Vec, } #[derive(Serialize, Deserialize)] pub struct Repository { pub url: String, + pub path: String, } pub fn load(config_file: &String) -> SyncConfig { let content = fs::read_to_string(config_file).expect("Unable to read config!"); - let config: SyncConfig = toml::from_str(&content).expect("Unable to deserialize config!"); - return config; + return toml::from_str(&content).expect("Unable to deserialize config!"); } pub fn save(config_file: &String, config: &SyncConfig) { diff --git a/src/git/git.rs b/src/git/git.rs index f1fc4c7..9d81098 100644 --- a/src/git/git.rs +++ b/src/git/git.rs @@ -1,112 +1,39 @@ #![deny(warnings)] use git2::build::{CheckoutBuilder, RepoBuilder}; -use git2::{FetchOptions, Progress, RemoteCallbacks}; -use std::cell::RefCell; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use git2::{FetchOptions, RemoteCallbacks, Repository, Error}; +use std::path::{Path}; -struct Args { - arg_url: String, - arg_path: String, -} - -struct State { - progress: Option>, - total: usize, - current: usize, - path: Option, - newline: bool, -} - -fn print(state: &mut State) { - let stats = state.progress.as_ref().unwrap(); - let network_pct = (100 * stats.received_objects()) / stats.total_objects(); - let index_pct = (100 * stats.indexed_objects()) / stats.total_objects(); - let co_pct = if state.total > 0 { - (100 * state.current) / state.total - } else { - 0 - }; - let kbytes = stats.received_bytes() / 1024; - if stats.received_objects() == stats.total_objects() { - if !state.newline { - println!(); - state.newline = true; - } - print!( - "Resolving deltas {}/{}\r", - stats.indexed_deltas(), - stats.total_deltas() - ); - } else { - print!( - "net {:3}% ({:4} kb, {:5}/{:5}) / idx {:3}% ({:5}/{:5}) \ - / chk {:3}% ({:4}/{:4}) {}\r", - network_pct, - kbytes, - stats.received_objects(), - stats.total_objects(), - index_pct, - stats.indexed_objects(), - stats.total_objects(), - co_pct, - state.current, - state.total, - state - .path - .as_ref() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or_default() - ) - } - io::stdout().flush().unwrap(); -} +pub fn clone(url: &String, path: &String) -> Result { -fn run(args: &Args) -> Result<(), git2::Error> { - let state = RefCell::new(State { - progress: None, - total: 0, - current: 0, - path: None, - newline: false, - }); - let mut cb = RemoteCallbacks::new(); - cb.transfer_progress(|stats| { - let mut state = state.borrow_mut(); - state.progress = Some(stats.to_owned()); - print(&mut *state); - true - }); + let cb = RemoteCallbacks::new(); + // cb.certificate_check(|cc| { + // true + // }); - let mut co = CheckoutBuilder::new(); - co.progress(|path, cur, total| { - let mut state = state.borrow_mut(); - state.path = path.map(|p| p.to_path_buf()); - state.current = cur; - state.total = total; - print(&mut *state); - }); + let co = CheckoutBuilder::new(); + // co.progress(|path, cur, total| { + // + // }); let mut fo = FetchOptions::new(); fo.remote_callbacks(cb); - RepoBuilder::new() + + return RepoBuilder::new() .fetch_options(fo) .with_checkout(co) - .clone(&args.arg_url, Path::new(&args.arg_path))?; - println!(); - - Ok(()) + .clone(&url, Path::new(&path)); } -pub fn main() { - let args = Args { - arg_url: "https://github.com/DEVELOPEST/gtm.git".parse().unwrap(), - arg_path: "./test".parse().unwrap() - }; - - match run(&args) { - Ok(()) => {} - Err(e) => println!("error: {}", e), - } -} \ No newline at end of file +pub fn fetch(repo: &Repository) { + let _b = repo.remote_add_fetch("origin","+refs/notes/gtm-data:refs/notes/gtm-data").unwrap(); + let mut remote = repo.find_remote("origin").unwrap(); + let gtm_ref= ["main".to_string(), "refs/notes/gtm-data".to_string()]; + remote.fetch(>m_ref, None, None).unwrap(); + remote.disconnect().unwrap(); + + //let mut notes_remote = repo.find_remote("gtm").unwrap(); + //notes_remote.fetch(>m_ref, None, None).unwrap(); + let a = repo.notes(Option::from("refs/notes/gtm-data")).unwrap(); + println!("{}", a.count()); +} diff --git a/src/main.rs b/src/main.rs index 6587b6c..354c24b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,14 +13,22 @@ fn index() -> &'static str { } fn main() { - // git::main(); let cfg = SyncConfig { target_host: "http://our.server.ee".to_string(), target_port: None, port: Some(8765), - repositories: vec![Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2018".to_string() }] + repositories_base_path: "./repos".to_string(), + repositories: vec![Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2018".to_string(), path: "taannu/a".to_string() }, + Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2019".to_string(), path: "taanni/b".to_string() }, + Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2020".to_string(), path: "taannu/c".to_string() }] }; let loc = "./test.toml".to_string(); config::save(&loc, &cfg); - rocket::ignite().mount("/", routes![index]).launch(); + let loaded_cfg = config::load(&loc); + println!("{}", &loaded_cfg.target_host); + let url = "https://github.com/DEVELOPEST/gtm-sync.git".parse().unwrap(); + let path = "./repo".parse().unwrap(); + let repo = git::clone(&url, &path).unwrap(); + let _res = git::fetch(&repo); + //rocket::ignite().mount("/", routes![index]).launch(); } \ No newline at end of file From c8420dcfbf27629e308171761ae8072c76ef70ff Mon Sep 17 00:00:00 2001 From: tavo Date: Sat, 21 Nov 2020 22:18:29 +0200 Subject: [PATCH 03/33] Add fetching notes --- src/git/git.rs | 49 ++++++++++++++++++++++++++++------------- src/git/gtm.rs | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 1 + 3 files changed, 95 insertions(+), 15 deletions(-) create mode 100644 src/git/gtm.rs diff --git a/src/git/git.rs b/src/git/git.rs index 9d81098..e884245 100644 --- a/src/git/git.rs +++ b/src/git/git.rs @@ -1,9 +1,15 @@ #![deny(warnings)] -use git2::build::{CheckoutBuilder, RepoBuilder}; -use git2::{FetchOptions, RemoteCallbacks, Repository, Error}; +use git2::build::{RepoBuilder}; +use git2::{FetchOptions, RemoteCallbacks, Repository, Error, Commit, Oid}; use std::path::{Path}; +mod gtm; + +static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; +static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; +static DEFAULT_ORIGIN: &str = "origin"; + pub fn clone(url: &String, path: &String) -> Result { let cb = RemoteCallbacks::new(); @@ -11,29 +17,42 @@ pub fn clone(url: &String, path: &String) -> Result { // true // }); - let co = CheckoutBuilder::new(); - // co.progress(|path, cur, total| { - // - // }); let mut fo = FetchOptions::new(); fo.remote_callbacks(cb); return RepoBuilder::new() .fetch_options(fo) - .with_checkout(co) .clone(&url, Path::new(&path)); } pub fn fetch(repo: &Repository) { - let _b = repo.remote_add_fetch("origin","+refs/notes/gtm-data:refs/notes/gtm-data").unwrap(); - let mut remote = repo.find_remote("origin").unwrap(); - let gtm_ref= ["main".to_string(), "refs/notes/gtm-data".to_string()]; - remote.fetch(>m_ref, None, None).unwrap(); + repo.remote_add_fetch(DEFAULT_ORIGIN,GTM_NOTES_REF_SPEC) + .expect("Unable to add fetch ref spec for gtm-data!"); + let mut remote = repo.find_remote(DEFAULT_ORIGIN) + .expect("Unable to find remote 'origin'"); + remote.fetch(&[] as &[&str], None, None).expect("Error fetching data!"); remote.disconnect().unwrap(); +} - //let mut notes_remote = repo.find_remote("gtm").unwrap(); - //notes_remote.fetch(>m_ref, None, None).unwrap(); - let a = repo.notes(Option::from("refs/notes/gtm-data")).unwrap(); - println!("{}", a.count()); +pub fn read_commits(repo: &Repository) -> Result, Error> { + let commits : Vec = Vec::new(); + let mut revwalk = repo.revwalk().expect("Unable to revwalk!"); + let _sorting = revwalk.set_sorting(git2::Sort::TIME); + let _head = revwalk.push_head(); + for commit_oid in revwalk { + let commit_oid = commit_oid?; + let commit = repo.find_commit(commit_oid)?; + let notes: Vec = repo.notes(Option::from(GTM_NOTES_REF))? + .map(|n| n.unwrap()) + .filter(|n| n.1 == commit_oid) + .map(|n| n.0) + .collect(); + + let res= gtm::parse_commit(&commit, ¬es)?; + println!("{}", res); + } + // let a = repo.notes(Option::from(GTM_NOTES_REF)) + // .expect("Unable to find gtm-notes"); + return Result::Ok(commits); } diff --git a/src/git/gtm.rs b/src/git/gtm.rs new file mode 100644 index 0000000..f12f424 --- /dev/null +++ b/src/git/gtm.rs @@ -0,0 +1,60 @@ +use git2::{Oid}; +use std::fmt; +use serde::export::Formatter; + +pub struct Commit { + hash: String, + author_email: String, + message: String, + time: i64, + files: Vec +} + +pub struct File { + path: String, + time: i64, + // added_lines: i32, + // deleted_lines: i32, + // changed_lines: i32, +} + +pub fn parse_commit(git_commit: &git2::Commit, note_oids: &[Oid]) -> Result { + let mut commit = Commit { + hash: git_commit.id().to_string(), + author_email: git_commit.author().to_string(), + message: git_commit.message().unwrap().to_string(), + time: git_commit.time().seconds(), // todo: validate + files: vec![] + }; + + for oid in note_oids { + let message = oid.to_string(); + commit.files.push(File { + path: message, + time: 0, + }); + } + + return Ok(commit); +} + +impl fmt::Display for Commit { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let _ = writeln!(f, "Commit: {}", self.hash); + let _ = writeln!(f, "Author: {}", self.author_email); + let _ = writeln!(f, "Time {}", self.time); + let _ = writeln!(f, "{}", self.message); + let _ = writeln!(f); + + for file in &self.files { + let _ = writeln!(f, "{}", &file); + } + writeln!(f) + } +} + +impl fmt::Display for File { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{} : {}", self.time, self.path) + } +} diff --git a/src/main.rs b/src/main.rs index 354c24b..56803c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,5 +30,6 @@ fn main() { let path = "./repo".parse().unwrap(); let repo = git::clone(&url, &path).unwrap(); let _res = git::fetch(&repo); + let _commits = git::read_commits(&repo); //rocket::ignite().mount("/", routes![index]).launch(); } \ No newline at end of file From a8016d9e10b0aca6d29bc721e63669be32320b12 Mon Sep 17 00:00:00 2001 From: tavo Date: Sun, 22 Nov 2020 17:21:00 +0200 Subject: [PATCH 04/33] Check if repo already exists --- src/git/git.rs | 17 +++++++++++++++-- src/main.rs | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/git/git.rs b/src/git/git.rs index e884245..3cd17b0 100644 --- a/src/git/git.rs +++ b/src/git/git.rs @@ -3,6 +3,7 @@ use git2::build::{RepoBuilder}; use git2::{FetchOptions, RemoteCallbacks, Repository, Error, Commit, Oid}; use std::path::{Path}; +use std::fs; mod gtm; @@ -10,7 +11,19 @@ static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; static DEFAULT_ORIGIN: &str = "origin"; -pub fn clone(url: &String, path: &String) -> Result { +pub fn clone_or_open(url: &String, repo_path: &String) -> Result { + + let path = Path::new(&repo_path); + + if path.exists() { + let repo = Repository::open(path); + if repo.is_ok() { + return repo; + } + let _remove = fs::remove_dir_all(&path) + .expect(&*format!("Unable to remove dir: {}", repo_path)); + return clone_or_open(&url, &repo_path); + } let cb = RemoteCallbacks::new(); // cb.certificate_check(|cc| { @@ -23,7 +36,7 @@ pub fn clone(url: &String, path: &String) -> Result { return RepoBuilder::new() .fetch_options(fo) - .clone(&url, Path::new(&path)); + .clone(&url, path); } pub fn fetch(repo: &Repository) { diff --git a/src/main.rs b/src/main.rs index 56803c2..c75bb45 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,7 +28,7 @@ fn main() { println!("{}", &loaded_cfg.target_host); let url = "https://github.com/DEVELOPEST/gtm-sync.git".parse().unwrap(); let path = "./repo".parse().unwrap(); - let repo = git::clone(&url, &path).unwrap(); + let repo = git::clone_or_open(&url, &path).unwrap(); let _res = git::fetch(&repo); let _commits = git::read_commits(&repo); //rocket::ignite().mount("/", routes![index]).launch(); From a3c2dbf54f1fc1626c4efe3a8a38497a7402c1e8 Mon Sep 17 00:00:00 2001 From: tavo Date: Sun, 22 Nov 2020 19:48:33 +0200 Subject: [PATCH 05/33] Correctly parse notes time --- Cargo.toml | 4 ++- src/git/git.rs | 6 ++-- src/git/gtm.rs | 86 +++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 80 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 61cef59..e3452f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,6 @@ rocket = "0.4.6" git2 = "0.13" serde = "1.0" serde_derive = "1.0" -toml = "0.5.7" \ No newline at end of file +toml = "0.5.7" +regex = "1" +lazy_static = "1.4.0" \ No newline at end of file diff --git a/src/git/git.rs b/src/git/git.rs index 3cd17b0..61bad1b 100644 --- a/src/git/git.rs +++ b/src/git/git.rs @@ -1,7 +1,7 @@ #![deny(warnings)] use git2::build::{RepoBuilder}; -use git2::{FetchOptions, RemoteCallbacks, Repository, Error, Commit, Oid}; +use git2::{FetchOptions, RemoteCallbacks, Repository, Error, Commit, Note}; use std::path::{Path}; use std::fs; @@ -56,10 +56,10 @@ pub fn read_commits(repo: &Repository) -> Result, Error> { for commit_oid in revwalk { let commit_oid = commit_oid?; let commit = repo.find_commit(commit_oid)?; - let notes: Vec = repo.notes(Option::from(GTM_NOTES_REF))? + let notes: Vec = repo.notes(Option::from(GTM_NOTES_REF))? .map(|n| n.unwrap()) .filter(|n| n.1 == commit_oid) - .map(|n| n.0) + .map(|n| repo.find_note(Option::from(GTM_NOTES_REF), n.1).unwrap()) .collect(); let res= gtm::parse_commit(&commit, ¬es)?; diff --git a/src/git/gtm.rs b/src/git/gtm.rs index f12f424..76075ad 100644 --- a/src/git/gtm.rs +++ b/src/git/gtm.rs @@ -1,43 +1,105 @@ -use git2::{Oid}; +use std::collections::HashMap; use std::fmt; + +use git2::Note; +use lazy_static::lazy_static; +use regex::Regex; use serde::export::Formatter; +lazy_static! { + static ref NOTE_HEADER_REGEX: Regex = Regex::new("\\[ver:\\d+,total:\\d+]").unwrap(); + static ref NOTE_HEADER_VALS_REGEX: Regex = Regex::new("\\d+").unwrap(); +} + pub struct Commit { hash: String, author_email: String, message: String, time: i64, - files: Vec + files: Vec, } pub struct File { path: String, - time: i64, + time_total: i64, + timeline: HashMap, // added_lines: i32, // deleted_lines: i32, // changed_lines: i32, } -pub fn parse_commit(git_commit: &git2::Commit, note_oids: &[Oid]) -> Result { +pub fn parse_commit(git_commit: &git2::Commit, notes: &[Note]) -> Result { let mut commit = Commit { hash: git_commit.id().to_string(), author_email: git_commit.author().to_string(), message: git_commit.message().unwrap().to_string(), time: git_commit.time().seconds(), // todo: validate - files: vec![] + files: vec![], }; - for oid in note_oids { - let message = oid.to_string(); - commit.files.push(File { - path: message, - time: 0, - }); + for note in notes { + let message = note.message().unwrap(); + commit.files.append(&mut parse_note_message(message)); } return Ok(commit); } +fn parse_note_message(message: &str) -> Vec { + let mut version: String = "".to_string(); + let mut files: Vec = Vec::new(); + let lines = message.split("\n"); + for line in lines { + if line.trim() == "" { + version = "".to_string(); + } else if NOTE_HEADER_REGEX.is_match(line) { + let matches: Vec = NOTE_HEADER_VALS_REGEX.find_iter(line) + .filter_map(|d| d.as_str().parse().ok()) + .collect(); + version = matches.get(0).unwrap().clone(); + } + if version == "1" { + let field_groups: Vec<&str> = line.split(",").collect(); + let mut file = File { + path: "".to_string(), + time_total: 0, + timeline: HashMap::new(), + }; + + if field_groups.len() < 3 { + continue; + } + for i in 0..field_groups.len() { + let fields: Vec<&str> = field_groups.get(i).unwrap().split(":").collect(); + if i == 0 && fields.len() == 2 { + file.path = fields.get(0).unwrap().replace("->", ":"); + file.time_total = fields.get(1).unwrap().parse().unwrap(); + } else if i == field_groups.len() - 1 && fields.len() == 1 { + // todo: file status? + } else if fields.len() == 2 { + let epoch_timeline: i64 = fields.get(0).unwrap().parse().unwrap(); + let epoch_total: i32 = fields.get(1).unwrap().parse().unwrap(); + file.timeline.insert(epoch_timeline, epoch_total); + } + } + let mut found: bool = false; + for mut added_file in files.iter_mut() { + if added_file.path == file.path { + added_file.time_total += file.time_total; + for (epoch, secs) in &file.timeline { + added_file.timeline.insert(*epoch, *secs); + } + found = true; + } + } + if !found { + files.push(file); + } + } + } + return files; +} + impl fmt::Display for Commit { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { let _ = writeln!(f, "Commit: {}", self.hash); @@ -55,6 +117,6 @@ impl fmt::Display for Commit { impl fmt::Display for File { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{} : {}", self.time, self.path) + write!(f, "{:>2}h{:>3}m{:>3}s : {}", self.time_total / 3600, (self.time_total % 3600) / 60, self.time_total % 60, self.path) } } From bde488819521c141d0eb967a971edc63ccb6fe6e Mon Sep 17 00:00:00 2001 From: tavo Date: Sun, 22 Nov 2020 21:05:25 +0200 Subject: [PATCH 06/33] Add finding diff for commits --- src/git/git.rs | 2 +- src/git/gtm.rs | 94 ++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 69 insertions(+), 27 deletions(-) diff --git a/src/git/git.rs b/src/git/git.rs index 61bad1b..1906bfa 100644 --- a/src/git/git.rs +++ b/src/git/git.rs @@ -62,7 +62,7 @@ pub fn read_commits(repo: &Repository) -> Result, Error> { .map(|n| repo.find_note(Option::from(GTM_NOTES_REF), n.1).unwrap()) .collect(); - let res= gtm::parse_commit(&commit, ¬es)?; + let res= gtm::parse_commit(&repo, &commit, ¬es)?; println!("{}", res); } // let a = repo.notes(Option::from(GTM_NOTES_REF)) diff --git a/src/git/gtm.rs b/src/git/gtm.rs index 76075ad..da0df99 100644 --- a/src/git/gtm.rs +++ b/src/git/gtm.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::fmt; -use git2::Note; +use git2::{DiffOptions, Note}; use lazy_static::lazy_static; use regex::Regex; use serde::export::Formatter; @@ -23,12 +23,11 @@ pub struct File { path: String, time_total: i64, timeline: HashMap, - // added_lines: i32, - // deleted_lines: i32, - // changed_lines: i32, + added_lines: i32, + deleted_lines: i32, } -pub fn parse_commit(git_commit: &git2::Commit, notes: &[Note]) -> Result { +pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note]) -> Result { let mut commit = Commit { hash: git_commit.id().to_string(), author_email: git_commit.author().to_string(), @@ -39,7 +38,9 @@ pub fn parse_commit(git_commit: &git2::Commit, notes: &[Note]) -> Result Vec { .collect(); version = matches.get(0).unwrap().clone(); } + + let mut file = File { + path: "".to_string(), + time_total: 0, + timeline: HashMap::new(), + added_lines: 0, + deleted_lines: 0, + }; + if version == "1" { let field_groups: Vec<&str> = line.split(",").collect(); - let mut file = File { - path: "".to_string(), - time_total: 0, - timeline: HashMap::new(), - }; - if field_groups.len() < 3 { continue; } @@ -76,47 +80,85 @@ fn parse_note_message(message: &str) -> Vec { file.time_total = fields.get(1).unwrap().parse().unwrap(); } else if i == field_groups.len() - 1 && fields.len() == 1 { // todo: file status? + continue; } else if fields.len() == 2 { let epoch_timeline: i64 = fields.get(0).unwrap().parse().unwrap(); let epoch_total: i32 = fields.get(1).unwrap().parse().unwrap(); file.timeline.insert(epoch_timeline, epoch_total); } } - let mut found: bool = false; - for mut added_file in files.iter_mut() { - if added_file.path == file.path { - added_file.time_total += file.time_total; - for (epoch, secs) in &file.timeline { - added_file.timeline.insert(*epoch, *secs); - } - found = true; + } else { + continue; + } + + let mut found: bool = false; + for mut added_file in files.iter_mut() { + if added_file.path == file.path { + added_file.time_total += file.time_total; + for (epoch, secs) in &file.timeline { + added_file.timeline.insert(*epoch, *secs); } - } - if !found { - files.push(file); + found = true; } } + if !found { + files.push(file); + } } return files; } +fn diff_parents(files: &mut Vec, commit: &git2::Commit, repo: &git2::Repository) -> Result<(), git2::Error> { + if commit.parent_count() == 0 { + // todo: first commit + return Ok(()); + } + + let parent = commit.parent(0)?; + let child_tree = commit.tree()?; + let parent_tree = parent.tree()?; + + for mut file in files { + if file.path.ends_with(".app") { + continue; // Skip app events + } + let mut diff_options = DiffOptions::new(); + diff_options.pathspec(&file.path); + let diff = repo.diff_tree_to_tree( + Option::from(&parent_tree), + Option::from(&child_tree), + Option::from(&mut diff_options))?; + let diff_stats = diff.stats()?; + file.added_lines = diff_stats.insertions() as i32; + file.deleted_lines = diff_stats.deletions() as i32; + } + + return Ok(()); +} + impl fmt::Display for Commit { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { let _ = writeln!(f, "Commit: {}", self.hash); let _ = writeln!(f, "Author: {}", self.author_email); let _ = writeln!(f, "Time {}", self.time); let _ = writeln!(f, "{}", self.message); - let _ = writeln!(f); for file in &self.files { let _ = writeln!(f, "{}", &file); } - writeln!(f) + Ok(()) } } impl fmt::Display for File { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{:>2}h{:>3}m{:>3}s : {}", self.time_total / 3600, (self.time_total % 3600) / 60, self.time_total % 60, self.path) + write!(f, "{:>2}h{:>3}m{:>3}s : {:<35} +{:<4} - {}", + self.time_total / 3600, + (self.time_total % 3600) / 60, + self.time_total % 60, + self.path, + self.added_lines, + self.deleted_lines, + ) } } From 6ef24674a637721ac88a5c338250b2404112c0f3 Mon Sep 17 00:00:00 2001 From: tavo Date: Mon, 23 Nov 2020 10:58:55 +0200 Subject: [PATCH 07/33] Add ssh authorization --- src/config/config.rs | 4 +++ src/git/git.rs | 72 ++++++++++++++++++++++++++++++++------------ src/git/gtm.rs | 2 +- src/main.rs | 18 ++++++----- 4 files changed, 68 insertions(+), 28 deletions(-) diff --git a/src/config/config.rs b/src/config/config.rs index a49eb45..8e0cab4 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -15,6 +15,10 @@ pub struct SyncConfig { pub struct Repository { pub url: String, pub path: String, + pub ssh_private_key: Option, + pub ssh_public_key: Option, + pub ssh_user: Option, + pub ssh_passphrase: Option, } pub fn load(config_file: &String) -> SyncConfig { diff --git a/src/git/git.rs b/src/git/git.rs index 1906bfa..5da6f81 100644 --- a/src/git/git.rs +++ b/src/git/git.rs @@ -1,19 +1,21 @@ #![deny(warnings)] -use git2::build::{RepoBuilder}; -use git2::{FetchOptions, RemoteCallbacks, Repository, Error, Commit, Note}; -use std::path::{Path}; use std::fs; +use std::path::Path; + +use git2::{Commit, Error, FetchOptions, Note, RemoteCallbacks, Repository}; +use git2::build::RepoBuilder; mod gtm; +#[path = "../config/config.rs"] +pub mod config; static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; static DEFAULT_ORIGIN: &str = "origin"; -pub fn clone_or_open(url: &String, repo_path: &String) -> Result { - - let path = Path::new(&repo_path); +pub fn clone_or_open(repo_config: &config::Repository) -> Result { + let path = Path::new(&repo_config.path); if path.exists() { let repo = Repository::open(path); @@ -21,30 +23,60 @@ pub fn clone_or_open(url: &String, repo_path: &String) -> Result FetchOptions { + let mut cb = RemoteCallbacks::new(); + let repo_config = repo_config.clone(); + cb.credentials(move |_c, _o, t| { + if t.is_ssh_key() { + return git2::Cred::ssh_key( + &repo_config.ssh_user.as_ref().unwrap(), + Option::from(Path::new(&repo_config.ssh_public_key.as_ref().unwrap())), + &Path::new(&repo_config.ssh_private_key.as_ref().unwrap()), + repo_config.ssh_passphrase.as_ref().map(|x| &**x), + ) + } + return git2::Cred::default(); + }); let mut fo = FetchOptions::new(); fo.remote_callbacks(cb); - - return RepoBuilder::new() - .fetch_options(fo) - .clone(&url, path); + return fo; } -pub fn fetch(repo: &Repository) { - repo.remote_add_fetch(DEFAULT_ORIGIN,GTM_NOTES_REF_SPEC) - .expect("Unable to add fetch ref spec for gtm-data!"); + +pub fn fetch(repo: &Repository, repo_config: &config::Repository) { let mut remote = repo.find_remote(DEFAULT_ORIGIN) .expect("Unable to find remote 'origin'"); - remote.fetch(&[] as &[&str], None, None).expect("Error fetching data!"); + let mut ref_added = false; + let refs = remote.fetch_refspecs().unwrap(); + for i in 0..refs.len() { + if refs.get(i).unwrap() == GTM_NOTES_REF_SPEC { + ref_added = true; + break; + } + } + if !ref_added { + repo.remote_add_fetch(DEFAULT_ORIGIN, GTM_NOTES_REF_SPEC) + .expect("Unable to add fetch ref spec for gtm-data!"); + remote = repo.find_remote(DEFAULT_ORIGIN) + .expect("Unable to find remote 'origin'"); + } + + let mut fo = generate_fetch_options(repo_config); + remote.fetch(&[] as &[&str], Option::from(&mut fo), None) + .expect("Error fetching data!"); remote.disconnect().unwrap(); } diff --git a/src/git/gtm.rs b/src/git/gtm.rs index da0df99..a25b962 100644 --- a/src/git/gtm.rs +++ b/src/git/gtm.rs @@ -152,7 +152,7 @@ impl fmt::Display for Commit { impl fmt::Display for File { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{:>2}h{:>3}m{:>3}s : {:<35} +{:<4} - {}", + write!(f, "{:>2}h{:>3}m{:>3}s : {:<45} +{:<4} - {}", self.time_total / 3600, (self.time_total % 3600) / 60, self.time_total % 60, diff --git a/src/main.rs b/src/main.rs index c75bb45..d3c3593 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,18 +18,22 @@ fn main() { target_port: None, port: Some(8765), repositories_base_path: "./repos".to_string(), - repositories: vec![Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2018".to_string(), path: "taannu/a".to_string() }, - Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2019".to_string(), path: "taanni/b".to_string() }, - Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2020".to_string(), path: "taannu/c".to_string() }] + repositories: vec![Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2018".to_string(), path: "taannu/a".to_string(), ssh_private_key: None, ssh_public_key: None, ssh_user: None, ssh_passphrase: None }] }; let loc = "./test.toml".to_string(); config::save(&loc, &cfg); let loaded_cfg = config::load(&loc); println!("{}", &loaded_cfg.target_host); - let url = "https://github.com/DEVELOPEST/gtm-sync.git".parse().unwrap(); - let path = "./repo".parse().unwrap(); - let repo = git::clone_or_open(&url, &path).unwrap(); - let _res = git::fetch(&repo); + let repo_to_clone: git::config::Repository = git::config::Repository{ + url: "git@gitlab.cs.ttu.ee:taannu/icd0008-2020f.git".to_string(), + path: "./repo".to_string(), + ssh_private_key: Option::from("/home/tavo/.ssh/id_git".to_string()), + ssh_public_key: Option::from("/home/tavo/.ssh/id_git.pub".to_string()), + ssh_user: Option::from("git".to_string()), + ssh_passphrase: None + }; + let repo = git::clone_or_open(&repo_to_clone).unwrap(); + let _res = git::fetch(&repo, &repo_to_clone); let _commits = git::read_commits(&repo); //rocket::ignite().mount("/", routes![index]).launch(); } \ No newline at end of file From e1d92d1efeead0ea2b10786848afcfa2648deffb Mon Sep 17 00:00:00 2001 From: tavo Date: Mon, 23 Nov 2020 21:10:37 +0200 Subject: [PATCH 08/33] Add modification types --- src/git/git.rs | 9 ++++----- src/git/gtm.rs | 28 +++++++++++++++------------- src/main.rs | 21 ++------------------- 3 files changed, 21 insertions(+), 37 deletions(-) diff --git a/src/git/git.rs b/src/git/git.rs index 5da6f81..a638a1e 100644 --- a/src/git/git.rs +++ b/src/git/git.rs @@ -5,10 +5,9 @@ use std::path::Path; use git2::{Commit, Error, FetchOptions, Note, RemoteCallbacks, Repository}; use git2::build::RepoBuilder; +use crate::config; mod gtm; -#[path = "../config/config.rs"] -pub mod config; static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; @@ -40,9 +39,9 @@ fn generate_fetch_options(repo_config: &config::Repository) -> FetchOptions { cb.credentials(move |_c, _o, t| { if t.is_ssh_key() { return git2::Cred::ssh_key( - &repo_config.ssh_user.as_ref().unwrap(), - Option::from(Path::new(&repo_config.ssh_public_key.as_ref().unwrap())), - &Path::new(&repo_config.ssh_private_key.as_ref().unwrap()), + &repo_config.ssh_user.as_ref().unwrap_or(&"git".to_string()), + Option::from(Path::new(&repo_config.ssh_public_key.as_ref().unwrap_or(&"".to_string()))), + &Path::new(&repo_config.ssh_private_key.as_ref().unwrap_or(&"".to_string())), repo_config.ssh_passphrase.as_ref().map(|x| &**x), ) } diff --git a/src/git/gtm.rs b/src/git/gtm.rs index a25b962..eb96f83 100644 --- a/src/git/gtm.rs +++ b/src/git/gtm.rs @@ -23,6 +23,7 @@ pub struct File { path: String, time_total: i64, timeline: HashMap, + status: String, added_lines: i32, deleted_lines: i32, } @@ -38,7 +39,7 @@ pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: & for note in notes { let message = note.message().unwrap(); - let mut files = parse_note_message(message); + let mut files = parse_note_message(message).unwrap_or(vec![]); let _diff = diff_parents(files.as_mut(), git_commit, repo); commit.files.append(files.as_mut()); } @@ -46,7 +47,7 @@ pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: & return Ok(commit); } -fn parse_note_message(message: &str) -> Vec { +fn parse_note_message(message: &str) -> Option> { let mut version: String = "".to_string(); let mut files: Vec = Vec::new(); let lines = message.split("\n"); @@ -57,13 +58,14 @@ fn parse_note_message(message: &str) -> Vec { let matches: Vec = NOTE_HEADER_VALS_REGEX.find_iter(line) .filter_map(|d| d.as_str().parse().ok()) .collect(); - version = matches.get(0).unwrap().clone(); + version = matches.get(0)?.clone(); } let mut file = File { path: "".to_string(), time_total: 0, timeline: HashMap::new(), + status: "".to_string(), added_lines: 0, deleted_lines: 0, }; @@ -74,16 +76,15 @@ fn parse_note_message(message: &str) -> Vec { continue; } for i in 0..field_groups.len() { - let fields: Vec<&str> = field_groups.get(i).unwrap().split(":").collect(); + let fields: Vec<&str> = field_groups.get(i)?.split(":").collect(); if i == 0 && fields.len() == 2 { - file.path = fields.get(0).unwrap().replace("->", ":"); - file.time_total = fields.get(1).unwrap().parse().unwrap(); + file.path = fields.get(0)?.replace("->", ":"); + file.time_total = fields.get(1)?.parse().unwrap_or(0); } else if i == field_groups.len() - 1 && fields.len() == 1 { - // todo: file status? - continue; + file.status = fields.get(0)?.to_string(); } else if fields.len() == 2 { - let epoch_timeline: i64 = fields.get(0).unwrap().parse().unwrap(); - let epoch_total: i32 = fields.get(1).unwrap().parse().unwrap(); + let epoch_timeline: i64 = fields.get(0)?.parse().unwrap_or(0); + let epoch_total: i32 = fields.get(1)?.parse().unwrap_or(0); file.timeline.insert(epoch_timeline, epoch_total); } } @@ -105,12 +106,12 @@ fn parse_note_message(message: &str) -> Vec { files.push(file); } } - return files; + return Option::from(files); } fn diff_parents(files: &mut Vec, commit: &git2::Commit, repo: &git2::Repository) -> Result<(), git2::Error> { if commit.parent_count() == 0 { - // todo: first commit + // TODO: Figure out how to handle initial commit return Ok(()); } @@ -152,13 +153,14 @@ impl fmt::Display for Commit { impl fmt::Display for File { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{:>2}h{:>3}m{:>3}s : {:<45} +{:<4} - {}", + write!(f, "{:>2}h{:>3}m{:>3}s : {:<45} +{:<4} - {:<4} {}", self.time_total / 3600, (self.time_total % 3600) / 60, self.time_total % 60, self.path, self.added_lines, self.deleted_lines, + self.status, ) } } diff --git a/src/main.rs b/src/main.rs index d3c3593..2bc5911 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,8 +2,6 @@ #[macro_use] extern crate rocket; -use crate::config::{SyncConfig, Repository}; - #[path = "git/git.rs"] mod git; #[path = "config/config.rs"] mod config; @@ -13,25 +11,10 @@ fn index() -> &'static str { } fn main() { - let cfg = SyncConfig { - target_host: "http://our.server.ee".to_string(), - target_port: None, - port: Some(8765), - repositories_base_path: "./repos".to_string(), - repositories: vec![Repository{ url: "http://gitlab.cs.ttu.ee/taannu/iti0201-2018".to_string(), path: "taannu/a".to_string(), ssh_private_key: None, ssh_public_key: None, ssh_user: None, ssh_passphrase: None }] - }; - let loc = "./test.toml".to_string(); - config::save(&loc, &cfg); + let loc = "./example_config.toml".to_string(); let loaded_cfg = config::load(&loc); println!("{}", &loaded_cfg.target_host); - let repo_to_clone: git::config::Repository = git::config::Repository{ - url: "git@gitlab.cs.ttu.ee:taannu/icd0008-2020f.git".to_string(), - path: "./repo".to_string(), - ssh_private_key: Option::from("/home/tavo/.ssh/id_git".to_string()), - ssh_public_key: Option::from("/home/tavo/.ssh/id_git.pub".to_string()), - ssh_user: Option::from("git".to_string()), - ssh_passphrase: None - }; + let repo_to_clone = loaded_cfg.repositories.get(0).unwrap(); let repo = git::clone_or_open(&repo_to_clone).unwrap(); let _res = git::fetch(&repo, &repo_to_clone); let _commits = git::read_commits(&repo); From 697ba422d06126e67fc1fda18a4c8b9464208bf2 Mon Sep 17 00:00:00 2001 From: tavo Date: Tue, 24 Nov 2020 12:02:44 +0200 Subject: [PATCH 09/33] Basically working json generation --- Cargo.toml | 11 ++++++++--- src/config/config.rs | 2 +- src/git/git.rs | 11 ++++++----- src/git/gtm.rs | 5 +++++ src/main.rs | 15 ++------------- src/server/controller.rs | 14 ++++++++++++++ src/server/server.rs | 7 +++++++ src/server/service.rs | 15 +++++++++++++++ 8 files changed, 58 insertions(+), 22 deletions(-) create mode 100644 src/server/controller.rs create mode 100644 src/server/server.rs create mode 100644 src/server/service.rs diff --git a/Cargo.toml b/Cargo.toml index e3452f6..bc47164 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,8 +9,13 @@ edition = "2018" [dependencies] rocket = "0.4.6" git2 = "0.13" -serde = "1.0" -serde_derive = "1.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.44" toml = "0.5.7" regex = "1" -lazy_static = "1.4.0" \ No newline at end of file +lazy_static = "1.4.0" + +[dependencies.rocket_contrib] +version = "0.4.2" +default-features = false +features = ["json"] \ No newline at end of file diff --git a/src/config/config.rs b/src/config/config.rs index 8e0cab4..ace2571 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -1,5 +1,5 @@ -use serde_derive::{Serialize, Deserialize}; +use serde::{Serialize, Deserialize}; use std::fs; #[derive(Serialize, Deserialize)] diff --git a/src/git/git.rs b/src/git/git.rs index a638a1e..35a9b17 100644 --- a/src/git/git.rs +++ b/src/git/git.rs @@ -3,11 +3,11 @@ use std::fs; use std::path::Path; -use git2::{Commit, Error, FetchOptions, Note, RemoteCallbacks, Repository}; +use git2::{Error, FetchOptions, Note, RemoteCallbacks, Repository}; use git2::build::RepoBuilder; use crate::config; -mod gtm; +pub(crate) mod gtm; static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; @@ -79,8 +79,8 @@ pub fn fetch(repo: &Repository, repo_config: &config::Repository) { remote.disconnect().unwrap(); } -pub fn read_commits(repo: &Repository) -> Result, Error> { - let commits : Vec = Vec::new(); +pub fn read_commits(repo: &Repository) -> Result, Error> { + let mut commits: Vec = Vec::new(); let mut revwalk = repo.revwalk().expect("Unable to revwalk!"); let _sorting = revwalk.set_sorting(git2::Sort::TIME); let _head = revwalk.push_head(); @@ -94,7 +94,8 @@ pub fn read_commits(repo: &Repository) -> Result, Error> { .collect(); let res= gtm::parse_commit(&repo, &commit, ¬es)?; - println!("{}", res); + println!("{}", &res); + commits.push(res); } // let a = repo.notes(Option::from(GTM_NOTES_REF)) // .expect("Unable to find gtm-notes"); diff --git a/src/git/gtm.rs b/src/git/gtm.rs index eb96f83..c1888ed 100644 --- a/src/git/gtm.rs +++ b/src/git/gtm.rs @@ -5,20 +5,24 @@ use git2::{DiffOptions, Note}; use lazy_static::lazy_static; use regex::Regex; use serde::export::Formatter; +use serde::Serialize; lazy_static! { static ref NOTE_HEADER_REGEX: Regex = Regex::new("\\[ver:\\d+,total:\\d+]").unwrap(); static ref NOTE_HEADER_VALS_REGEX: Regex = Regex::new("\\d+").unwrap(); } +#[derive(Serialize)] pub struct Commit { hash: String, + // branch: String, author_email: String, message: String, time: i64, files: Vec, } +#[derive(Serialize)] pub struct File { path: String, time_total: i64, @@ -31,6 +35,7 @@ pub struct File { pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note]) -> Result { let mut commit = Commit { hash: git_commit.id().to_string(), + // branch: "".to_string(), author_email: git_commit.author().to_string(), message: git_commit.message().unwrap().to_string(), time: git_commit.time().seconds(), // todo: validate diff --git a/src/main.rs b/src/main.rs index 2bc5911..5cfe8df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,19 +4,8 @@ #[path = "git/git.rs"] mod git; #[path = "config/config.rs"] mod config; - -#[get("/")] -fn index() -> &'static str { - "Hello, world!" -} +#[path = "server/server.rs"] mod server; fn main() { - let loc = "./example_config.toml".to_string(); - let loaded_cfg = config::load(&loc); - println!("{}", &loaded_cfg.target_host); - let repo_to_clone = loaded_cfg.repositories.get(0).unwrap(); - let repo = git::clone_or_open(&repo_to_clone).unwrap(); - let _res = git::fetch(&repo, &repo_to_clone); - let _commits = git::read_commits(&repo); - //rocket::ignite().mount("/", routes![index]).launch(); + server::run(); } \ No newline at end of file diff --git a/src/server/controller.rs b/src/server/controller.rs new file mode 100644 index 0000000..d7150d7 --- /dev/null +++ b/src/server/controller.rs @@ -0,0 +1,14 @@ +use rocket_contrib::json::{JsonValue}; + +use crate::server::service; + +#[get("/")] +pub fn index() -> &'static str { + "Hello, world!" +} + +#[get("/repo")] +pub fn repo() -> JsonValue { + let repo = service::get_repo(); + rocket_contrib::json!(&repo) +} \ No newline at end of file diff --git a/src/server/server.rs b/src/server/server.rs new file mode 100644 index 0000000..5f6ae20 --- /dev/null +++ b/src/server/server.rs @@ -0,0 +1,7 @@ + +mod controller; +mod service; + +pub fn run() { + rocket::ignite().mount("/", routes![controller::index, controller::repo]).launch(); +} \ No newline at end of file diff --git a/src/server/service.rs b/src/server/service.rs new file mode 100644 index 0000000..495e8f8 --- /dev/null +++ b/src/server/service.rs @@ -0,0 +1,15 @@ +use crate::config; +use crate::git; +use crate::git::gtm; + +// TODO: (Tavo) Repo wrapper +pub fn get_repo() -> Vec { + let loc = "./example_config.toml".to_string(); + let loaded_cfg = config::load(&loc); + println!("{}", &loaded_cfg.target_host); + let repo_to_clone = loaded_cfg.repositories.get(0).unwrap(); + let repo = git::clone_or_open(&repo_to_clone).unwrap(); + let _res = git::fetch(&repo, &repo_to_clone); + let commits = git::read_commits(&repo).unwrap(); + return commits; +} \ No newline at end of file From 113316db0f7a65fed2d711dbbc4860d5dc68c99c Mon Sep 17 00:00:00 2001 From: tavo Date: Wed, 25 Nov 2020 21:42:49 +0200 Subject: [PATCH 10/33] Add api for adding repositories --- src/config/config.rs | 26 +++--------------------- src/dto/mod.rs | 2 ++ src/dto/request.rs | 24 ++++++++++++++++++++++ src/dto/response.rs | 7 +++++++ src/git/git.rs | 8 ++++---- src/main.rs | 2 ++ src/model/model.rs | 21 ++++++++++++++++++++ src/server/controller.rs | 15 ++++++++++---- src/server/server.rs | 8 +++++++- src/server/service.rs | 43 +++++++++++++++++++++++++++++++++++++++- 10 files changed, 123 insertions(+), 33 deletions(-) create mode 100644 src/dto/mod.rs create mode 100644 src/dto/request.rs create mode 100644 src/dto/response.rs create mode 100644 src/model/model.rs diff --git a/src/config/config.rs b/src/config/config.rs index ace2571..2db7560 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -1,32 +1,12 @@ - -use serde::{Serialize, Deserialize}; use std::fs; +use crate::model::Config; -#[derive(Serialize, Deserialize)] -pub struct SyncConfig { - pub target_host: String, - pub target_port: Option, - pub port: Option, - pub repositories_base_path: String, - pub repositories: Vec, -} - -#[derive(Serialize, Deserialize)] -pub struct Repository { - pub url: String, - pub path: String, - pub ssh_private_key: Option, - pub ssh_public_key: Option, - pub ssh_user: Option, - pub ssh_passphrase: Option, -} - -pub fn load(config_file: &String) -> SyncConfig { +pub fn load(config_file: &String) -> Config { let content = fs::read_to_string(config_file).expect("Unable to read config!"); return toml::from_str(&content).expect("Unable to deserialize config!"); } -pub fn save(config_file: &String, config: &SyncConfig) { +pub fn save(config_file: &String, config: &Config) { let content = toml::to_string(config).expect("Unable to serialize config!"); fs::write(config_file, content).expect("Unable to save config!"); } \ No newline at end of file diff --git a/src/dto/mod.rs b/src/dto/mod.rs new file mode 100644 index 0000000..4015ff8 --- /dev/null +++ b/src/dto/mod.rs @@ -0,0 +1,2 @@ +pub mod request; +pub mod response; \ No newline at end of file diff --git a/src/dto/request.rs b/src/dto/request.rs new file mode 100644 index 0000000..fcfb822 --- /dev/null +++ b/src/dto/request.rs @@ -0,0 +1,24 @@ +use serde::{Serialize, Deserialize}; +use crate::model::Repository; + +#[derive(Serialize, Deserialize)] +pub struct AddRepositoryDto { + pub url: String, + pub ssh_private_key: Option, + pub ssh_public_key: Option, + pub ssh_user: Option, + pub ssh_passphrase: Option, +} + +impl AddRepositoryDto { + pub fn to_repository(&self, f: &dyn Fn(&String) -> String) -> Repository { + return Repository { + url: self.url.clone(), + path: f(&self.url.to_string()), + ssh_private_key: self.ssh_private_key.clone(), + ssh_public_key: self.ssh_public_key.clone(), + ssh_user: self.ssh_user.clone(), + ssh_passphrase: self.ssh_passphrase.clone(), + } + } +} diff --git a/src/dto/response.rs b/src/dto/response.rs new file mode 100644 index 0000000..81fb5a3 --- /dev/null +++ b/src/dto/response.rs @@ -0,0 +1,7 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct BoolResponseDto { + pub success: bool, + pub message: Option, +} \ No newline at end of file diff --git a/src/git/git.rs b/src/git/git.rs index 35a9b17..ded3dcd 100644 --- a/src/git/git.rs +++ b/src/git/git.rs @@ -5,7 +5,7 @@ use std::path::Path; use git2::{Error, FetchOptions, Note, RemoteCallbacks, Repository}; use git2::build::RepoBuilder; -use crate::config; +use crate::model; pub(crate) mod gtm; @@ -13,7 +13,7 @@ static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; static DEFAULT_ORIGIN: &str = "origin"; -pub fn clone_or_open(repo_config: &config::Repository) -> Result { +pub fn clone_or_open(repo_config: &model::Repository) -> Result { let path = Path::new(&repo_config.path); if path.exists() { @@ -33,7 +33,7 @@ pub fn clone_or_open(repo_config: &config::Repository) -> Result FetchOptions { +fn generate_fetch_options(repo_config: &model::Repository) -> FetchOptions { let mut cb = RemoteCallbacks::new(); let repo_config = repo_config.clone(); cb.credentials(move |_c, _o, t| { @@ -55,7 +55,7 @@ fn generate_fetch_options(repo_config: &config::Repository) -> FetchOptions { } -pub fn fetch(repo: &Repository, repo_config: &config::Repository) { +pub fn fetch(repo: &Repository, repo_config: &model::Repository) { let mut remote = repo.find_remote(DEFAULT_ORIGIN) .expect("Unable to find remote 'origin'"); let mut ref_added = false; diff --git a/src/main.rs b/src/main.rs index 5cfe8df..a768a4a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,8 @@ #[path = "git/git.rs"] mod git; #[path = "config/config.rs"] mod config; #[path = "server/server.rs"] mod server; +#[path = "model/model.rs"] mod model; +mod dto; fn main() { server::run(); diff --git a/src/model/model.rs b/src/model/model.rs new file mode 100644 index 0000000..5932b3f --- /dev/null +++ b/src/model/model.rs @@ -0,0 +1,21 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct Config { + pub target_host: String, + pub target_port: Option, + pub port: Option, + pub repositories_base_path: String, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub repositories: Vec, +} + +#[derive(Serialize, Deserialize)] +pub struct Repository { + pub url: String, + pub path: String, + pub ssh_private_key: Option, + pub ssh_public_key: Option, + pub ssh_user: Option, + pub ssh_passphrase: Option, +} diff --git a/src/server/controller.rs b/src/server/controller.rs index d7150d7..f0c5b4c 100644 --- a/src/server/controller.rs +++ b/src/server/controller.rs @@ -1,14 +1,21 @@ -use rocket_contrib::json::{JsonValue}; +use rocket_contrib::json::{JsonValue, Json}; use crate::server::service; +use crate::dto::request::AddRepositoryDto; #[get("/")] pub fn index() -> &'static str { "Hello, world!" } -#[get("/repo")] -pub fn repo() -> JsonValue { - let repo = service::get_repo(); +#[get("/repository///")] +pub fn repo(provider: String, user: String, repo: String) -> JsonValue { + let repo = service::get_repo(); // TODO: How to match credentials? rocket_contrib::json!(&repo) +} + +#[post("/repository", data="")] +pub fn add_repo(repo: Json) -> JsonValue { + let response = service::add_repo(repo.into_inner()); + rocket_contrib::json!(&response) } \ No newline at end of file diff --git a/src/server/server.rs b/src/server/server.rs index 5f6ae20..7be4fda 100644 --- a/src/server/server.rs +++ b/src/server/server.rs @@ -3,5 +3,11 @@ mod controller; mod service; pub fn run() { - rocket::ignite().mount("/", routes![controller::index, controller::repo]).launch(); + rocket::ignite() + .mount("/", + routes![ + controller::index, + controller::repo, + controller::add_repo], + ).launch(); } \ No newline at end of file diff --git a/src/server/service.rs b/src/server/service.rs index 495e8f8..673f23d 100644 --- a/src/server/service.rs +++ b/src/server/service.rs @@ -1,15 +1,56 @@ +use lazy_static::lazy_static; +use regex::Regex; + use crate::config; +use crate::dto::request::AddRepositoryDto; +use crate::dto::response::BoolResponseDto; use crate::git; use crate::git::gtm; +lazy_static! { + static ref PATH_FROM_URL_REGEX: Regex = + Regex::new(r#"(git@|https://)([a-zA-Z0-9.]+)[:/]([a-zA-Z0-9-]+)/([a-zA-Z0-9-]+)\.git"#).unwrap(); +} + // TODO: (Tavo) Repo wrapper pub fn get_repo() -> Vec { let loc = "./example_config.toml".to_string(); let loaded_cfg = config::load(&loc); - println!("{}", &loaded_cfg.target_host); let repo_to_clone = loaded_cfg.repositories.get(0).unwrap(); let repo = git::clone_or_open(&repo_to_clone).unwrap(); let _res = git::fetch(&repo, &repo_to_clone); let commits = git::read_commits(&repo).unwrap(); return commits; +} + +pub fn add_repo(repo_dto: AddRepositoryDto) -> BoolResponseDto { + let loc = "./example_config.toml".to_string(); + let mut loaded_cfg = config::load(&loc); + let repo = repo_dto.to_repository(&|url: &String| { generate_path_from_git_url(url, &loaded_cfg.repositories_base_path) }); + let cloned_repo = git::clone_or_open(&repo); + if cloned_repo.is_ok() { + if !loaded_cfg.repositories.iter().any(|r| r.url == repo_dto.url) { + loaded_cfg.repositories.push(repo); + config::save(&loc, &loaded_cfg); + } + return BoolResponseDto { + success: true, + message: None, + }; + } + return BoolResponseDto { + success: false, + message: cloned_repo.err().map(|e| e.to_string()), + }; +} + +pub fn generate_path_from_git_url(url: &String, base_path: &String) -> String { + let caps = PATH_FROM_URL_REGEX.captures(url).unwrap(); + let path = format!("{}/{}/{}/{}", + base_path.trim_end_matches("/"), + caps.get(2).map_or("provider", |m| m.as_str()), + caps.get(3).map_or("user", |m| m.as_str()), + caps.get(4).map_or("repo", |m| m.as_str()) + ); + return path; } \ No newline at end of file From 0df2c85ba30ffb50609a613ef0d5d46ee206bda5 Mon Sep 17 00:00:00 2001 From: tavo Date: Sat, 28 Nov 2020 21:24:29 +0200 Subject: [PATCH 11/33] Make use of mod.rs --- src/config/config.rs | 2 +- src/config/mod.rs | 1 + src/dto/request.rs | 2 +- src/{git => gtm}/git.rs | 10 +++++----- src/{git => gtm}/gtm.rs | 0 src/gtm/mod.rs | 2 ++ src/main.rs | 10 +++++----- src/model/{model.rs => config.rs} | 0 src/model/mod.rs | 1 + src/server/mod.rs | 4 ++++ src/server/server.rs | 4 ++-- src/server/service.rs | 6 +++--- 12 files changed, 25 insertions(+), 17 deletions(-) create mode 100644 src/config/mod.rs rename src/{git => gtm}/git.rs (92%) rename src/{git => gtm}/gtm.rs (100%) create mode 100644 src/gtm/mod.rs rename src/model/{model.rs => config.rs} (100%) create mode 100644 src/model/mod.rs create mode 100644 src/server/mod.rs diff --git a/src/config/config.rs b/src/config/config.rs index 2db7560..7489574 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -1,5 +1,5 @@ use std::fs; -use crate::model::Config; +use crate::model::config::Config; pub fn load(config_file: &String) -> Config { let content = fs::read_to_string(config_file).expect("Unable to read config!"); diff --git a/src/config/mod.rs b/src/config/mod.rs new file mode 100644 index 0000000..a105933 --- /dev/null +++ b/src/config/mod.rs @@ -0,0 +1 @@ +pub mod config; \ No newline at end of file diff --git a/src/dto/request.rs b/src/dto/request.rs index fcfb822..15dc0f4 100644 --- a/src/dto/request.rs +++ b/src/dto/request.rs @@ -1,5 +1,5 @@ use serde::{Serialize, Deserialize}; -use crate::model::Repository; +use crate::model::config::Repository; #[derive(Serialize, Deserialize)] pub struct AddRepositoryDto { diff --git a/src/git/git.rs b/src/gtm/git.rs similarity index 92% rename from src/git/git.rs rename to src/gtm/git.rs index ded3dcd..90e71e9 100644 --- a/src/git/git.rs +++ b/src/gtm/git.rs @@ -5,15 +5,15 @@ use std::path::Path; use git2::{Error, FetchOptions, Note, RemoteCallbacks, Repository}; use git2::build::RepoBuilder; -use crate::model; +use crate::model::config; -pub(crate) mod gtm; +use crate::gtm::gtm; static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; static DEFAULT_ORIGIN: &str = "origin"; -pub fn clone_or_open(repo_config: &model::Repository) -> Result { +pub fn clone_or_open(repo_config: &config::Repository) -> Result { let path = Path::new(&repo_config.path); if path.exists() { @@ -33,7 +33,7 @@ pub fn clone_or_open(repo_config: &model::Repository) -> Result FetchOptions { +fn generate_fetch_options(repo_config: &config::Repository) -> FetchOptions { let mut cb = RemoteCallbacks::new(); let repo_config = repo_config.clone(); cb.credentials(move |_c, _o, t| { @@ -55,7 +55,7 @@ fn generate_fetch_options(repo_config: &model::Repository) -> FetchOptions { } -pub fn fetch(repo: &Repository, repo_config: &model::Repository) { +pub fn fetch(repo: &Repository, repo_config: &config::Repository) { let mut remote = repo.find_remote(DEFAULT_ORIGIN) .expect("Unable to find remote 'origin'"); let mut ref_added = false; diff --git a/src/git/gtm.rs b/src/gtm/gtm.rs similarity index 100% rename from src/git/gtm.rs rename to src/gtm/gtm.rs diff --git a/src/gtm/mod.rs b/src/gtm/mod.rs new file mode 100644 index 0000000..888af2c --- /dev/null +++ b/src/gtm/mod.rs @@ -0,0 +1,2 @@ +pub mod git; +pub mod gtm; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index a768a4a..55b062f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,12 +2,12 @@ #[macro_use] extern crate rocket; -#[path = "git/git.rs"] mod git; -#[path = "config/config.rs"] mod config; -#[path = "server/server.rs"] mod server; -#[path = "model/model.rs"] mod model; +mod server; mod dto; +mod gtm; +mod config; +mod model; fn main() { - server::run(); + server::server::run(); } \ No newline at end of file diff --git a/src/model/model.rs b/src/model/config.rs similarity index 100% rename from src/model/model.rs rename to src/model/config.rs diff --git a/src/model/mod.rs b/src/model/mod.rs new file mode 100644 index 0000000..a105933 --- /dev/null +++ b/src/model/mod.rs @@ -0,0 +1 @@ +pub mod config; \ No newline at end of file diff --git a/src/server/mod.rs b/src/server/mod.rs new file mode 100644 index 0000000..a49c438 --- /dev/null +++ b/src/server/mod.rs @@ -0,0 +1,4 @@ +pub mod server; + +mod controller; +mod service; \ No newline at end of file diff --git a/src/server/server.rs b/src/server/server.rs index 7be4fda..cd690bf 100644 --- a/src/server/server.rs +++ b/src/server/server.rs @@ -1,6 +1,6 @@ -mod controller; -mod service; +use crate::server::controller; +use crate::server::service; pub fn run() { rocket::ignite() diff --git a/src/server/service.rs b/src/server/service.rs index 673f23d..8c7447b 100644 --- a/src/server/service.rs +++ b/src/server/service.rs @@ -1,11 +1,11 @@ use lazy_static::lazy_static; use regex::Regex; -use crate::config; +use crate::config::config; use crate::dto::request::AddRepositoryDto; use crate::dto::response::BoolResponseDto; -use crate::git; -use crate::git::gtm; +use crate::gtm::git; +use crate::gtm::gtm; lazy_static! { static ref PATH_FROM_URL_REGEX: Regex = From 42a0b6000343e47edc3c6eeaf07fa90820fc96e6 Mon Sep 17 00:00:00 2001 From: tavo Date: Sat, 28 Nov 2020 21:38:27 +0200 Subject: [PATCH 12/33] Minor json fixes --- src/dto/response.rs | 10 ++++++++-- src/gtm/git.rs | 4 +--- src/gtm/gtm.rs | 10 +++++----- src/server/service.rs | 10 ++++++---- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/dto/response.rs b/src/dto/response.rs index 81fb5a3..616be63 100644 --- a/src/dto/response.rs +++ b/src/dto/response.rs @@ -1,7 +1,13 @@ -use serde::{Serialize, Deserialize}; +use serde::{Serialize}; +use crate::gtm::gtm::Commit; -#[derive(Serialize, Deserialize)] +#[derive(Serialize)] pub struct BoolResponseDto { pub success: bool, pub message: Option, +} + +#[derive(Serialize)] +pub struct Repo { + pub commits: Vec, } \ No newline at end of file diff --git a/src/gtm/git.rs b/src/gtm/git.rs index 90e71e9..8411c22 100644 --- a/src/gtm/git.rs +++ b/src/gtm/git.rs @@ -94,10 +94,8 @@ pub fn read_commits(repo: &Repository) -> Result, Error> { .collect(); let res= gtm::parse_commit(&repo, &commit, ¬es)?; - println!("{}", &res); + // println!("{}", &res); commits.push(res); } - // let a = repo.notes(Option::from(GTM_NOTES_REF)) - // .expect("Unable to find gtm-notes"); return Result::Ok(commits); } diff --git a/src/gtm/gtm.rs b/src/gtm/gtm.rs index c1888ed..adebfb4 100644 --- a/src/gtm/gtm.rs +++ b/src/gtm/gtm.rs @@ -15,8 +15,8 @@ lazy_static! { #[derive(Serialize)] pub struct Commit { hash: String, - // branch: String, - author_email: String, + branch: String, + author: String, message: String, time: i64, files: Vec, @@ -35,8 +35,8 @@ pub struct File { pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note]) -> Result { let mut commit = Commit { hash: git_commit.id().to_string(), - // branch: "".to_string(), - author_email: git_commit.author().to_string(), + branch: "todo".to_string(), + author: git_commit.author().to_string(), message: git_commit.message().unwrap().to_string(), time: git_commit.time().seconds(), // todo: validate files: vec![], @@ -145,7 +145,7 @@ fn diff_parents(files: &mut Vec, commit: &git2::Commit, repo: &git2::Repos impl fmt::Display for Commit { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { let _ = writeln!(f, "Commit: {}", self.hash); - let _ = writeln!(f, "Author: {}", self.author_email); + let _ = writeln!(f, "Author: {}", self.author); let _ = writeln!(f, "Time {}", self.time); let _ = writeln!(f, "{}", self.message); diff --git a/src/server/service.rs b/src/server/service.rs index 8c7447b..382d732 100644 --- a/src/server/service.rs +++ b/src/server/service.rs @@ -3,7 +3,7 @@ use regex::Regex; use crate::config::config; use crate::dto::request::AddRepositoryDto; -use crate::dto::response::BoolResponseDto; +use crate::dto::response::{BoolResponseDto, Repo}; use crate::gtm::git; use crate::gtm::gtm; @@ -12,15 +12,17 @@ lazy_static! { Regex::new(r#"(git@|https://)([a-zA-Z0-9.]+)[:/]([a-zA-Z0-9-]+)/([a-zA-Z0-9-]+)\.git"#).unwrap(); } -// TODO: (Tavo) Repo wrapper -pub fn get_repo() -> Vec { +pub fn get_repo() -> Repo { let loc = "./example_config.toml".to_string(); let loaded_cfg = config::load(&loc); let repo_to_clone = loaded_cfg.repositories.get(0).unwrap(); let repo = git::clone_or_open(&repo_to_clone).unwrap(); let _res = git::fetch(&repo, &repo_to_clone); let commits = git::read_commits(&repo).unwrap(); - return commits; + let gtm_repo : Repo = Repo { + commits + }; + return gtm_repo; } pub fn add_repo(repo_dto: AddRepositoryDto) -> BoolResponseDto { From 8b2510c1cad941bb0bfed027339d5f3fc8ed88cc Mon Sep 17 00:00:00 2001 From: tavo Date: Sat, 28 Nov 2020 21:52:44 +0200 Subject: [PATCH 13/33] Fix bad path var --- src/server/controller.rs | 2 +- src/server/server.rs | 1 - src/server/service.rs | 28 ++++++++++++++++++++++++---- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/server/controller.rs b/src/server/controller.rs index f0c5b4c..ab35481 100644 --- a/src/server/controller.rs +++ b/src/server/controller.rs @@ -10,7 +10,7 @@ pub fn index() -> &'static str { #[get("/repository///")] pub fn repo(provider: String, user: String, repo: String) -> JsonValue { - let repo = service::get_repo(); // TODO: How to match credentials? + let repo = service::get_repo(&provider, &user, &repo); // TODO: How to match credentials? rocket_contrib::json!(&repo) } diff --git a/src/server/server.rs b/src/server/server.rs index cd690bf..88717c6 100644 --- a/src/server/server.rs +++ b/src/server/server.rs @@ -1,6 +1,5 @@ use crate::server::controller; -use crate::server::service; pub fn run() { rocket::ignite() diff --git a/src/server/service.rs b/src/server/service.rs index 382d732..76c8c61 100644 --- a/src/server/service.rs +++ b/src/server/service.rs @@ -5,21 +5,30 @@ use crate::config::config; use crate::dto::request::AddRepositoryDto; use crate::dto::response::{BoolResponseDto, Repo}; use crate::gtm::git; -use crate::gtm::gtm; lazy_static! { static ref PATH_FROM_URL_REGEX: Regex = Regex::new(r#"(git@|https://)([a-zA-Z0-9.]+)[:/]([a-zA-Z0-9-]+)/([a-zA-Z0-9-]+)\.git"#).unwrap(); } -pub fn get_repo() -> Repo { +pub fn get_repo(provider: &String, user: &String, repo: &String) -> Repo { let loc = "./example_config.toml".to_string(); let loaded_cfg = config::load(&loc); - let repo_to_clone = loaded_cfg.repositories.get(0).unwrap(); + let repo_to_clone = loaded_cfg.repositories.iter() + .find(|r| r.path == generate_path_from_provider_user_repo(&provider, &user, &repo, &loaded_cfg.repositories_base_path)); + + if repo_to_clone.is_none() { + // TODO: Some error thingy + return Repo { + commits: vec![] + } + } + let repo_to_clone = repo_to_clone.unwrap(); + let repo = git::clone_or_open(&repo_to_clone).unwrap(); let _res = git::fetch(&repo, &repo_to_clone); let commits = git::read_commits(&repo).unwrap(); - let gtm_repo : Repo = Repo { + let gtm_repo: Repo = Repo { commits }; return gtm_repo; @@ -55,4 +64,15 @@ pub fn generate_path_from_git_url(url: &String, base_path: &String) -> String { caps.get(4).map_or("repo", |m| m.as_str()) ); return path; +} + +pub fn generate_path_from_provider_user_repo( + provider: &String, + user: &String, + repo: &String, + base_path: &String, +) -> String { + return format!("{}/{}/{}/{}", base_path.trim_end_matches("/"), + provider, user, repo + ); } \ No newline at end of file From 4f28e3de4bc096197b570cd59ad0f1566e3725f9 Mon Sep 17 00:00:00 2001 From: tavo Date: Sun, 29 Nov 2020 17:22:00 +0200 Subject: [PATCH 14/33] Return repo details on create --- src/dto/response.rs | 7 +++-- src/server/controller.rs | 5 ++++ src/server/server.rs | 3 +- src/server/service.rs | 60 ++++++++++++++++++++++------------------ 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/src/dto/response.rs b/src/dto/response.rs index 616be63..cf85ed6 100644 --- a/src/dto/response.rs +++ b/src/dto/response.rs @@ -2,12 +2,15 @@ use serde::{Serialize}; use crate::gtm::gtm::Commit; #[derive(Serialize)] -pub struct BoolResponseDto { +pub struct AddRepoDto { pub success: bool, + pub provider: Option, + pub user: Option, + pub repo: Option, pub message: Option, } #[derive(Serialize)] -pub struct Repo { +pub struct RepoDto { pub commits: Vec, } \ No newline at end of file diff --git a/src/server/controller.rs b/src/server/controller.rs index ab35481..c786b45 100644 --- a/src/server/controller.rs +++ b/src/server/controller.rs @@ -18,4 +18,9 @@ pub fn repo(provider: String, user: String, repo: String) -> JsonValue { pub fn add_repo(repo: Json) -> JsonValue { let response = service::add_repo(repo.into_inner()); rocket_contrib::json!(&response) +} + +#[get("/sync")] +pub fn sync() -> JsonValue { + rocket_contrib::json!("{}") } \ No newline at end of file diff --git a/src/server/server.rs b/src/server/server.rs index 88717c6..e7d89f0 100644 --- a/src/server/server.rs +++ b/src/server/server.rs @@ -7,6 +7,7 @@ pub fn run() { routes![ controller::index, controller::repo, - controller::add_repo], + controller::add_repo, + controller::sync], ).launch(); } \ No newline at end of file diff --git a/src/server/service.rs b/src/server/service.rs index 76c8c61..2a43710 100644 --- a/src/server/service.rs +++ b/src/server/service.rs @@ -3,23 +3,24 @@ use regex::Regex; use crate::config::config; use crate::dto::request::AddRepositoryDto; -use crate::dto::response::{BoolResponseDto, Repo}; +use crate::dto::response::{AddRepoDto, RepoDto}; use crate::gtm::git; lazy_static! { static ref PATH_FROM_URL_REGEX: Regex = Regex::new(r#"(git@|https://)([a-zA-Z0-9.]+)[:/]([a-zA-Z0-9-]+)/([a-zA-Z0-9-]+)\.git"#).unwrap(); + + static ref CONFIG_PATH: String = "./example_config.toml".to_string(); } -pub fn get_repo(provider: &String, user: &String, repo: &String) -> Repo { - let loc = "./example_config.toml".to_string(); - let loaded_cfg = config::load(&loc); - let repo_to_clone = loaded_cfg.repositories.iter() - .find(|r| r.path == generate_path_from_provider_user_repo(&provider, &user, &repo, &loaded_cfg.repositories_base_path)); +pub fn get_repo(provider: &String, user: &String, repo: &String) -> RepoDto { + let cfg = config::load(&CONFIG_PATH); + let repo_to_clone = cfg.repositories.iter() + .find(|r| r.path == generate_path_from_provider_user_repo(&provider, &user, &repo, &cfg.repositories_base_path)); if repo_to_clone.is_none() { // TODO: Some error thingy - return Repo { + return RepoDto { commits: vec![] } } @@ -28,42 +29,49 @@ pub fn get_repo(provider: &String, user: &String, repo: &String) -> Repo { let repo = git::clone_or_open(&repo_to_clone).unwrap(); let _res = git::fetch(&repo, &repo_to_clone); let commits = git::read_commits(&repo).unwrap(); - let gtm_repo: Repo = Repo { + let gtm_repo: RepoDto = RepoDto { commits }; return gtm_repo; } -pub fn add_repo(repo_dto: AddRepositoryDto) -> BoolResponseDto { - let loc = "./example_config.toml".to_string(); - let mut loaded_cfg = config::load(&loc); - let repo = repo_dto.to_repository(&|url: &String| { generate_path_from_git_url(url, &loaded_cfg.repositories_base_path) }); +pub fn add_repo(repo_dto: AddRepositoryDto) -> AddRepoDto { + let mut cfg = config::load(&CONFIG_PATH); + let repo = repo_dto.to_repository(&|url: &String| { generate_path_from_git_url(url, &cfg.repositories_base_path) }); let cloned_repo = git::clone_or_open(&repo); if cloned_repo.is_ok() { - if !loaded_cfg.repositories.iter().any(|r| r.url == repo_dto.url) { - loaded_cfg.repositories.push(repo); - config::save(&loc, &loaded_cfg); + let (provider, user, repository) = get_credentials_from_clone_url(&repo.url); + if !cfg.repositories.iter().any(|r| r.url == repo_dto.url) { + cfg.repositories.push(repo); + config::save(&CONFIG_PATH, &cfg); } - return BoolResponseDto { + return AddRepoDto { success: true, + provider: Option::from(provider), + user: Option::from(user), + repo: Option::from(repository), message: None, }; } - return BoolResponseDto { + return AddRepoDto { success: false, + provider: None, + user: None, + repo: None, message: cloned_repo.err().map(|e| e.to_string()), }; } pub fn generate_path_from_git_url(url: &String, base_path: &String) -> String { + let (provider, user, repo) = get_credentials_from_clone_url(url); + return format!("{}/{}/{}/{}", base_path.trim_end_matches("/"), provider, user, repo); +} + +pub fn get_credentials_from_clone_url(url: &String) -> (String, String, String) { let caps = PATH_FROM_URL_REGEX.captures(url).unwrap(); - let path = format!("{}/{}/{}/{}", - base_path.trim_end_matches("/"), - caps.get(2).map_or("provider", |m| m.as_str()), - caps.get(3).map_or("user", |m| m.as_str()), - caps.get(4).map_or("repo", |m| m.as_str()) - ); - return path; + return (caps.get(2).map_or("provider".to_string(), |m| m.as_str().to_string()), + caps.get(3).map_or("user".to_string(), |m| m.as_str().to_string()), + caps.get(4).map_or("repo".to_string(), |m| m.as_str().to_string())) } pub fn generate_path_from_provider_user_repo( @@ -72,7 +80,5 @@ pub fn generate_path_from_provider_user_repo( repo: &String, base_path: &String, ) -> String { - return format!("{}/{}/{}/{}", base_path.trim_end_matches("/"), - provider, user, repo - ); + return format!("{}/{}/{}/{}", base_path.trim_end_matches("/"), provider, user, repo); } \ No newline at end of file From fa9e368e8fd8a8e64d271b10076735bd33e836bd Mon Sep 17 00:00:00 2001 From: tavo Date: Mon, 30 Nov 2020 21:02:03 +0200 Subject: [PATCH 15/33] Iterate over commits on all branches --- src/gtm/git.rs | 36 ++++++++++++++++++++++++++++++------ src/gtm/gtm.rs | 4 ++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/gtm/git.rs b/src/gtm/git.rs index 8411c22..2ad16dc 100644 --- a/src/gtm/git.rs +++ b/src/gtm/git.rs @@ -3,11 +3,11 @@ use std::fs; use std::path::Path; -use git2::{Error, FetchOptions, Note, RemoteCallbacks, Repository}; +use git2::{BranchType, Error, FetchOptions, Note, RemoteCallbacks, Repository}; use git2::build::RepoBuilder; -use crate::model::config; use crate::gtm::gtm; +use crate::model::config; static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; @@ -48,7 +48,6 @@ fn generate_fetch_options(repo_config: &config::Repository) -> FetchOptions { return git2::Cred::default(); }); - let mut fo = FetchOptions::new(); fo.remote_callbacks(cb); return fo; @@ -73,17 +72,42 @@ pub fn fetch(repo: &Repository, repo_config: &config::Repository) { .expect("Unable to find remote 'origin'"); } + let branches = repo.branches(Option::from(BranchType::Remote)).unwrap(); + let mut fetch_refs: Vec = vec![]; + for branch in branches { + let (branch, _) = branch.unwrap(); + let refspec = branch.get() + .name() + .unwrap() + .strip_prefix("refs/remotes/origin/") + .unwrap(); + if refspec != "HEAD" { + fetch_refs.push(format!("refs/heads/{}", refspec.to_string())); + } + } + fetch_refs.push(GTM_NOTES_REF.parse().unwrap()); + let mut fo = generate_fetch_options(repo_config); - remote.fetch(&[] as &[&str], Option::from(&mut fo), None) + remote.fetch(&fetch_refs, Option::from(&mut fo), None) .expect("Error fetching data!"); remote.disconnect().unwrap(); } pub fn read_commits(repo: &Repository) -> Result, Error> { let mut commits: Vec = Vec::new(); + //let mut hashes: HashSet = HashSet::new(); let mut revwalk = repo.revwalk().expect("Unable to revwalk!"); let _sorting = revwalk.set_sorting(git2::Sort::TIME); - let _head = revwalk.push_head(); + let branches = repo.branches(Option::from(BranchType::Remote)).unwrap(); + for branch in branches { + let (branch, _) = branch.unwrap(); + let refspec = branch.get().name().unwrap(); + if refspec == "refs/remotes/origin/HEAD" { + continue + } + let _ = revwalk.push_ref(refspec); + } + // let _head = revwalk.push_glob("refs/heads/*"); for commit_oid in revwalk { let commit_oid = commit_oid?; let commit = repo.find_commit(commit_oid)?; @@ -93,7 +117,7 @@ pub fn read_commits(repo: &Repository) -> Result, Error> { .map(|n| repo.find_note(Option::from(GTM_NOTES_REF), n.1).unwrap()) .collect(); - let res= gtm::parse_commit(&repo, &commit, ¬es)?; + let res = gtm::parse_commit(&repo, &commit, ¬es, "".to_string())?; // println!("{}", &res); commits.push(res); } diff --git a/src/gtm/gtm.rs b/src/gtm/gtm.rs index adebfb4..706237a 100644 --- a/src/gtm/gtm.rs +++ b/src/gtm/gtm.rs @@ -32,10 +32,10 @@ pub struct File { deleted_lines: i32, } -pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note]) -> Result { +pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note], git_branch: String) -> Result { let mut commit = Commit { hash: git_commit.id().to_string(), - branch: "todo".to_string(), + branch: git_branch, author: git_commit.author().to_string(), message: git_commit.message().unwrap().to_string(), time: git_commit.time().seconds(), // todo: validate From 6d87bad90c09f9d9614d867f8e961116e2467812 Mon Sep 17 00:00:00 2001 From: tavo Date: Mon, 30 Nov 2020 21:38:07 +0200 Subject: [PATCH 16/33] Get branch names --- src/gtm/git.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/gtm/git.rs b/src/gtm/git.rs index 2ad16dc..07bce82 100644 --- a/src/gtm/git.rs +++ b/src/gtm/git.rs @@ -1,9 +1,10 @@ #![deny(warnings)] +use std::collections::HashMap; use std::fs; use std::path::Path; -use git2::{BranchType, Error, FetchOptions, Note, RemoteCallbacks, Repository}; +use git2::{BranchType, Error, FetchOptions, Note, Oid, RemoteCallbacks, Repository}; use git2::build::RepoBuilder; use crate::gtm::gtm; @@ -12,6 +13,8 @@ use crate::model::config; static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; static DEFAULT_ORIGIN: &str = "origin"; +static ORIGIN_PREFIX: &str = "refs/remotes/origin/"; +static ORIGIN_HEAD: &str = "refs/remotes/origin/HEAD"; pub fn clone_or_open(repo_config: &config::Repository) -> Result { let path = Path::new(&repo_config.path); @@ -79,7 +82,7 @@ pub fn fetch(repo: &Repository, repo_config: &config::Repository) { let refspec = branch.get() .name() .unwrap() - .strip_prefix("refs/remotes/origin/") + .strip_prefix(ORIGIN_PREFIX) .unwrap(); if refspec != "HEAD" { fetch_refs.push(format!("refs/heads/{}", refspec.to_string())); @@ -95,30 +98,40 @@ pub fn fetch(repo: &Repository, repo_config: &config::Repository) { pub fn read_commits(repo: &Repository) -> Result, Error> { let mut commits: Vec = Vec::new(); - //let mut hashes: HashSet = HashSet::new(); + let mut branch_map: HashMap = HashMap::new(); let mut revwalk = repo.revwalk().expect("Unable to revwalk!"); - let _sorting = revwalk.set_sorting(git2::Sort::TIME); + revwalk.set_sorting(git2::Sort::TOPOLOGICAL).expect("Unable to set revwalk sorting!"); + revwalk.set_sorting(git2::Sort::REVERSE).expect("Unable to reverse revalk sorting!"); let branches = repo.branches(Option::from(BranchType::Remote)).unwrap(); for branch in branches { let (branch, _) = branch.unwrap(); let refspec = branch.get().name().unwrap(); - if refspec == "refs/remotes/origin/HEAD" { + if refspec == ORIGIN_HEAD { continue } + branch_map.insert( + branch.get().target().unwrap(), + refspec.strip_prefix(ORIGIN_PREFIX).unwrap().to_string(), + ); let _ = revwalk.push_ref(refspec); } - // let _head = revwalk.push_glob("refs/heads/*"); + for commit_oid in revwalk { let commit_oid = commit_oid?; let commit = repo.find_commit(commit_oid)?; + let mut branch = "".to_string(); // TODO: Some more intelligent choice than last wins + for (oid, name) in &branch_map { + if repo.merge_base(commit_oid, *oid).unwrap() == commit_oid && branch != "master" { + branch = name.clone(); + } + } let notes: Vec = repo.notes(Option::from(GTM_NOTES_REF))? .map(|n| n.unwrap()) .filter(|n| n.1 == commit_oid) .map(|n| repo.find_note(Option::from(GTM_NOTES_REF), n.1).unwrap()) .collect(); - let res = gtm::parse_commit(&repo, &commit, ¬es, "".to_string())?; - // println!("{}", &res); + let res = gtm::parse_commit(&repo, &commit, ¬es, branch)?; commits.push(res); } return Result::Ok(commits); From 704dbac61064cda5c70e2b424f82b5084d40973b Mon Sep 17 00:00:00 2001 From: tavo Date: Mon, 30 Nov 2020 21:48:50 +0200 Subject: [PATCH 17/33] Make default branch dynamic --- src/gtm/git.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gtm/git.rs b/src/gtm/git.rs index 07bce82..b9e5a5d 100644 --- a/src/gtm/git.rs +++ b/src/gtm/git.rs @@ -56,7 +56,6 @@ fn generate_fetch_options(repo_config: &config::Repository) -> FetchOptions { return fo; } - pub fn fetch(repo: &Repository, repo_config: &config::Repository) { let mut remote = repo.find_remote(DEFAULT_ORIGIN) .expect("Unable to find remote 'origin'"); @@ -103,6 +102,7 @@ pub fn read_commits(repo: &Repository) -> Result, Error> { revwalk.set_sorting(git2::Sort::TOPOLOGICAL).expect("Unable to set revwalk sorting!"); revwalk.set_sorting(git2::Sort::REVERSE).expect("Unable to reverse revalk sorting!"); let branches = repo.branches(Option::from(BranchType::Remote)).unwrap(); + let default_branch = repo.head().unwrap().name().unwrap().to_string(); for branch in branches { let (branch, _) = branch.unwrap(); let refspec = branch.get().name().unwrap(); @@ -121,7 +121,7 @@ pub fn read_commits(repo: &Repository) -> Result, Error> { let commit = repo.find_commit(commit_oid)?; let mut branch = "".to_string(); // TODO: Some more intelligent choice than last wins for (oid, name) in &branch_map { - if repo.merge_base(commit_oid, *oid).unwrap() == commit_oid && branch != "master" { + if repo.merge_base(commit_oid, *oid).unwrap() == commit_oid && branch != default_branch { branch = name.clone(); } } From 0597ddf5388305ee3cda2226666b66840b7e272f Mon Sep 17 00:00:00 2001 From: tavo Date: Tue, 22 Dec 2020 14:20:15 +0200 Subject: [PATCH 18/33] Somewhat working sync --- Rocket.toml | 3 +++ src/dto/response.rs | 11 +++++++++++ src/gtm/gtm.rs | 22 +++++++++++++++------- src/model/config.rs | 2 ++ src/server/service.rs | 25 ++++++++++++++++--------- 5 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 Rocket.toml diff --git a/Rocket.toml b/Rocket.toml new file mode 100644 index 0000000..547c6c4 --- /dev/null +++ b/Rocket.toml @@ -0,0 +1,3 @@ +[development] +address = "localhost" +port = 8090 \ No newline at end of file diff --git a/src/dto/response.rs b/src/dto/response.rs index cf85ed6..8a5fbdd 100644 --- a/src/dto/response.rs +++ b/src/dto/response.rs @@ -12,5 +12,16 @@ pub struct AddRepoDto { #[derive(Serialize)] pub struct RepoDto { + pub provider: String, + pub user: String, + pub repo: String, + pub sync_url: String, + pub access_token: Option, pub commits: Vec, +} + +#[derive(Serialize)] +pub struct RepoWrapperDto { + pub repository: Option, + // TODO: Errors } \ No newline at end of file diff --git a/src/gtm/gtm.rs b/src/gtm/gtm.rs index 706237a..7f340bf 100644 --- a/src/gtm/gtm.rs +++ b/src/gtm/gtm.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::fmt; use git2::{DiffOptions, Note}; @@ -26,12 +25,18 @@ pub struct Commit { pub struct File { path: String, time_total: i64, - timeline: HashMap, + timeline: Vec, status: String, added_lines: i32, deleted_lines: i32, } +#[derive(Serialize, Copy, Clone)] +pub struct TimelineEntry { + timestamp: i64, + time: i64, +} + pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note], git_branch: String) -> Result { let mut commit = Commit { hash: git_commit.id().to_string(), @@ -69,7 +74,7 @@ fn parse_note_message(message: &str) -> Option> { let mut file = File { path: "".to_string(), time_total: 0, - timeline: HashMap::new(), + timeline: vec![], status: "".to_string(), added_lines: 0, deleted_lines: 0, @@ -89,8 +94,11 @@ fn parse_note_message(message: &str) -> Option> { file.status = fields.get(0)?.to_string(); } else if fields.len() == 2 { let epoch_timeline: i64 = fields.get(0)?.parse().unwrap_or(0); - let epoch_total: i32 = fields.get(1)?.parse().unwrap_or(0); - file.timeline.insert(epoch_timeline, epoch_total); + let epoch_total: i64 = fields.get(1)?.parse().unwrap_or(0); + file.timeline.push(TimelineEntry { + timestamp: epoch_timeline, + time: epoch_total, + }); } } } else { @@ -101,8 +109,8 @@ fn parse_note_message(message: &str) -> Option> { for mut added_file in files.iter_mut() { if added_file.path == file.path { added_file.time_total += file.time_total; - for (epoch, secs) in &file.timeline { - added_file.timeline.insert(*epoch, *secs); + for timeline_entry in &file.timeline { + added_file.timeline.push(timeline_entry.clone()); } found = true; } diff --git a/src/model/config.rs b/src/model/config.rs index 5932b3f..8e8343a 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -5,6 +5,8 @@ pub struct Config { pub target_host: String, pub target_port: Option, pub port: Option, + pub address: Option, + pub access_token: Option, pub repositories_base_path: String, #[serde(skip_serializing_if = "Vec::is_empty", default)] pub repositories: Vec, diff --git a/src/server/service.rs b/src/server/service.rs index 2a43710..d52ceab 100644 --- a/src/server/service.rs +++ b/src/server/service.rs @@ -3,7 +3,7 @@ use regex::Regex; use crate::config::config; use crate::dto::request::AddRepositoryDto; -use crate::dto::response::{AddRepoDto, RepoDto}; +use crate::dto::response::{AddRepoDto, RepoDto, RepoWrapperDto}; use crate::gtm::git; lazy_static! { @@ -13,26 +13,33 @@ lazy_static! { static ref CONFIG_PATH: String = "./example_config.toml".to_string(); } -pub fn get_repo(provider: &String, user: &String, repo: &String) -> RepoDto { +pub fn get_repo(provider: &String, user: &String, repo: &String) -> RepoWrapperDto { let cfg = config::load(&CONFIG_PATH); let repo_to_clone = cfg.repositories.iter() .find(|r| r.path == generate_path_from_provider_user_repo(&provider, &user, &repo, &cfg.repositories_base_path)); if repo_to_clone.is_none() { // TODO: Some error thingy - return RepoDto { - commits: vec![] - } + return RepoWrapperDto { + repository: None + }; } let repo_to_clone = repo_to_clone.unwrap(); - let repo = git::clone_or_open(&repo_to_clone).unwrap(); - let _res = git::fetch(&repo, &repo_to_clone); - let commits = git::read_commits(&repo).unwrap(); + let git_repo = git::clone_or_open(&repo_to_clone).unwrap(); + let _res = git::fetch(&git_repo, &repo_to_clone); + let commits = git::read_commits(&git_repo).unwrap(); let gtm_repo: RepoDto = RepoDto { + provider: provider.clone(), + user: user.clone(), + repo: repo.clone(), + sync_url: format!("{}:{}", cfg.address.unwrap_or("localhost".to_string()), cfg.port.unwrap_or(8000)), + access_token: cfg.access_token, commits }; - return gtm_repo; + return RepoWrapperDto { + repository: Option::from(gtm_repo) + }; } pub fn add_repo(repo_dto: AddRepositoryDto) -> AddRepoDto { From 15ccd329c365081bcda6a1cbc7143d425d87966d Mon Sep 17 00:00:00 2001 From: tavo Date: Tue, 22 Dec 2020 17:04:35 +0200 Subject: [PATCH 19/33] Refacto for domain based architecture --- Cargo.toml | 2 ++ src/config/config.rs | 49 +++++++++++++++++++++++++++++++++++++++- src/config/mod.rs | 3 ++- src/config/repository.rs | 32 ++++++++++++++++++++++++++ src/dto/request.rs | 2 +- src/gtm/git.rs | 8 +++---- src/main.rs | 1 - src/model/config.rs | 23 ------------------- src/model/mod.rs | 1 - src/server/controller.rs | 3 +++ src/server/mod.rs | 3 ++- src/server/service.rs | 45 +++++++----------------------------- src/server/sync.rs | 39 ++++++++++++++++++++++++++++++++ 13 files changed, 141 insertions(+), 70 deletions(-) create mode 100644 src/config/repository.rs delete mode 100644 src/model/config.rs delete mode 100644 src/model/mod.rs create mode 100644 src/server/sync.rs diff --git a/Cargo.toml b/Cargo.toml index bc47164..5aa75fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,8 @@ serde_json = "1.0.44" toml = "0.5.7" regex = "1" lazy_static = "1.4.0" +reqwest = { version = "0.10.9", features = ["json"] } +tokio = { version = "0.2", features = ["full"] } [dependencies.rocket_contrib] version = "0.4.2" diff --git a/src/config/config.rs b/src/config/config.rs index 7489574..5551a5e 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -1,5 +1,26 @@ use std::fs; -use crate::model::config::Config; + +use lazy_static::lazy_static; +use serde::{Deserialize, Serialize}; + +use crate::config::repository::{generate_credentials_from_clone_url, Repository}; + +lazy_static! { + pub static ref CONFIG_PATH: String = "./example_config.toml".to_string(); +} + + +#[derive(Serialize, Deserialize)] +pub struct Config { + pub target_host: String, + pub target_port: Option, + pub port: Option, + pub address: Option, + pub access_token: Option, + pub repositories_base_path: String, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub repositories: Vec, +} pub fn load(config_file: &String) -> Config { let content = fs::read_to_string(config_file).expect("Unable to read config!"); @@ -9,4 +30,30 @@ pub fn load(config_file: &String) -> Config { pub fn save(config_file: &String, config: &Config) { let content = toml::to_string(config).expect("Unable to serialize config!"); fs::write(config_file, content).expect("Unable to save config!"); +} + +impl Config { + pub fn get_target_url(&self) -> String { + return format!("{}:{}", self.target_host, self.target_port.unwrap_or(8000)); + } + + pub fn get_sync_url(&self) -> String { + return format!("{}:{}", + self.address.clone().unwrap_or("localhost".to_string()), + self.port.clone().unwrap_or(8000) + ); + } + + pub fn generate_path_from_git_url(&self, url: &String) -> String { + let (provider, user, repo) = generate_credentials_from_clone_url(url); + return format!("{}/{}/{}/{}", self.repositories_base_path.trim_end_matches("/"), provider, user, repo); + } + + pub fn generate_path_from_provider_user_repo(&self, + provider: &String, + user: &String, + repo: &String, + ) -> String { + return format!("{}/{}/{}/{}", self.repositories_base_path.trim_end_matches("/"), provider, user, repo); + } } \ No newline at end of file diff --git a/src/config/mod.rs b/src/config/mod.rs index a105933..f084d00 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1 +1,2 @@ -pub mod config; \ No newline at end of file +pub mod config; +pub mod repository; \ No newline at end of file diff --git a/src/config/repository.rs b/src/config/repository.rs new file mode 100644 index 0000000..192fb4f --- /dev/null +++ b/src/config/repository.rs @@ -0,0 +1,32 @@ +use lazy_static::lazy_static; +use regex::Regex; +use serde::{Deserialize, Serialize}; + +lazy_static! { + static ref PATH_FROM_URL_REGEX: Regex = + Regex::new(r#"(git@|https://)([a-zA-Z0-9.]+)[:/]([a-zA-Z0-9-]+)/([a-zA-Z0-9-]+)\.git"#).unwrap(); +} + +#[derive(Serialize, Deserialize)] +pub struct Repository { + pub url: String, + pub path: String, + pub ssh_private_key: Option, + pub ssh_public_key: Option, + pub ssh_user: Option, + pub ssh_passphrase: Option, +} + +// pub fn generate_path_from_git_url(url: &String, base_path: &String) -> String { +// let (provider, user, repo) = generate_credentials_from_clone_url(url); +// return format!("{}/{}/{}/{}", base_path.trim_end_matches("/"), provider, user, repo); +// } + +pub fn generate_credentials_from_clone_url(url: &String) -> (String, String, String) { + let caps = PATH_FROM_URL_REGEX.captures(url).unwrap(); + return (caps.get(2).map_or("provider".to_string(), |m| m.as_str().to_string()), + caps.get(3).map_or("user".to_string(), |m| m.as_str().to_string()), + caps.get(4).map_or("repo".to_string(), |m| m.as_str().to_string())); +} + +impl Repository {} \ No newline at end of file diff --git a/src/dto/request.rs b/src/dto/request.rs index 15dc0f4..1eb42df 100644 --- a/src/dto/request.rs +++ b/src/dto/request.rs @@ -1,5 +1,5 @@ use serde::{Serialize, Deserialize}; -use crate::model::config::Repository; +use crate::config::repository::Repository; #[derive(Serialize, Deserialize)] pub struct AddRepositoryDto { diff --git a/src/gtm/git.rs b/src/gtm/git.rs index b9e5a5d..db51428 100644 --- a/src/gtm/git.rs +++ b/src/gtm/git.rs @@ -8,7 +8,7 @@ use git2::{BranchType, Error, FetchOptions, Note, Oid, RemoteCallbacks, Reposito use git2::build::RepoBuilder; use crate::gtm::gtm; -use crate::model::config; +use crate::config::repository; static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; @@ -16,7 +16,7 @@ static DEFAULT_ORIGIN: &str = "origin"; static ORIGIN_PREFIX: &str = "refs/remotes/origin/"; static ORIGIN_HEAD: &str = "refs/remotes/origin/HEAD"; -pub fn clone_or_open(repo_config: &config::Repository) -> Result { +pub fn clone_or_open(repo_config: &repository::Repository) -> Result { let path = Path::new(&repo_config.path); if path.exists() { @@ -36,7 +36,7 @@ pub fn clone_or_open(repo_config: &config::Repository) -> Result FetchOptions { +fn generate_fetch_options(repo_config: &repository::Repository) -> FetchOptions { let mut cb = RemoteCallbacks::new(); let repo_config = repo_config.clone(); cb.credentials(move |_c, _o, t| { @@ -56,7 +56,7 @@ fn generate_fetch_options(repo_config: &config::Repository) -> FetchOptions { return fo; } -pub fn fetch(repo: &Repository, repo_config: &config::Repository) { +pub fn fetch(repo: &Repository, repo_config: &repository::Repository) { let mut remote = repo.find_remote(DEFAULT_ORIGIN) .expect("Unable to find remote 'origin'"); let mut ref_added = false; diff --git a/src/main.rs b/src/main.rs index 55b062f..fba803b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,6 @@ mod server; mod dto; mod gtm; mod config; -mod model; fn main() { server::server::run(); diff --git a/src/model/config.rs b/src/model/config.rs deleted file mode 100644 index 8e8343a..0000000 --- a/src/model/config.rs +++ /dev/null @@ -1,23 +0,0 @@ -use serde::{Serialize, Deserialize}; - -#[derive(Serialize, Deserialize)] -pub struct Config { - pub target_host: String, - pub target_port: Option, - pub port: Option, - pub address: Option, - pub access_token: Option, - pub repositories_base_path: String, - #[serde(skip_serializing_if = "Vec::is_empty", default)] - pub repositories: Vec, -} - -#[derive(Serialize, Deserialize)] -pub struct Repository { - pub url: String, - pub path: String, - pub ssh_private_key: Option, - pub ssh_public_key: Option, - pub ssh_user: Option, - pub ssh_passphrase: Option, -} diff --git a/src/model/mod.rs b/src/model/mod.rs deleted file mode 100644 index a105933..0000000 --- a/src/model/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod config; \ No newline at end of file diff --git a/src/server/controller.rs b/src/server/controller.rs index c786b45..fcf0aa2 100644 --- a/src/server/controller.rs +++ b/src/server/controller.rs @@ -2,6 +2,7 @@ use rocket_contrib::json::{JsonValue, Json}; use crate::server::service; use crate::dto::request::AddRepositoryDto; +use crate::server::sync::sync_all; #[get("/")] pub fn index() -> &'static str { @@ -22,5 +23,7 @@ pub fn add_repo(repo: Json) -> JsonValue { #[get("/sync")] pub fn sync() -> JsonValue { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(sync_all()); rocket_contrib::json!("{}") } \ No newline at end of file diff --git a/src/server/mod.rs b/src/server/mod.rs index a49c438..a6d56db 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,4 +1,5 @@ pub mod server; mod controller; -mod service; \ No newline at end of file +mod service; +mod sync; // TODO: Move to some other module \ No newline at end of file diff --git a/src/server/service.rs b/src/server/service.rs index d52ceab..c559b16 100644 --- a/src/server/service.rs +++ b/src/server/service.rs @@ -1,22 +1,14 @@ -use lazy_static::lazy_static; -use regex::Regex; use crate::config::config; use crate::dto::request::AddRepositoryDto; use crate::dto::response::{AddRepoDto, RepoDto, RepoWrapperDto}; use crate::gtm::git; - -lazy_static! { - static ref PATH_FROM_URL_REGEX: Regex = - Regex::new(r#"(git@|https://)([a-zA-Z0-9.]+)[:/]([a-zA-Z0-9-]+)/([a-zA-Z0-9-]+)\.git"#).unwrap(); - - static ref CONFIG_PATH: String = "./example_config.toml".to_string(); -} +use crate::config::repository::generate_credentials_from_clone_url; pub fn get_repo(provider: &String, user: &String, repo: &String) -> RepoWrapperDto { - let cfg = config::load(&CONFIG_PATH); + let cfg = config::load(&config::CONFIG_PATH); let repo_to_clone = cfg.repositories.iter() - .find(|r| r.path == generate_path_from_provider_user_repo(&provider, &user, &repo, &cfg.repositories_base_path)); + .find(|r| r.path == cfg.generate_path_from_provider_user_repo(&provider, &user, &repo)); if repo_to_clone.is_none() { // TODO: Some error thingy @@ -33,7 +25,7 @@ pub fn get_repo(provider: &String, user: &String, repo: &String) -> RepoWrapperD provider: provider.clone(), user: user.clone(), repo: repo.clone(), - sync_url: format!("{}:{}", cfg.address.unwrap_or("localhost".to_string()), cfg.port.unwrap_or(8000)), + sync_url: cfg.get_sync_url(), access_token: cfg.access_token, commits }; @@ -43,14 +35,14 @@ pub fn get_repo(provider: &String, user: &String, repo: &String) -> RepoWrapperD } pub fn add_repo(repo_dto: AddRepositoryDto) -> AddRepoDto { - let mut cfg = config::load(&CONFIG_PATH); - let repo = repo_dto.to_repository(&|url: &String| { generate_path_from_git_url(url, &cfg.repositories_base_path) }); + let mut cfg = config::load(&config::CONFIG_PATH); + let repo = repo_dto.to_repository(&|url: &String| { cfg.generate_path_from_git_url(url) }); let cloned_repo = git::clone_or_open(&repo); if cloned_repo.is_ok() { - let (provider, user, repository) = get_credentials_from_clone_url(&repo.url); + let (provider, user, repository) = generate_credentials_from_clone_url(&repo.url); if !cfg.repositories.iter().any(|r| r.url == repo_dto.url) { cfg.repositories.push(repo); - config::save(&CONFIG_PATH, &cfg); + config::save(&config::CONFIG_PATH, &cfg); } return AddRepoDto { success: true, @@ -68,24 +60,3 @@ pub fn add_repo(repo_dto: AddRepositoryDto) -> AddRepoDto { message: cloned_repo.err().map(|e| e.to_string()), }; } - -pub fn generate_path_from_git_url(url: &String, base_path: &String) -> String { - let (provider, user, repo) = get_credentials_from_clone_url(url); - return format!("{}/{}/{}/{}", base_path.trim_end_matches("/"), provider, user, repo); -} - -pub fn get_credentials_from_clone_url(url: &String) -> (String, String, String) { - let caps = PATH_FROM_URL_REGEX.captures(url).unwrap(); - return (caps.get(2).map_or("provider".to_string(), |m| m.as_str().to_string()), - caps.get(3).map_or("user".to_string(), |m| m.as_str().to_string()), - caps.get(4).map_or("repo".to_string(), |m| m.as_str().to_string())) -} - -pub fn generate_path_from_provider_user_repo( - provider: &String, - user: &String, - repo: &String, - base_path: &String, -) -> String { - return format!("{}/{}/{}/{}", base_path.trim_end_matches("/"), provider, user, repo); -} \ No newline at end of file diff --git a/src/server/sync.rs b/src/server/sync.rs new file mode 100644 index 0000000..c2c4202 --- /dev/null +++ b/src/server/sync.rs @@ -0,0 +1,39 @@ +use crate::config::config; +use crate::config::repository::generate_credentials_from_clone_url; +use crate::dto::response::{RepoDto, RepoWrapperDto}; +use crate::gtm::git; + +pub async fn sync_all() -> bool { + let cfg = config::load(&config::CONFIG_PATH); + let client = reqwest::Client::new(); + + + for repo in &cfg.repositories { + let git_repo = git::clone_or_open(&repo).unwrap(); + let _res = git::fetch(&git_repo, &repo); + let commits = git::read_commits(&git_repo).unwrap(); + let (provider, user, repo) = generate_credentials_from_clone_url(&repo.url); + let gtm_repo: RepoDto = RepoDto { + provider: provider.clone(), + user: user.clone(), + repo: repo.clone(), + sync_url: cfg.get_sync_url(), + access_token: cfg.access_token.clone(), + commits, + }; + let dto = RepoWrapperDto { + repository: Option::from(gtm_repo) + }; + + let _res = client.post(&cfg.get_target_url()) // TODO: provider / user / repo + .body(serde_json::to_string(&dto).unwrap()) + .send() + .await; + } + + return true; +} + +pub fn sync_single() -> bool { + false +} \ No newline at end of file From bcb3afa885b49468401d1292140c289cb0d6b444 Mon Sep 17 00:00:00 2001 From: tavo Date: Mon, 28 Dec 2020 11:49:04 +0200 Subject: [PATCH 20/33] Force sync all should work --- src/main.rs | 1 + src/server/controller.rs | 2 +- src/server/mod.rs | 3 +-- src/sync/mod.rs | 1 + src/{server => sync}/sync.rs | 17 ++++++++++++----- 5 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 src/sync/mod.rs rename src/{server => sync}/sync.rs (71%) diff --git a/src/main.rs b/src/main.rs index fba803b..0532e40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ mod server; mod dto; mod gtm; mod config; +mod sync; fn main() { server::server::run(); diff --git a/src/server/controller.rs b/src/server/controller.rs index fcf0aa2..b20301a 100644 --- a/src/server/controller.rs +++ b/src/server/controller.rs @@ -2,7 +2,7 @@ use rocket_contrib::json::{JsonValue, Json}; use crate::server::service; use crate::dto::request::AddRepositoryDto; -use crate::server::sync::sync_all; +use crate::sync::sync::sync_all; #[get("/")] pub fn index() -> &'static str { diff --git a/src/server/mod.rs b/src/server/mod.rs index a6d56db..a49c438 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,5 +1,4 @@ pub mod server; mod controller; -mod service; -mod sync; // TODO: Move to some other module \ No newline at end of file +mod service; \ No newline at end of file diff --git a/src/sync/mod.rs b/src/sync/mod.rs new file mode 100644 index 0000000..de7c164 --- /dev/null +++ b/src/sync/mod.rs @@ -0,0 +1 @@ +pub mod sync; \ No newline at end of file diff --git a/src/server/sync.rs b/src/sync/sync.rs similarity index 71% rename from src/server/sync.rs rename to src/sync/sync.rs index c2c4202..e24b24f 100644 --- a/src/server/sync.rs +++ b/src/sync/sync.rs @@ -3,10 +3,10 @@ use crate::config::repository::generate_credentials_from_clone_url; use crate::dto::response::{RepoDto, RepoWrapperDto}; use crate::gtm::git; -pub async fn sync_all() -> bool { +pub async fn sync_all() -> i32 { let cfg = config::load(&config::CONFIG_PATH); let client = reqwest::Client::new(); - + let mut synced_count = 0; for repo in &cfg.repositories { let git_repo = git::clone_or_open(&repo).unwrap(); @@ -25,15 +25,22 @@ pub async fn sync_all() -> bool { repository: Option::from(gtm_repo) }; - let _res = client.post(&cfg.get_target_url()) // TODO: provider / user / repo - .body(serde_json::to_string(&dto).unwrap()) + let res = client.post(&generate_repo_sync_url(&cfg.get_target_url())) + .json(&dto) .send() .await; + if res.is_ok() { + synced_count += 1; + } // TODO: Handle error } - return true; + return synced_count; } pub fn sync_single() -> bool { false +} + +fn generate_repo_sync_url(target_host: &String) -> String { + return format!("{}/api/repositories", target_host) } \ No newline at end of file From 119df185ff3beef90e27ee7b5c5a8e65b03b48b6 Mon Sep 17 00:00:00 2001 From: tavo Date: Fri, 1 Jan 2021 20:47:43 +0200 Subject: [PATCH 21/33] Optimize sync to only sync untracked commits --- src/gtm/gtm.rs | 2 +- src/server/controller.rs | 4 +-- src/sync/sync.rs | 55 +++++++++++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/gtm/gtm.rs b/src/gtm/gtm.rs index 7f340bf..53a842a 100644 --- a/src/gtm/gtm.rs +++ b/src/gtm/gtm.rs @@ -13,7 +13,7 @@ lazy_static! { #[derive(Serialize)] pub struct Commit { - hash: String, + pub hash: String, branch: String, author: String, message: String, diff --git a/src/server/controller.rs b/src/server/controller.rs index b20301a..bb91e7e 100644 --- a/src/server/controller.rs +++ b/src/server/controller.rs @@ -24,6 +24,6 @@ pub fn add_repo(repo: Json) -> JsonValue { #[get("/sync")] pub fn sync() -> JsonValue { let mut rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(sync_all()); - rocket_contrib::json!("{}") + let response = rt.block_on(sync_all()); + rocket_contrib::json!(&response) } \ No newline at end of file diff --git a/src/sync/sync.rs b/src/sync/sync.rs index e24b24f..1ac5e98 100644 --- a/src/sync/sync.rs +++ b/src/sync/sync.rs @@ -1,17 +1,45 @@ +use reqwest::Client; +use serde::{Deserialize, Serialize}; + use crate::config::config; -use crate::config::repository::generate_credentials_from_clone_url; +use crate::config::repository::{generate_credentials_from_clone_url, Repository}; use crate::dto::response::{RepoDto, RepoWrapperDto}; use crate::gtm::git; -pub async fn sync_all() -> i32 { +#[derive(Serialize)] +pub struct SyncAllResult { + error: Option, + synced_count: i32, +} + +#[derive(Deserialize)] +pub struct LastSyncResponse { + hash: String, + timestamp: i64, + tracked_commit_hashes: Vec, +} + +pub async fn sync_all() -> SyncAllResult { let cfg = config::load(&config::CONFIG_PATH); let client = reqwest::Client::new(); - let mut synced_count = 0; + let mut result = SyncAllResult { error: None, synced_count: 0 }; for repo in &cfg.repositories { let git_repo = git::clone_or_open(&repo).unwrap(); let _res = git::fetch(&git_repo, &repo); - let commits = git::read_commits(&git_repo).unwrap(); + let last_sync = fetch_synced_hashes(&client, &repo) + .await + .unwrap_or(LastSyncResponse { + hash: "".to_string(), + timestamp: 0, + tracked_commit_hashes: vec![], + }); + let commits = git::read_commits(&git_repo) + .unwrap() + .into_iter() + .filter(|c| !last_sync.tracked_commit_hashes.contains(&c.hash)) + .collect(); + let (provider, user, repo) = generate_credentials_from_clone_url(&repo.url); let gtm_repo: RepoDto = RepoDto { provider: provider.clone(), @@ -30,11 +58,13 @@ pub async fn sync_all() -> i32 { .send() .await; if res.is_ok() { - synced_count += 1; - } // TODO: Handle error + result.synced_count += 1; + } else { + result.error = Option::from(res.err().unwrap().to_string()) + } } - return synced_count; + return result; } pub fn sync_single() -> bool { @@ -43,4 +73,15 @@ pub fn sync_single() -> bool { fn generate_repo_sync_url(target_host: &String) -> String { return format!("{}/api/repositories", target_host) +} + +async fn fetch_synced_hashes(client: &Client, repo: &Repository) -> Result { + let (provider, user, repo) = generate_credentials_from_clone_url(&repo.url); + let url = format!("/commits/{}/{}/{}/hash", provider, user, repo); + + return Ok(client.get(&url) + .send() + .await? + .json::() + .await?) } \ No newline at end of file From 85676fd1e3b75df418d2347123a59fd96c04067a Mon Sep 17 00:00:00 2001 From: tavo Date: Wed, 6 Jan 2021 15:04:51 +0200 Subject: [PATCH 22/33] Squashed commit of the following: commit 02e09a237dcff9446efc30c7d4b6b0b776eb815e Author: tavo Date: Tue Jan 5 20:53:11 2021 +0200 master -> main commit 7a4d6cc0f627f7348d8fbf9c2183542f2dbc2211 Author: tavo Date: Tue Jan 5 20:48:55 2021 +0200 Remove test commit 08822320c886b681388fffcbc62eec73d269b0c6 Author: tavo Date: Tue Jan 5 20:43:40 2021 +0200 Add release workflow commit ac83228d9e8c6dc719330de9d88023f490f3d85c Author: tavo Date: Mon Jan 4 21:27:04 2021 +0200 Fix path commit 08c7fe54f32c6a812da2db52bd9f0d2e34cc0702 Author: tavo Date: Mon Jan 4 21:23:02 2021 +0200 test commit 671deb1ac369102325cddbc6060a9be7854f14dd Author: tavo Date: Mon Jan 4 21:14:54 2021 +0200 test commit 75dc404ceb4a983ff3d6b5aff30668d54ab6d877 Author: tavo Date: Mon Jan 4 20:48:07 2021 +0200 Add correct path commit 118170d649d2bf012b72b414ab543aae3dad9784 Author: tavo Date: Mon Jan 4 20:43:22 2021 +0200 Test again commit 1438467070bc72d386223cd2d860251bfd9a9255 Author: tavo Date: Mon Jan 4 20:41:42 2021 +0200 Add develop workflow --- .github/workflows/develop.yml | 31 ++++++++++++++++++++ .github/workflows/release.yaml | 53 ++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 .github/workflows/develop.yml create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml new file mode 100644 index 0000000..567a05a --- /dev/null +++ b/.github/workflows/develop.yml @@ -0,0 +1,31 @@ +name: Develop +on: + push: + branches: + - develop + - main + pull_request: + branches: + - develop + - main + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install latest nightly + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + components: rustfmt, clippy + - name: Run cargo build + uses: actions-rs/cargo@v1 + with: + command: build + args: --package gtm-sync --bin gtm-sync --release + - uses: actions/upload-artifact@v2 + with: + name: gtm-sync + path: ./target/release/gtm-sync diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..c490470 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,53 @@ +name: Release + +on: + workflow_dispatch: + inputs: + release_version: + description: 'Release version' + required: true + +jobs: + build-linux: + runs-on: ubuntu-latest + env: + APP_NAME: 'gtm-sync' + MAINTAINER: 'DEVELOPEST' + DESC: 'gtm-sync client for linux' + steps: + - uses: actions/checkout@v2 + - name: Install latest nightly + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + components: rustfmt, clippy + - name: Run cargo build + uses: actions-rs/cargo@v1 + with: + command: build + args: --package gtm-sync --bin gtm-sync --release + - uses: actions/upload-artifact@v2 + with: + name: gtm-sync + path: ./target/release/gtm-sync + - name: Prepare deb package + run: | + mkdir -p .debpkg/usr/bin + cp ./target/release/gtm-sync .debpkg/usr/bin + chmod +x .debpkg/usr/bin + - name: Build deb package + uses: jiro4989/build-deb-action@v2 + with: + package: ${{ env.APP_NAME }} + package_root: .debpkg + maintainer: ${{ env.MAINTAINER }} + version: ${{ github.event.inputs.release_version }} + arch: 'amd64' + desc: ${{ env.DESC }} + - name: Upload deb package + uses: actions/upload-artifact@v2 + with: + name: gtm-sync-debian + path: | + ./*.deb \ No newline at end of file From e19cf484b2331865363feee57b6ddd6a44a79baf Mon Sep 17 00:00:00 2001 From: tavo Date: Fri, 8 Jan 2021 21:28:29 +0200 Subject: [PATCH 23/33] Add sync single --- src/config/repository.rs | 2 +- src/server/controller.rs | 20 ++++---- src/server/server.rs | 4 +- src/sync/sync.rs | 103 ++++++++++++++++++++++++++------------- 4 files changed, 82 insertions(+), 47 deletions(-) diff --git a/src/config/repository.rs b/src/config/repository.rs index 192fb4f..9e749a0 100644 --- a/src/config/repository.rs +++ b/src/config/repository.rs @@ -7,7 +7,7 @@ lazy_static! { Regex::new(r#"(git@|https://)([a-zA-Z0-9.]+)[:/]([a-zA-Z0-9-]+)/([a-zA-Z0-9-]+)\.git"#).unwrap(); } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone)] pub struct Repository { pub url: String, pub path: String, diff --git a/src/server/controller.rs b/src/server/controller.rs index bb91e7e..12d3d0d 100644 --- a/src/server/controller.rs +++ b/src/server/controller.rs @@ -2,12 +2,7 @@ use rocket_contrib::json::{JsonValue, Json}; use crate::server::service; use crate::dto::request::AddRepositoryDto; -use crate::sync::sync::sync_all; - -#[get("/")] -pub fn index() -> &'static str { - "Hello, world!" -} +use crate::sync::sync; #[get("/repository///")] pub fn repo(provider: String, user: String, repo: String) -> JsonValue { @@ -21,9 +16,16 @@ pub fn add_repo(repo: Json) -> JsonValue { rocket_contrib::json!(&response) } -#[get("/sync")] -pub fn sync() -> JsonValue { +#[get("/sync-all")] +pub fn sync_all() -> JsonValue { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + let response = rt.block_on(sync::sync_all()); + rocket_contrib::json!(&response) +} + +#[get("/repository////sync")] +pub fn sync_repo(provider: String, user: String, repo: String) -> JsonValue { let mut rt = tokio::runtime::Runtime::new().unwrap(); - let response = rt.block_on(sync_all()); + let response = rt.block_on(sync::sync_repo(&provider, &user, &repo)); rocket_contrib::json!(&response) } \ No newline at end of file diff --git a/src/server/server.rs b/src/server/server.rs index e7d89f0..0abdba3 100644 --- a/src/server/server.rs +++ b/src/server/server.rs @@ -5,9 +5,9 @@ pub fn run() { rocket::ignite() .mount("/", routes![ - controller::index, controller::repo, controller::add_repo, - controller::sync], + controller::sync_repo, + controller::sync_all], ).launch(); } \ No newline at end of file diff --git a/src/sync/sync.rs b/src/sync/sync.rs index 1ac5e98..1e8e89b 100644 --- a/src/sync/sync.rs +++ b/src/sync/sync.rs @@ -12,6 +12,12 @@ pub struct SyncAllResult { synced_count: i32, } +#[derive(Serialize)] +pub struct SyncSingleResult { + error: Option, + ok: bool, +} + #[derive(Deserialize)] pub struct LastSyncResponse { hash: String, @@ -24,39 +30,13 @@ pub async fn sync_all() -> SyncAllResult { let client = reqwest::Client::new(); let mut result = SyncAllResult { error: None, synced_count: 0 }; - for repo in &cfg.repositories { - let git_repo = git::clone_or_open(&repo).unwrap(); - let _res = git::fetch(&git_repo, &repo); - let last_sync = fetch_synced_hashes(&client, &repo) - .await - .unwrap_or(LastSyncResponse { - hash: "".to_string(), - timestamp: 0, - tracked_commit_hashes: vec![], - }); - let commits = git::read_commits(&git_repo) - .unwrap() - .into_iter() - .filter(|c| !last_sync.tracked_commit_hashes.contains(&c.hash)) - .collect(); - - let (provider, user, repo) = generate_credentials_from_clone_url(&repo.url); - let gtm_repo: RepoDto = RepoDto { - provider: provider.clone(), - user: user.clone(), - repo: repo.clone(), - sync_url: cfg.get_sync_url(), - access_token: cfg.access_token.clone(), - commits, - }; - let dto = RepoWrapperDto { - repository: Option::from(gtm_repo) - }; - - let res = client.post(&generate_repo_sync_url(&cfg.get_target_url())) - .json(&dto) - .send() - .await; + let tasks: Vec<_> = cfg.repositories + .iter() + .map(|repo| sync_single(&repo, &cfg, &client)) + .collect(); + + for task in tasks { + let res = task.await; if res.is_ok() { result.synced_count += 1; } else { @@ -67,8 +47,61 @@ pub async fn sync_all() -> SyncAllResult { return result; } -pub fn sync_single() -> bool { - false +pub async fn sync_repo(provider: &String, user: &String, repo: &String) -> SyncSingleResult { + let cfg = config::load(&config::CONFIG_PATH); + let client = reqwest::Client::new(); + + let repo_to_sync = cfg.repositories.iter() + .find(|&r| r.path == cfg.generate_path_from_provider_user_repo(&provider, &user, &repo)); + if repo_to_sync.is_none() { + return SyncSingleResult { error: Option::from("No matching repository found!".to_string()), ok: false } + } + let repo_to_sync = repo_to_sync.unwrap(); + let res = sync_single(&repo_to_sync, &cfg, &client).await; + + if res.is_err() { + return SyncSingleResult { error: Option::from("Error syncing repo!".to_string()), ok: false } + } + return SyncSingleResult { error: None, ok: true } +} + +async fn sync_single( + repo: &Repository, + cfg: &config::Config, + client: &reqwest::Client, +) -> Result { + let git_repo = git::clone_or_open(&repo).unwrap(); + let _res = git::fetch(&git_repo, &repo); + let last_sync = fetch_synced_hashes(&client, &repo) + .await + .unwrap_or(LastSyncResponse { + hash: "".to_string(), + timestamp: 0, + tracked_commit_hashes: vec![], + }); + let commits = git::read_commits(&git_repo) + .unwrap() + .into_iter() + .filter(|c| !last_sync.tracked_commit_hashes.contains(&c.hash)) + .collect(); + + let (provider, user, repo) = generate_credentials_from_clone_url(&repo.url); + let gtm_repo: RepoDto = RepoDto { + provider: provider.clone(), + user: user.clone(), + repo: repo.clone(), + sync_url: cfg.get_sync_url(), + access_token: cfg.access_token.clone(), + commits, + }; + let dto = RepoWrapperDto { + repository: Option::from(gtm_repo) + }; + + return client.post(&generate_repo_sync_url(&cfg.get_target_url())) + .json(&dto) + .send() + .await; } fn generate_repo_sync_url(target_host: &String) -> String { From 75914ac24d19dd972d8d209c8e8581d29206fbe5 Mon Sep 17 00:00:00 2001 From: tavo Date: Fri, 8 Jan 2021 21:32:51 +0200 Subject: [PATCH 24/33] Refactor for readability --- src/main.rs | 1 + src/repo/mod.rs | 1 + src/{server/service.rs => repo/repo_manager.rs} | 0 src/server/controller.rs | 6 +++--- src/server/mod.rs | 1 - 5 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 src/repo/mod.rs rename src/{server/service.rs => repo/repo_manager.rs} (100%) diff --git a/src/main.rs b/src/main.rs index 0532e40..c0dd5ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ mod dto; mod gtm; mod config; mod sync; +mod repo; fn main() { server::server::run(); diff --git a/src/repo/mod.rs b/src/repo/mod.rs new file mode 100644 index 0000000..0dd512f --- /dev/null +++ b/src/repo/mod.rs @@ -0,0 +1 @@ +pub mod repo_manager; diff --git a/src/server/service.rs b/src/repo/repo_manager.rs similarity index 100% rename from src/server/service.rs rename to src/repo/repo_manager.rs diff --git a/src/server/controller.rs b/src/server/controller.rs index 12d3d0d..9f24f2c 100644 --- a/src/server/controller.rs +++ b/src/server/controller.rs @@ -1,18 +1,18 @@ use rocket_contrib::json::{JsonValue, Json}; -use crate::server::service; +use crate::repo::repo_manager; use crate::dto::request::AddRepositoryDto; use crate::sync::sync; #[get("/repository///")] pub fn repo(provider: String, user: String, repo: String) -> JsonValue { - let repo = service::get_repo(&provider, &user, &repo); // TODO: How to match credentials? + let repo = repo_manager::get_repo(&provider, &user, &repo); // TODO: How to match credentials? rocket_contrib::json!(&repo) } #[post("/repository", data="")] pub fn add_repo(repo: Json) -> JsonValue { - let response = service::add_repo(repo.into_inner()); + let response = repo_manager::add_repo(repo.into_inner()); rocket_contrib::json!(&response) } diff --git a/src/server/mod.rs b/src/server/mod.rs index a49c438..53ca2b0 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,4 +1,3 @@ pub mod server; mod controller; -mod service; \ No newline at end of file From 6bfafa83c0b4f0cadeb7393c1e9f01013ffbdc24 Mon Sep 17 00:00:00 2001 From: tavo Date: Sat, 9 Jan 2021 10:28:10 +0200 Subject: [PATCH 25/33] Small refacto --- src/dto/mod.rs | 2 -- src/dto/request.rs | 24 ------------------ src/dto/response.rs | 27 -------------------- src/main.rs | 1 - src/repo/repo_manager.rs | 54 +++++++++++++++++++++++++++++++++++++--- src/server/controller.rs | 10 ++++---- src/sync/sync.rs | 2 +- 7 files changed, 57 insertions(+), 63 deletions(-) delete mode 100644 src/dto/mod.rs delete mode 100644 src/dto/request.rs delete mode 100644 src/dto/response.rs diff --git a/src/dto/mod.rs b/src/dto/mod.rs deleted file mode 100644 index 4015ff8..0000000 --- a/src/dto/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod request; -pub mod response; \ No newline at end of file diff --git a/src/dto/request.rs b/src/dto/request.rs deleted file mode 100644 index 1eb42df..0000000 --- a/src/dto/request.rs +++ /dev/null @@ -1,24 +0,0 @@ -use serde::{Serialize, Deserialize}; -use crate::config::repository::Repository; - -#[derive(Serialize, Deserialize)] -pub struct AddRepositoryDto { - pub url: String, - pub ssh_private_key: Option, - pub ssh_public_key: Option, - pub ssh_user: Option, - pub ssh_passphrase: Option, -} - -impl AddRepositoryDto { - pub fn to_repository(&self, f: &dyn Fn(&String) -> String) -> Repository { - return Repository { - url: self.url.clone(), - path: f(&self.url.to_string()), - ssh_private_key: self.ssh_private_key.clone(), - ssh_public_key: self.ssh_public_key.clone(), - ssh_user: self.ssh_user.clone(), - ssh_passphrase: self.ssh_passphrase.clone(), - } - } -} diff --git a/src/dto/response.rs b/src/dto/response.rs deleted file mode 100644 index 8a5fbdd..0000000 --- a/src/dto/response.rs +++ /dev/null @@ -1,27 +0,0 @@ -use serde::{Serialize}; -use crate::gtm::gtm::Commit; - -#[derive(Serialize)] -pub struct AddRepoDto { - pub success: bool, - pub provider: Option, - pub user: Option, - pub repo: Option, - pub message: Option, -} - -#[derive(Serialize)] -pub struct RepoDto { - pub provider: String, - pub user: String, - pub repo: String, - pub sync_url: String, - pub access_token: Option, - pub commits: Vec, -} - -#[derive(Serialize)] -pub struct RepoWrapperDto { - pub repository: Option, - // TODO: Errors -} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index c0dd5ab..2c4879b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,6 @@ #[macro_use] extern crate rocket; mod server; -mod dto; mod gtm; mod config; mod sync; diff --git a/src/repo/repo_manager.rs b/src/repo/repo_manager.rs index c559b16..39488cc 100644 --- a/src/repo/repo_manager.rs +++ b/src/repo/repo_manager.rs @@ -1,9 +1,57 @@ +use serde::{Deserialize, Serialize}; use crate::config::config; -use crate::dto::request::AddRepositoryDto; -use crate::dto::response::{AddRepoDto, RepoDto, RepoWrapperDto}; -use crate::gtm::git; use crate::config::repository::generate_credentials_from_clone_url; +use crate::config::repository::Repository; +use crate::gtm::git; +use crate::gtm::gtm::Commit; + +#[derive(Serialize, Deserialize)] +pub struct AddRepositoryDto { + pub url: String, + pub ssh_private_key: Option, + pub ssh_public_key: Option, + pub ssh_user: Option, + pub ssh_passphrase: Option, +} + +#[derive(Serialize)] +pub struct AddRepoDto { + pub success: bool, + pub provider: Option, + pub user: Option, + pub repo: Option, + pub message: Option, +} + +#[derive(Serialize)] +pub struct RepoDto { + pub provider: String, + pub user: String, + pub repo: String, + pub sync_url: String, + pub access_token: Option, + pub commits: Vec, +} + +#[derive(Serialize)] +pub struct RepoWrapperDto { + pub repository: Option, + // TODO: Errors +} + +impl AddRepositoryDto { + pub fn to_repository(&self, f: &dyn Fn(&String) -> String) -> Repository { + return Repository { + url: self.url.clone(), + path: f(&self.url.to_string()), + ssh_private_key: self.ssh_private_key.clone(), + ssh_public_key: self.ssh_public_key.clone(), + ssh_user: self.ssh_user.clone(), + ssh_passphrase: self.ssh_passphrase.clone(), + } + } +} pub fn get_repo(provider: &String, user: &String, repo: &String) -> RepoWrapperDto { let cfg = config::load(&config::CONFIG_PATH); diff --git a/src/server/controller.rs b/src/server/controller.rs index 9f24f2c..602ddd8 100644 --- a/src/server/controller.rs +++ b/src/server/controller.rs @@ -1,29 +1,29 @@ use rocket_contrib::json::{JsonValue, Json}; use crate::repo::repo_manager; -use crate::dto::request::AddRepositoryDto; use crate::sync::sync; +use crate::repo::repo_manager::AddRepositoryDto; -#[get("/repository///")] +#[get("/repositories///")] pub fn repo(provider: String, user: String, repo: String) -> JsonValue { let repo = repo_manager::get_repo(&provider, &user, &repo); // TODO: How to match credentials? rocket_contrib::json!(&repo) } -#[post("/repository", data="")] +#[post("/repositories", data="")] pub fn add_repo(repo: Json) -> JsonValue { let response = repo_manager::add_repo(repo.into_inner()); rocket_contrib::json!(&response) } -#[get("/sync-all")] +#[get("/repositories/sync-all")] pub fn sync_all() -> JsonValue { let mut rt = tokio::runtime::Runtime::new().unwrap(); let response = rt.block_on(sync::sync_all()); rocket_contrib::json!(&response) } -#[get("/repository////sync")] +#[get("/repositories////sync")] pub fn sync_repo(provider: String, user: String, repo: String) -> JsonValue { let mut rt = tokio::runtime::Runtime::new().unwrap(); let response = rt.block_on(sync::sync_repo(&provider, &user, &repo)); diff --git a/src/sync/sync.rs b/src/sync/sync.rs index 1e8e89b..96c75c2 100644 --- a/src/sync/sync.rs +++ b/src/sync/sync.rs @@ -3,8 +3,8 @@ use serde::{Deserialize, Serialize}; use crate::config::config; use crate::config::repository::{generate_credentials_from_clone_url, Repository}; -use crate::dto::response::{RepoDto, RepoWrapperDto}; use crate::gtm::git; +use crate::repo::repo_manager::{RepoDto, RepoWrapperDto}; #[derive(Serialize)] pub struct SyncAllResult { From 34d520155412a493b1b42a32cbfd8ccb4ad561a3 Mon Sep 17 00:00:00 2001 From: tavo Date: Tue, 12 Jan 2021 14:36:17 +0200 Subject: [PATCH 26/33] Add global ssh key options --- src/config/config.rs | 4 ++++ src/gtm/git.rs | 25 +++++++++++++++---------- src/repo/repo_manager.rs | 6 +++--- src/sync/sync.rs | 4 ++-- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/config/config.rs b/src/config/config.rs index 5551a5e..9e8b22a 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -17,6 +17,10 @@ pub struct Config { pub port: Option, pub address: Option, pub access_token: Option, + pub ssh_public_key: Option, + pub ssh_private_key: Option, + pub ssh_user: Option, + pub ssh_passphrase: Option, pub repositories_base_path: String, #[serde(skip_serializing_if = "Vec::is_empty", default)] pub repositories: Vec, diff --git a/src/gtm/git.rs b/src/gtm/git.rs index db51428..a6b04a8 100644 --- a/src/gtm/git.rs +++ b/src/gtm/git.rs @@ -9,6 +9,7 @@ use git2::build::RepoBuilder; use crate::gtm::gtm; use crate::config::repository; +use crate::config::config::Config; static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; @@ -16,7 +17,7 @@ static DEFAULT_ORIGIN: &str = "origin"; static ORIGIN_PREFIX: &str = "refs/remotes/origin/"; static ORIGIN_HEAD: &str = "refs/remotes/origin/HEAD"; -pub fn clone_or_open(repo_config: &repository::Repository) -> Result { +pub fn clone_or_open(repo_config: &repository::Repository, cfg: &Config) -> Result { let path = Path::new(&repo_config.path); if path.exists() { @@ -26,26 +27,30 @@ pub fn clone_or_open(repo_config: &repository::Repository) -> Result FetchOptions { +fn generate_fetch_options<'a>(repo_config: &'a repository::Repository, cfg: &'a Config) -> FetchOptions<'a> { let mut cb = RemoteCallbacks::new(); let repo_config = repo_config.clone(); cb.credentials(move |_c, _o, t| { if t.is_ssh_key() { return git2::Cred::ssh_key( - &repo_config.ssh_user.as_ref().unwrap_or(&"git".to_string()), - Option::from(Path::new(&repo_config.ssh_public_key.as_ref().unwrap_or(&"".to_string()))), - &Path::new(&repo_config.ssh_private_key.as_ref().unwrap_or(&"".to_string())), - repo_config.ssh_passphrase.as_ref().map(|x| &**x), + &repo_config.ssh_user.as_ref() + .unwrap_or(cfg.ssh_user.as_ref().unwrap_or(&"git".to_string())), + Option::from(Path::new(&repo_config.ssh_public_key.as_ref() + .unwrap_or(cfg.ssh_public_key.as_ref().unwrap_or(&"".to_string())))), + &Path::new(&repo_config.ssh_private_key.as_ref() + .unwrap_or(cfg.ssh_private_key.as_ref().unwrap_or(&"".to_string()))), + repo_config.ssh_passphrase.as_ref() + .or(cfg.ssh_passphrase.as_ref()).map(|x| &**x), ) } return git2::Cred::default(); @@ -56,7 +61,7 @@ fn generate_fetch_options(repo_config: &repository::Repository) -> FetchOptions return fo; } -pub fn fetch(repo: &Repository, repo_config: &repository::Repository) { +pub fn fetch(repo: &Repository, repo_config: &repository::Repository, cfg: &Config) { let mut remote = repo.find_remote(DEFAULT_ORIGIN) .expect("Unable to find remote 'origin'"); let mut ref_added = false; @@ -89,7 +94,7 @@ pub fn fetch(repo: &Repository, repo_config: &repository::Repository) { } fetch_refs.push(GTM_NOTES_REF.parse().unwrap()); - let mut fo = generate_fetch_options(repo_config); + let mut fo = generate_fetch_options(repo_config, cfg); remote.fetch(&fetch_refs, Option::from(&mut fo), None) .expect("Error fetching data!"); remote.disconnect().unwrap(); diff --git a/src/repo/repo_manager.rs b/src/repo/repo_manager.rs index 39488cc..064e7da 100644 --- a/src/repo/repo_manager.rs +++ b/src/repo/repo_manager.rs @@ -66,8 +66,8 @@ pub fn get_repo(provider: &String, user: &String, repo: &String) -> RepoWrapperD } let repo_to_clone = repo_to_clone.unwrap(); - let git_repo = git::clone_or_open(&repo_to_clone).unwrap(); - let _res = git::fetch(&git_repo, &repo_to_clone); + let git_repo = git::clone_or_open(&repo_to_clone, &cfg).unwrap(); + let _res = git::fetch(&git_repo, &repo_to_clone, &cfg); let commits = git::read_commits(&git_repo).unwrap(); let gtm_repo: RepoDto = RepoDto { provider: provider.clone(), @@ -85,7 +85,7 @@ pub fn get_repo(provider: &String, user: &String, repo: &String) -> RepoWrapperD pub fn add_repo(repo_dto: AddRepositoryDto) -> AddRepoDto { let mut cfg = config::load(&config::CONFIG_PATH); let repo = repo_dto.to_repository(&|url: &String| { cfg.generate_path_from_git_url(url) }); - let cloned_repo = git::clone_or_open(&repo); + let cloned_repo = git::clone_or_open(&repo, &cfg); if cloned_repo.is_ok() { let (provider, user, repository) = generate_credentials_from_clone_url(&repo.url); if !cfg.repositories.iter().any(|r| r.url == repo_dto.url) { diff --git a/src/sync/sync.rs b/src/sync/sync.rs index 96c75c2..20945c9 100644 --- a/src/sync/sync.rs +++ b/src/sync/sync.rs @@ -70,8 +70,8 @@ async fn sync_single( cfg: &config::Config, client: &reqwest::Client, ) -> Result { - let git_repo = git::clone_or_open(&repo).unwrap(); - let _res = git::fetch(&git_repo, &repo); + let git_repo = git::clone_or_open(&repo, &cfg).unwrap(); + let _res = git::fetch(&git_repo, &repo, &cfg); let last_sync = fetch_synced_hashes(&client, &repo) .await .unwrap_or(LastSyncResponse { From 0aa3296649ff5ad0606c857addc9c1cb166ba503 Mon Sep 17 00:00:00 2001 From: tavo Date: Tue, 12 Jan 2021 14:52:23 +0200 Subject: [PATCH 27/33] Fix gtm syncing items twice --- src/config/config.rs | 4 ++-- src/sync/sync.rs | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/config/config.rs b/src/config/config.rs index 9e8b22a..70ff74f 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -12,8 +12,8 @@ lazy_static! { #[derive(Serialize, Deserialize)] pub struct Config { - pub target_host: String, - pub target_port: Option, + target_host: String, + target_port: Option, pub port: Option, pub address: Option, pub access_token: Option, diff --git a/src/sync/sync.rs b/src/sync/sync.rs index 20945c9..b1bedd7 100644 --- a/src/sync/sync.rs +++ b/src/sync/sync.rs @@ -72,7 +72,7 @@ async fn sync_single( ) -> Result { let git_repo = git::clone_or_open(&repo, &cfg).unwrap(); let _res = git::fetch(&git_repo, &repo, &cfg); - let last_sync = fetch_synced_hashes(&client, &repo) + let last_sync = fetch_synced_hashes(&client, &repo, &cfg.get_target_url()) .await .unwrap_or(LastSyncResponse { hash: "".to_string(), @@ -108,9 +108,13 @@ fn generate_repo_sync_url(target_host: &String) -> String { return format!("{}/api/repositories", target_host) } -async fn fetch_synced_hashes(client: &Client, repo: &Repository) -> Result { +async fn fetch_synced_hashes( + client: &Client, + repo: &Repository, + target_host: &str +) -> Result { let (provider, user, repo) = generate_credentials_from_clone_url(&repo.url); - let url = format!("/commits/{}/{}/{}/hash", provider, user, repo); + let url = format!("{}/api/commits/{}/{}/{}/hash", target_host, provider, user, repo); return Ok(client.get(&url) .send() From a60977022a43fc9dfeed39f8328c2c1dc9ee2287 Mon Sep 17 00:00:00 2001 From: tavo Date: Tue, 12 Jan 2021 15:44:16 +0200 Subject: [PATCH 28/33] Sync seems to be working --- src/gtm/gtm.rs | 2 +- src/sync/sync.rs | 27 +++++++++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/gtm/gtm.rs b/src/gtm/gtm.rs index 53a842a..78964d8 100644 --- a/src/gtm/gtm.rs +++ b/src/gtm/gtm.rs @@ -3,8 +3,8 @@ use std::fmt; use git2::{DiffOptions, Note}; use lazy_static::lazy_static; use regex::Regex; -use serde::export::Formatter; use serde::Serialize; +use std::fmt::Formatter; lazy_static! { static ref NOTE_HEADER_REGEX: Regex = Regex::new("\\[ver:\\d+,total:\\d+]").unwrap(); diff --git a/src/sync/sync.rs b/src/sync/sync.rs index b1bedd7..e668b3b 100644 --- a/src/sync/sync.rs +++ b/src/sync/sync.rs @@ -1,10 +1,11 @@ -use reqwest::Client; +use reqwest::{Client, Method}; use serde::{Deserialize, Serialize}; use crate::config::config; use crate::config::repository::{generate_credentials_from_clone_url, Repository}; use crate::gtm::git; use crate::repo::repo_manager::{RepoDto, RepoWrapperDto}; +use crate::gtm::gtm::Commit; #[derive(Serialize)] pub struct SyncAllResult { @@ -76,16 +77,22 @@ async fn sync_single( .await .unwrap_or(LastSyncResponse { hash: "".to_string(), - timestamp: 0, + timestamp: -1, tracked_commit_hashes: vec![], }); - let commits = git::read_commits(&git_repo) - .unwrap() - .into_iter() - .filter(|c| !last_sync.tracked_commit_hashes.contains(&c.hash)) - .collect(); - + let mut commits: Vec = git::read_commits(&git_repo).unwrap(); + let commit_hashes: Vec = commits.iter().map(|c| c.hash.clone()).collect(); let (provider, user, repo) = generate_credentials_from_clone_url(&repo.url); + + let mut method = Method::POST; + if last_sync.timestamp > 0 && last_sync.tracked_commit_hashes.iter() + .all(|h| commit_hashes.contains(h)) { + commits = commits.into_iter() + .filter(|c| !last_sync.tracked_commit_hashes.contains(&c.hash)) + .collect(); + method = Method::PUT; + } + let gtm_repo: RepoDto = RepoDto { provider: provider.clone(), user: user.clone(), @@ -98,7 +105,7 @@ async fn sync_single( repository: Option::from(gtm_repo) }; - return client.post(&generate_repo_sync_url(&cfg.get_target_url())) + return client.request(method, &generate_repo_sync_url(&cfg.get_target_url())) .json(&dto) .send() .await; @@ -121,4 +128,4 @@ async fn fetch_synced_hashes( .await? .json::() .await?) -} \ No newline at end of file +} From 1e87cd07108afecd24d7c1ec87431cd94ab6f681 Mon Sep 17 00:00:00 2001 From: tavo Date: Wed, 6 Jan 2021 15:04:51 +0200 Subject: [PATCH 29/33] Squashed commit of the following: commit 02e09a237dcff9446efc30c7d4b6b0b776eb815e Author: tavo Date: Tue Jan 5 20:53:11 2021 +0200 master -> main commit 7a4d6cc0f627f7348d8fbf9c2183542f2dbc2211 Author: tavo Date: Tue Jan 5 20:48:55 2021 +0200 Remove test commit 08822320c886b681388fffcbc62eec73d269b0c6 Author: tavo Date: Tue Jan 5 20:43:40 2021 +0200 Add release workflow commit ac83228d9e8c6dc719330de9d88023f490f3d85c Author: tavo Date: Mon Jan 4 21:27:04 2021 +0200 Fix path commit 08c7fe54f32c6a812da2db52bd9f0d2e34cc0702 Author: tavo Date: Mon Jan 4 21:23:02 2021 +0200 test commit 671deb1ac369102325cddbc6060a9be7854f14dd Author: tavo Date: Mon Jan 4 21:14:54 2021 +0200 test commit 75dc404ceb4a983ff3d6b5aff30668d54ab6d877 Author: tavo Date: Mon Jan 4 20:48:07 2021 +0200 Add correct path commit 118170d649d2bf012b72b414ab543aae3dad9784 Author: tavo Date: Mon Jan 4 20:43:22 2021 +0200 Test again commit 1438467070bc72d386223cd2d860251bfd9a9255 Author: tavo Date: Mon Jan 4 20:41:42 2021 +0200 Add develop workflow --- .github/workflows/develop.yml | 31 ++++++++++++++++++++ .github/workflows/release.yaml | 53 ++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 .github/workflows/develop.yml create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml new file mode 100644 index 0000000..567a05a --- /dev/null +++ b/.github/workflows/develop.yml @@ -0,0 +1,31 @@ +name: Develop +on: + push: + branches: + - develop + - main + pull_request: + branches: + - develop + - main + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install latest nightly + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + components: rustfmt, clippy + - name: Run cargo build + uses: actions-rs/cargo@v1 + with: + command: build + args: --package gtm-sync --bin gtm-sync --release + - uses: actions/upload-artifact@v2 + with: + name: gtm-sync + path: ./target/release/gtm-sync diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..c490470 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,53 @@ +name: Release + +on: + workflow_dispatch: + inputs: + release_version: + description: 'Release version' + required: true + +jobs: + build-linux: + runs-on: ubuntu-latest + env: + APP_NAME: 'gtm-sync' + MAINTAINER: 'DEVELOPEST' + DESC: 'gtm-sync client for linux' + steps: + - uses: actions/checkout@v2 + - name: Install latest nightly + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + components: rustfmt, clippy + - name: Run cargo build + uses: actions-rs/cargo@v1 + with: + command: build + args: --package gtm-sync --bin gtm-sync --release + - uses: actions/upload-artifact@v2 + with: + name: gtm-sync + path: ./target/release/gtm-sync + - name: Prepare deb package + run: | + mkdir -p .debpkg/usr/bin + cp ./target/release/gtm-sync .debpkg/usr/bin + chmod +x .debpkg/usr/bin + - name: Build deb package + uses: jiro4989/build-deb-action@v2 + with: + package: ${{ env.APP_NAME }} + package_root: .debpkg + maintainer: ${{ env.MAINTAINER }} + version: ${{ github.event.inputs.release_version }} + arch: 'amd64' + desc: ${{ env.DESC }} + - name: Upload deb package + uses: actions/upload-artifact@v2 + with: + name: gtm-sync-debian + path: | + ./*.deb \ No newline at end of file From 6804f169f2e56c12ee230774ee3d4e4076b23550 Mon Sep 17 00:00:00 2001 From: tavo Date: Wed, 13 Jan 2021 21:19:06 +0200 Subject: [PATCH 30/33] Add git branch parsing from note --- src/gtm/git.rs | 17 ++--------------- src/gtm/gtm.rs | 26 ++++++++++++++------------ 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/src/gtm/git.rs b/src/gtm/git.rs index a6b04a8..98e1ea0 100644 --- a/src/gtm/git.rs +++ b/src/gtm/git.rs @@ -1,10 +1,9 @@ #![deny(warnings)] -use std::collections::HashMap; use std::fs; use std::path::Path; -use git2::{BranchType, Error, FetchOptions, Note, Oid, RemoteCallbacks, Repository}; +use git2::{BranchType, Error, FetchOptions, Note, RemoteCallbacks, Repository}; use git2::build::RepoBuilder; use crate::gtm::gtm; @@ -102,41 +101,29 @@ pub fn fetch(repo: &Repository, repo_config: &repository::Repository, cfg: &Conf pub fn read_commits(repo: &Repository) -> Result, Error> { let mut commits: Vec = Vec::new(); - let mut branch_map: HashMap = HashMap::new(); let mut revwalk = repo.revwalk().expect("Unable to revwalk!"); revwalk.set_sorting(git2::Sort::TOPOLOGICAL).expect("Unable to set revwalk sorting!"); revwalk.set_sorting(git2::Sort::REVERSE).expect("Unable to reverse revalk sorting!"); let branches = repo.branches(Option::from(BranchType::Remote)).unwrap(); - let default_branch = repo.head().unwrap().name().unwrap().to_string(); for branch in branches { let (branch, _) = branch.unwrap(); let refspec = branch.get().name().unwrap(); if refspec == ORIGIN_HEAD { continue } - branch_map.insert( - branch.get().target().unwrap(), - refspec.strip_prefix(ORIGIN_PREFIX).unwrap().to_string(), - ); let _ = revwalk.push_ref(refspec); } for commit_oid in revwalk { let commit_oid = commit_oid?; let commit = repo.find_commit(commit_oid)?; - let mut branch = "".to_string(); // TODO: Some more intelligent choice than last wins - for (oid, name) in &branch_map { - if repo.merge_base(commit_oid, *oid).unwrap() == commit_oid && branch != default_branch { - branch = name.clone(); - } - } let notes: Vec = repo.notes(Option::from(GTM_NOTES_REF))? .map(|n| n.unwrap()) .filter(|n| n.1 == commit_oid) .map(|n| repo.find_note(Option::from(GTM_NOTES_REF), n.1).unwrap()) .collect(); - let res = gtm::parse_commit(&repo, &commit, ¬es, branch)?; + let res = gtm::parse_commit(&repo, &commit, ¬es)?; commits.push(res); } return Result::Ok(commits); diff --git a/src/gtm/gtm.rs b/src/gtm/gtm.rs index 78964d8..58beef4 100644 --- a/src/gtm/gtm.rs +++ b/src/gtm/gtm.rs @@ -1,14 +1,13 @@ use std::fmt; +use std::fmt::Formatter; use git2::{DiffOptions, Note}; use lazy_static::lazy_static; use regex::Regex; use serde::Serialize; -use std::fmt::Formatter; lazy_static! { - static ref NOTE_HEADER_REGEX: Regex = Regex::new("\\[ver:\\d+,total:\\d+]").unwrap(); - static ref NOTE_HEADER_VALS_REGEX: Regex = Regex::new("\\d+").unwrap(); + static ref NOTE_HEADER_REGEX: Regex = Regex::new("\\[ver:(\\d+),total:(\\d+)(\\s*|,branch:([^]]+))]").unwrap(); } #[derive(Serialize)] @@ -37,10 +36,10 @@ pub struct TimelineEntry { time: i64, } -pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note], git_branch: String) -> Result { +pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note]) -> Result { let mut commit = Commit { hash: git_commit.id().to_string(), - branch: git_branch, + branch: "".to_string(), author: git_commit.author().to_string(), message: git_commit.message().unwrap().to_string(), time: git_commit.time().seconds(), // todo: validate @@ -49,26 +48,29 @@ pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: & for note in notes { let message = note.message().unwrap(); - let mut files = parse_note_message(message).unwrap_or(vec![]); + let (mut files, branch) = parse_note_message(message) + .unwrap_or((vec![], "".to_string())); let _diff = diff_parents(files.as_mut(), git_commit, repo); commit.files.append(files.as_mut()); + commit.branch = branch; } return Ok(commit); } -fn parse_note_message(message: &str) -> Option> { +fn parse_note_message(message: &str) -> Option<(Vec, String)> { let mut version: String = "".to_string(); + let mut branch: String = "".to_string(); let mut files: Vec = Vec::new(); let lines = message.split("\n"); for line in lines { if line.trim() == "" { version = "".to_string(); } else if NOTE_HEADER_REGEX.is_match(line) { - let matches: Vec = NOTE_HEADER_VALS_REGEX.find_iter(line) - .filter_map(|d| d.as_str().parse().ok()) - .collect(); - version = matches.get(0)?.clone(); + let matches = NOTE_HEADER_REGEX.captures(line)?; + + version = matches.get(1)?.as_str().to_string(); + branch = matches.get(3)?.as_str().to_string(); } let mut file = File { @@ -119,7 +121,7 @@ fn parse_note_message(message: &str) -> Option> { files.push(file); } } - return Option::from(files); + return Option::from((files, branch)); } fn diff_parents(files: &mut Vec, commit: &git2::Commit, repo: &git2::Repository) -> Result<(), git2::Error> { From 52d99082edbdec040e48f04785d1297cad6d3aeb Mon Sep 17 00:00:00 2001 From: tavo Date: Tue, 19 Jan 2021 22:15:32 +0200 Subject: [PATCH 31/33] Pr fixes --- src/config/repository.rs | 5 ----- src/gtm/gtm.rs | 26 +++++++++++++------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/config/repository.rs b/src/config/repository.rs index 9e749a0..bc23151 100644 --- a/src/config/repository.rs +++ b/src/config/repository.rs @@ -17,11 +17,6 @@ pub struct Repository { pub ssh_passphrase: Option, } -// pub fn generate_path_from_git_url(url: &String, base_path: &String) -> String { -// let (provider, user, repo) = generate_credentials_from_clone_url(url); -// return format!("{}/{}/{}/{}", base_path.trim_end_matches("/"), provider, user, repo); -// } - pub fn generate_credentials_from_clone_url(url: &String) -> (String, String, String) { let caps = PATH_FROM_URL_REGEX.captures(url).unwrap(); return (caps.get(2).map_or("provider".to_string(), |m| m.as_str().to_string()), diff --git a/src/gtm/gtm.rs b/src/gtm/gtm.rs index 78964d8..1c5e3ab 100644 --- a/src/gtm/gtm.rs +++ b/src/gtm/gtm.rs @@ -14,27 +14,27 @@ lazy_static! { #[derive(Serialize)] pub struct Commit { pub hash: String, - branch: String, - author: String, - message: String, - time: i64, - files: Vec, + pub branch: String, + pub author: String, + pub message: String, + pub time: i64, + pub files: Vec, } #[derive(Serialize)] pub struct File { - path: String, - time_total: i64, - timeline: Vec, - status: String, - added_lines: i32, - deleted_lines: i32, + pub path: String, + pub time_total: i64, + pub timeline: Vec, + pub status: String, + pub added_lines: i32, + pub deleted_lines: i32, } #[derive(Serialize, Copy, Clone)] pub struct TimelineEntry { - timestamp: i64, - time: i64, + pub timestamp: i64, + pub time: i64, } pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note], git_branch: String) -> Result { From 768eb1289245c665c3078eb38ba1f7531c4c3f05 Mon Sep 17 00:00:00 2001 From: tavo Date: Fri, 22 Jan 2021 20:50:20 +0200 Subject: [PATCH 32/33] Alpha launch --- src/config/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.rs b/src/config/config.rs index 70ff74f..bcb4f92 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use crate::config::repository::{generate_credentials_from_clone_url, Repository}; lazy_static! { - pub static ref CONFIG_PATH: String = "./example_config.toml".to_string(); + pub static ref CONFIG_PATH: String = "./config.toml".to_string(); } From 2147845c57bbc31e4b9f3e5f1c10693af3539338 Mon Sep 17 00:00:00 2001 From: Tavo Annus Date: Mon, 23 Nov 2020 21:58:59 +0200 Subject: [PATCH 33/33] SYNC-1 Implement git clone for sync (#1) * Add initial version of config * Something works with fetch * Add fetching notes * Check if repo already exists * Correctly parse notes time * Add finding diff for commits * Add ssh authorization * Add modification types --- src/git/git.rs | 102 ++++++++++++++++++++++++++++++ src/git/gtm.rs | 166 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 src/git/git.rs create mode 100644 src/git/gtm.rs diff --git a/src/git/git.rs b/src/git/git.rs new file mode 100644 index 0000000..a638a1e --- /dev/null +++ b/src/git/git.rs @@ -0,0 +1,102 @@ +#![deny(warnings)] + +use std::fs; +use std::path::Path; + +use git2::{Commit, Error, FetchOptions, Note, RemoteCallbacks, Repository}; +use git2::build::RepoBuilder; +use crate::config; + +mod gtm; + +static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; +static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; +static DEFAULT_ORIGIN: &str = "origin"; + +pub fn clone_or_open(repo_config: &config::Repository) -> Result { + let path = Path::new(&repo_config.path); + + if path.exists() { + let repo = Repository::open(path); + if repo.is_ok() { + return repo; + } + let _remove = fs::remove_dir_all(&path) + .expect(&*format!("Unable to remove dir: {}", repo_config.path)); + return clone_or_open(&repo_config); + } + + let fo = generate_fetch_options(repo_config); + + return RepoBuilder::new() + .fetch_options(fo) + .clone(&repo_config.url, Path::new(&repo_config.path)); +} + +fn generate_fetch_options(repo_config: &config::Repository) -> FetchOptions { + let mut cb = RemoteCallbacks::new(); + let repo_config = repo_config.clone(); + cb.credentials(move |_c, _o, t| { + if t.is_ssh_key() { + return git2::Cred::ssh_key( + &repo_config.ssh_user.as_ref().unwrap_or(&"git".to_string()), + Option::from(Path::new(&repo_config.ssh_public_key.as_ref().unwrap_or(&"".to_string()))), + &Path::new(&repo_config.ssh_private_key.as_ref().unwrap_or(&"".to_string())), + repo_config.ssh_passphrase.as_ref().map(|x| &**x), + ) + } + return git2::Cred::default(); + }); + + + let mut fo = FetchOptions::new(); + fo.remote_callbacks(cb); + return fo; +} + + +pub fn fetch(repo: &Repository, repo_config: &config::Repository) { + let mut remote = repo.find_remote(DEFAULT_ORIGIN) + .expect("Unable to find remote 'origin'"); + let mut ref_added = false; + let refs = remote.fetch_refspecs().unwrap(); + for i in 0..refs.len() { + if refs.get(i).unwrap() == GTM_NOTES_REF_SPEC { + ref_added = true; + break; + } + } + if !ref_added { + repo.remote_add_fetch(DEFAULT_ORIGIN, GTM_NOTES_REF_SPEC) + .expect("Unable to add fetch ref spec for gtm-data!"); + remote = repo.find_remote(DEFAULT_ORIGIN) + .expect("Unable to find remote 'origin'"); + } + + let mut fo = generate_fetch_options(repo_config); + remote.fetch(&[] as &[&str], Option::from(&mut fo), None) + .expect("Error fetching data!"); + remote.disconnect().unwrap(); +} + +pub fn read_commits(repo: &Repository) -> Result, Error> { + let commits : Vec = Vec::new(); + let mut revwalk = repo.revwalk().expect("Unable to revwalk!"); + let _sorting = revwalk.set_sorting(git2::Sort::TIME); + let _head = revwalk.push_head(); + for commit_oid in revwalk { + let commit_oid = commit_oid?; + let commit = repo.find_commit(commit_oid)?; + let notes: Vec = repo.notes(Option::from(GTM_NOTES_REF))? + .map(|n| n.unwrap()) + .filter(|n| n.1 == commit_oid) + .map(|n| repo.find_note(Option::from(GTM_NOTES_REF), n.1).unwrap()) + .collect(); + + let res= gtm::parse_commit(&repo, &commit, ¬es)?; + println!("{}", res); + } + // let a = repo.notes(Option::from(GTM_NOTES_REF)) + // .expect("Unable to find gtm-notes"); + return Result::Ok(commits); +} diff --git a/src/git/gtm.rs b/src/git/gtm.rs new file mode 100644 index 0000000..eb96f83 --- /dev/null +++ b/src/git/gtm.rs @@ -0,0 +1,166 @@ +use std::collections::HashMap; +use std::fmt; + +use git2::{DiffOptions, Note}; +use lazy_static::lazy_static; +use regex::Regex; +use serde::export::Formatter; + +lazy_static! { + static ref NOTE_HEADER_REGEX: Regex = Regex::new("\\[ver:\\d+,total:\\d+]").unwrap(); + static ref NOTE_HEADER_VALS_REGEX: Regex = Regex::new("\\d+").unwrap(); +} + +pub struct Commit { + hash: String, + author_email: String, + message: String, + time: i64, + files: Vec, +} + +pub struct File { + path: String, + time_total: i64, + timeline: HashMap, + status: String, + added_lines: i32, + deleted_lines: i32, +} + +pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note]) -> Result { + let mut commit = Commit { + hash: git_commit.id().to_string(), + author_email: git_commit.author().to_string(), + message: git_commit.message().unwrap().to_string(), + time: git_commit.time().seconds(), // todo: validate + files: vec![], + }; + + for note in notes { + let message = note.message().unwrap(); + let mut files = parse_note_message(message).unwrap_or(vec![]); + let _diff = diff_parents(files.as_mut(), git_commit, repo); + commit.files.append(files.as_mut()); + } + + return Ok(commit); +} + +fn parse_note_message(message: &str) -> Option> { + let mut version: String = "".to_string(); + let mut files: Vec = Vec::new(); + let lines = message.split("\n"); + for line in lines { + if line.trim() == "" { + version = "".to_string(); + } else if NOTE_HEADER_REGEX.is_match(line) { + let matches: Vec = NOTE_HEADER_VALS_REGEX.find_iter(line) + .filter_map(|d| d.as_str().parse().ok()) + .collect(); + version = matches.get(0)?.clone(); + } + + let mut file = File { + path: "".to_string(), + time_total: 0, + timeline: HashMap::new(), + status: "".to_string(), + added_lines: 0, + deleted_lines: 0, + }; + + if version == "1" { + let field_groups: Vec<&str> = line.split(",").collect(); + if field_groups.len() < 3 { + continue; + } + for i in 0..field_groups.len() { + let fields: Vec<&str> = field_groups.get(i)?.split(":").collect(); + if i == 0 && fields.len() == 2 { + file.path = fields.get(0)?.replace("->", ":"); + file.time_total = fields.get(1)?.parse().unwrap_or(0); + } else if i == field_groups.len() - 1 && fields.len() == 1 { + file.status = fields.get(0)?.to_string(); + } else if fields.len() == 2 { + let epoch_timeline: i64 = fields.get(0)?.parse().unwrap_or(0); + let epoch_total: i32 = fields.get(1)?.parse().unwrap_or(0); + file.timeline.insert(epoch_timeline, epoch_total); + } + } + } else { + continue; + } + + let mut found: bool = false; + for mut added_file in files.iter_mut() { + if added_file.path == file.path { + added_file.time_total += file.time_total; + for (epoch, secs) in &file.timeline { + added_file.timeline.insert(*epoch, *secs); + } + found = true; + } + } + if !found { + files.push(file); + } + } + return Option::from(files); +} + +fn diff_parents(files: &mut Vec, commit: &git2::Commit, repo: &git2::Repository) -> Result<(), git2::Error> { + if commit.parent_count() == 0 { + // TODO: Figure out how to handle initial commit + return Ok(()); + } + + let parent = commit.parent(0)?; + let child_tree = commit.tree()?; + let parent_tree = parent.tree()?; + + for mut file in files { + if file.path.ends_with(".app") { + continue; // Skip app events + } + let mut diff_options = DiffOptions::new(); + diff_options.pathspec(&file.path); + let diff = repo.diff_tree_to_tree( + Option::from(&parent_tree), + Option::from(&child_tree), + Option::from(&mut diff_options))?; + let diff_stats = diff.stats()?; + file.added_lines = diff_stats.insertions() as i32; + file.deleted_lines = diff_stats.deletions() as i32; + } + + return Ok(()); +} + +impl fmt::Display for Commit { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let _ = writeln!(f, "Commit: {}", self.hash); + let _ = writeln!(f, "Author: {}", self.author_email); + let _ = writeln!(f, "Time {}", self.time); + let _ = writeln!(f, "{}", self.message); + + for file in &self.files { + let _ = writeln!(f, "{}", &file); + } + Ok(()) + } +} + +impl fmt::Display for File { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{:>2}h{:>3}m{:>3}s : {:<45} +{:<4} - {:<4} {}", + self.time_total / 3600, + (self.time_total % 3600) / 60, + self.time_total % 60, + self.path, + self.added_lines, + self.deleted_lines, + self.status, + ) + } +}