From 83a08ac7450910dc0be800e428b983c23d1ebf6d Mon Sep 17 00:00:00 2001 From: Tavo Annus Date: Mon, 23 Nov 2020 21:58:59 +0200 Subject: [PATCH] SYNC-1 Implement git clone for sync (#1) * Add initial version of config * Something works with fetch * Add fetching notes * Check if repo already exists * Correctly parse notes time * Add finding diff for commits * Add ssh authorization * Add modification types --- Cargo.toml | 8 ++- src/config/config.rs | 32 +++++++++ src/git/git.rs | 102 ++++++++++++++++++++++++++ src/git/gtm.rs | 166 +++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 12 +++- 5 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 src/config/config.rs create mode 100644 src/git/git.rs create mode 100644 src/git/gtm.rs diff --git a/Cargo.toml b/Cargo.toml index 0a3e4f6..e3452f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,4 +7,10 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -rocket = "0.4.6" \ No newline at end of file +rocket = "0.4.6" +git2 = "0.13" +serde = "1.0" +serde_derive = "1.0" +toml = "0.5.7" +regex = "1" +lazy_static = "1.4.0" \ No newline at end of file diff --git a/src/config/config.rs b/src/config/config.rs new file mode 100644 index 0000000..8e0cab4 --- /dev/null +++ b/src/config/config.rs @@ -0,0 +1,32 @@ + +use serde_derive::{Serialize, Deserialize}; +use std::fs; + +#[derive(Serialize, Deserialize)] +pub struct SyncConfig { + pub target_host: String, + pub target_port: Option, + pub port: Option, + pub repositories_base_path: String, + pub repositories: Vec, +} + +#[derive(Serialize, Deserialize)] +pub struct Repository { + pub url: String, + pub path: String, + pub ssh_private_key: Option, + pub ssh_public_key: Option, + pub ssh_user: Option, + pub ssh_passphrase: Option, +} + +pub fn load(config_file: &String) -> SyncConfig { + let content = fs::read_to_string(config_file).expect("Unable to read config!"); + return toml::from_str(&content).expect("Unable to deserialize config!"); +} + +pub fn save(config_file: &String, config: &SyncConfig) { + let content = toml::to_string(config).expect("Unable to serialize config!"); + fs::write(config_file, content).expect("Unable to save config!"); +} \ No newline at end of file diff --git a/src/git/git.rs b/src/git/git.rs new file mode 100644 index 0000000..a638a1e --- /dev/null +++ b/src/git/git.rs @@ -0,0 +1,102 @@ +#![deny(warnings)] + +use std::fs; +use std::path::Path; + +use git2::{Commit, Error, FetchOptions, Note, RemoteCallbacks, Repository}; +use git2::build::RepoBuilder; +use crate::config; + +mod gtm; + +static GTM_NOTES_REF: &str = "refs/notes/gtm-data"; +static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data"; +static DEFAULT_ORIGIN: &str = "origin"; + +pub fn clone_or_open(repo_config: &config::Repository) -> Result { + let path = Path::new(&repo_config.path); + + if path.exists() { + let repo = Repository::open(path); + if repo.is_ok() { + return repo; + } + let _remove = fs::remove_dir_all(&path) + .expect(&*format!("Unable to remove dir: {}", repo_config.path)); + return clone_or_open(&repo_config); + } + + let fo = generate_fetch_options(repo_config); + + return RepoBuilder::new() + .fetch_options(fo) + .clone(&repo_config.url, Path::new(&repo_config.path)); +} + +fn generate_fetch_options(repo_config: &config::Repository) -> FetchOptions { + let mut cb = RemoteCallbacks::new(); + let repo_config = repo_config.clone(); + cb.credentials(move |_c, _o, t| { + if t.is_ssh_key() { + return git2::Cred::ssh_key( + &repo_config.ssh_user.as_ref().unwrap_or(&"git".to_string()), + Option::from(Path::new(&repo_config.ssh_public_key.as_ref().unwrap_or(&"".to_string()))), + &Path::new(&repo_config.ssh_private_key.as_ref().unwrap_or(&"".to_string())), + repo_config.ssh_passphrase.as_ref().map(|x| &**x), + ) + } + return git2::Cred::default(); + }); + + + let mut fo = FetchOptions::new(); + fo.remote_callbacks(cb); + return fo; +} + + +pub fn fetch(repo: &Repository, repo_config: &config::Repository) { + let mut remote = repo.find_remote(DEFAULT_ORIGIN) + .expect("Unable to find remote 'origin'"); + let mut ref_added = false; + let refs = remote.fetch_refspecs().unwrap(); + for i in 0..refs.len() { + if refs.get(i).unwrap() == GTM_NOTES_REF_SPEC { + ref_added = true; + break; + } + } + if !ref_added { + repo.remote_add_fetch(DEFAULT_ORIGIN, GTM_NOTES_REF_SPEC) + .expect("Unable to add fetch ref spec for gtm-data!"); + remote = repo.find_remote(DEFAULT_ORIGIN) + .expect("Unable to find remote 'origin'"); + } + + let mut fo = generate_fetch_options(repo_config); + remote.fetch(&[] as &[&str], Option::from(&mut fo), None) + .expect("Error fetching data!"); + remote.disconnect().unwrap(); +} + +pub fn read_commits(repo: &Repository) -> Result, Error> { + let commits : Vec = Vec::new(); + let mut revwalk = repo.revwalk().expect("Unable to revwalk!"); + let _sorting = revwalk.set_sorting(git2::Sort::TIME); + let _head = revwalk.push_head(); + for commit_oid in revwalk { + let commit_oid = commit_oid?; + let commit = repo.find_commit(commit_oid)?; + let notes: Vec = repo.notes(Option::from(GTM_NOTES_REF))? + .map(|n| n.unwrap()) + .filter(|n| n.1 == commit_oid) + .map(|n| repo.find_note(Option::from(GTM_NOTES_REF), n.1).unwrap()) + .collect(); + + let res= gtm::parse_commit(&repo, &commit, ¬es)?; + println!("{}", res); + } + // let a = repo.notes(Option::from(GTM_NOTES_REF)) + // .expect("Unable to find gtm-notes"); + return Result::Ok(commits); +} diff --git a/src/git/gtm.rs b/src/git/gtm.rs new file mode 100644 index 0000000..eb96f83 --- /dev/null +++ b/src/git/gtm.rs @@ -0,0 +1,166 @@ +use std::collections::HashMap; +use std::fmt; + +use git2::{DiffOptions, Note}; +use lazy_static::lazy_static; +use regex::Regex; +use serde::export::Formatter; + +lazy_static! { + static ref NOTE_HEADER_REGEX: Regex = Regex::new("\\[ver:\\d+,total:\\d+]").unwrap(); + static ref NOTE_HEADER_VALS_REGEX: Regex = Regex::new("\\d+").unwrap(); +} + +pub struct Commit { + hash: String, + author_email: String, + message: String, + time: i64, + files: Vec, +} + +pub struct File { + path: String, + time_total: i64, + timeline: HashMap, + status: String, + added_lines: i32, + deleted_lines: i32, +} + +pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note]) -> Result { + let mut commit = Commit { + hash: git_commit.id().to_string(), + author_email: git_commit.author().to_string(), + message: git_commit.message().unwrap().to_string(), + time: git_commit.time().seconds(), // todo: validate + files: vec![], + }; + + for note in notes { + let message = note.message().unwrap(); + let mut files = parse_note_message(message).unwrap_or(vec![]); + let _diff = diff_parents(files.as_mut(), git_commit, repo); + commit.files.append(files.as_mut()); + } + + return Ok(commit); +} + +fn parse_note_message(message: &str) -> Option> { + let mut version: String = "".to_string(); + let mut files: Vec = Vec::new(); + let lines = message.split("\n"); + for line in lines { + if line.trim() == "" { + version = "".to_string(); + } else if NOTE_HEADER_REGEX.is_match(line) { + let matches: Vec = NOTE_HEADER_VALS_REGEX.find_iter(line) + .filter_map(|d| d.as_str().parse().ok()) + .collect(); + version = matches.get(0)?.clone(); + } + + let mut file = File { + path: "".to_string(), + time_total: 0, + timeline: HashMap::new(), + status: "".to_string(), + added_lines: 0, + deleted_lines: 0, + }; + + if version == "1" { + let field_groups: Vec<&str> = line.split(",").collect(); + if field_groups.len() < 3 { + continue; + } + for i in 0..field_groups.len() { + let fields: Vec<&str> = field_groups.get(i)?.split(":").collect(); + if i == 0 && fields.len() == 2 { + file.path = fields.get(0)?.replace("->", ":"); + file.time_total = fields.get(1)?.parse().unwrap_or(0); + } else if i == field_groups.len() - 1 && fields.len() == 1 { + file.status = fields.get(0)?.to_string(); + } else if fields.len() == 2 { + let epoch_timeline: i64 = fields.get(0)?.parse().unwrap_or(0); + let epoch_total: i32 = fields.get(1)?.parse().unwrap_or(0); + file.timeline.insert(epoch_timeline, epoch_total); + } + } + } else { + continue; + } + + let mut found: bool = false; + for mut added_file in files.iter_mut() { + if added_file.path == file.path { + added_file.time_total += file.time_total; + for (epoch, secs) in &file.timeline { + added_file.timeline.insert(*epoch, *secs); + } + found = true; + } + } + if !found { + files.push(file); + } + } + return Option::from(files); +} + +fn diff_parents(files: &mut Vec, commit: &git2::Commit, repo: &git2::Repository) -> Result<(), git2::Error> { + if commit.parent_count() == 0 { + // TODO: Figure out how to handle initial commit + return Ok(()); + } + + let parent = commit.parent(0)?; + let child_tree = commit.tree()?; + let parent_tree = parent.tree()?; + + for mut file in files { + if file.path.ends_with(".app") { + continue; // Skip app events + } + let mut diff_options = DiffOptions::new(); + diff_options.pathspec(&file.path); + let diff = repo.diff_tree_to_tree( + Option::from(&parent_tree), + Option::from(&child_tree), + Option::from(&mut diff_options))?; + let diff_stats = diff.stats()?; + file.added_lines = diff_stats.insertions() as i32; + file.deleted_lines = diff_stats.deletions() as i32; + } + + return Ok(()); +} + +impl fmt::Display for Commit { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let _ = writeln!(f, "Commit: {}", self.hash); + let _ = writeln!(f, "Author: {}", self.author_email); + let _ = writeln!(f, "Time {}", self.time); + let _ = writeln!(f, "{}", self.message); + + for file in &self.files { + let _ = writeln!(f, "{}", &file); + } + Ok(()) + } +} + +impl fmt::Display for File { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{:>2}h{:>3}m{:>3}s : {:<45} +{:<4} - {:<4} {}", + self.time_total / 3600, + (self.time_total % 3600) / 60, + self.time_total % 60, + self.path, + self.added_lines, + self.deleted_lines, + self.status, + ) + } +} diff --git a/src/main.rs b/src/main.rs index 7f88e41..2bc5911 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,11 +2,21 @@ #[macro_use] extern crate rocket; +#[path = "git/git.rs"] mod git; +#[path = "config/config.rs"] mod config; + #[get("/")] fn index() -> &'static str { "Hello, world!" } fn main() { - rocket::ignite().mount("/", routes![index]).launch(); + let loc = "./example_config.toml".to_string(); + let loaded_cfg = config::load(&loc); + println!("{}", &loaded_cfg.target_host); + let repo_to_clone = loaded_cfg.repositories.get(0).unwrap(); + let repo = git::clone_or_open(&repo_to_clone).unwrap(); + let _res = git::fetch(&repo, &repo_to_clone); + let _commits = git::read_commits(&repo); + //rocket::ignite().mount("/", routes![index]).launch(); } \ No newline at end of file