SYNC-1 Implement git clone for sync (#1)

* Add initial version of config

* Something works with fetch

* Add fetching notes

* Check if repo already exists

* Correctly parse notes time

* Add finding diff for commits

* Add ssh authorization

* Add modification types

Cargo.toml

@@ -7,4 +7,10 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rocket = "0.4.6"
+rocket = "0.4.6"
+git2 = "0.13"
+serde = "1.0"
+serde_derive = "1.0"
+toml = "0.5.7"
+regex = "1"
+lazy_static = "1.4.0"

src/config/config.rs

@@ -0,0 +1,32 @@
+
+use serde_derive::{Serialize, Deserialize};
+use std::fs;
+
+#[derive(Serialize, Deserialize)]
+pub struct SyncConfig {
+    pub target_host: String,
+    pub target_port: Option<u16>,
+    pub port: Option<u16>,
+    pub repositories_base_path: String,
+    pub repositories: Vec<Repository>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Repository {
+    pub url: String,
+    pub path: String,
+    pub ssh_private_key: Option<String>,
+    pub ssh_public_key: Option<String>,
+    pub ssh_user: Option<String>,
+    pub ssh_passphrase: Option<String>,
+}
+
+pub fn load(config_file: &String) -> SyncConfig {
+    let content = fs::read_to_string(config_file).expect("Unable to read config!");
+    return toml::from_str(&content).expect("Unable to deserialize config!");
+}
+
+pub fn save(config_file: &String, config: &SyncConfig) {
+    let content = toml::to_string(config).expect("Unable to serialize config!");
+    fs::write(config_file, content).expect("Unable to save config!");
+}

src/git/git.rs

@@ -0,0 +1,102 @@
+#![deny(warnings)]
+
+use std::fs;
+use std::path::Path;
+
+use git2::{Commit, Error, FetchOptions, Note, RemoteCallbacks, Repository};
+use git2::build::RepoBuilder;
+use crate::config;
+
+mod gtm;
+
+static GTM_NOTES_REF: &str = "refs/notes/gtm-data";
+static GTM_NOTES_REF_SPEC: &str = "+refs/notes/gtm-data:refs/notes/gtm-data";
+static DEFAULT_ORIGIN: &str = "origin";
+
+pub fn clone_or_open(repo_config: &config::Repository) -> Result<Repository, Error> {
+    let path = Path::new(&repo_config.path);
+
+    if path.exists() {
+        let repo = Repository::open(path);
+        if repo.is_ok() {
+            return repo;
+        }
+        let _remove = fs::remove_dir_all(&path)
+            .expect(&*format!("Unable to remove dir: {}", repo_config.path));
+        return clone_or_open(&repo_config);
+    }
+
+    let fo = generate_fetch_options(repo_config);
+
+    return RepoBuilder::new()
+        .fetch_options(fo)
+        .clone(&repo_config.url, Path::new(&repo_config.path));
+}
+
+fn generate_fetch_options(repo_config: &config::Repository) -> FetchOptions {
+    let mut cb = RemoteCallbacks::new();
+    let repo_config = repo_config.clone();
+    cb.credentials(move |_c, _o, t| {
+        if t.is_ssh_key() {
+            return git2::Cred::ssh_key(
+                &repo_config.ssh_user.as_ref().unwrap_or(&"git".to_string()),
+                Option::from(Path::new(&repo_config.ssh_public_key.as_ref().unwrap_or(&"".to_string()))),
+                &Path::new(&repo_config.ssh_private_key.as_ref().unwrap_or(&"".to_string())),
+                repo_config.ssh_passphrase.as_ref().map(|x| &**x),
+            )
+        }
+        return git2::Cred::default();
+    });
+
+
+    let mut fo = FetchOptions::new();
+    fo.remote_callbacks(cb);
+    return fo;
+}
+
+
+pub fn fetch(repo: &Repository, repo_config: &config::Repository) {
+    let mut remote = repo.find_remote(DEFAULT_ORIGIN)
+        .expect("Unable to find remote 'origin'");
+    let mut ref_added = false;
+    let refs = remote.fetch_refspecs().unwrap();
+    for i in 0..refs.len() {
+        if refs.get(i).unwrap() == GTM_NOTES_REF_SPEC {
+            ref_added = true;
+            break;
+        }
+    }
+    if !ref_added {
+        repo.remote_add_fetch(DEFAULT_ORIGIN, GTM_NOTES_REF_SPEC)
+            .expect("Unable to add fetch ref spec for gtm-data!");
+        remote = repo.find_remote(DEFAULT_ORIGIN)
+            .expect("Unable to find remote 'origin'");
+    }
+
+    let mut fo = generate_fetch_options(repo_config);
+    remote.fetch(&[] as &[&str], Option::from(&mut fo), None)
+        .expect("Error fetching data!");
+    remote.disconnect().unwrap();
+}
+
+pub fn read_commits(repo: &Repository) -> Result<Vec<Commit>, Error> {
+    let commits : Vec<Commit> = Vec::new();
+    let mut revwalk = repo.revwalk().expect("Unable to revwalk!");
+    let _sorting = revwalk.set_sorting(git2::Sort::TIME);
+    let _head = revwalk.push_head();
+    for commit_oid in revwalk {
+        let commit_oid = commit_oid?;
+        let commit = repo.find_commit(commit_oid)?;
+        let notes: Vec<Note> = repo.notes(Option::from(GTM_NOTES_REF))?
+            .map(|n| n.unwrap())
+            .filter(|n| n.1 == commit_oid)
+            .map(|n| repo.find_note(Option::from(GTM_NOTES_REF), n.1).unwrap())
+            .collect();
+
+        let res=  gtm::parse_commit(&repo, &commit, &notes)?;
+        println!("{}", res);
+    }
+    // let a = repo.notes(Option::from(GTM_NOTES_REF))
+    //     .expect("Unable to find gtm-notes");
+    return Result::Ok(commits);
+}

src/git/gtm.rs

@@ -0,0 +1,166 @@
+use std::collections::HashMap;
+use std::fmt;
+
+use git2::{DiffOptions, Note};
+use lazy_static::lazy_static;
+use regex::Regex;
+use serde::export::Formatter;
+
+lazy_static! {
+    static ref NOTE_HEADER_REGEX: Regex = Regex::new("\\[ver:\\d+,total:\\d+]").unwrap();
+    static ref NOTE_HEADER_VALS_REGEX: Regex = Regex::new("\\d+").unwrap();
+}
+
+pub struct Commit {
+    hash: String,
+    author_email: String,
+    message: String,
+    time: i64,
+    files: Vec<File>,
+}
+
+pub struct File {
+    path: String,
+    time_total: i64,
+    timeline: HashMap<i64, i32>,
+    status: String,
+    added_lines: i32,
+    deleted_lines: i32,
+}
+
+pub fn parse_commit(repo: &git2::Repository, git_commit: &git2::Commit, notes: &[Note]) -> Result<Commit, git2::Error> {
+    let mut commit = Commit {
+        hash: git_commit.id().to_string(),
+        author_email: git_commit.author().to_string(),
+        message: git_commit.message().unwrap().to_string(),
+        time: git_commit.time().seconds(), // todo: validate
+        files: vec![],
+    };
+
+    for note in notes {
+        let message = note.message().unwrap();
+        let mut files = parse_note_message(message).unwrap_or(vec![]);
+        let _diff = diff_parents(files.as_mut(), git_commit, repo);
+        commit.files.append(files.as_mut());
+    }
+
+    return Ok(commit);
+}
+
+fn parse_note_message(message: &str) -> Option<Vec<File>> {
+    let mut version: String = "".to_string();
+    let mut files: Vec<File> = Vec::new();
+    let lines = message.split("\n");
+    for line in lines {
+        if line.trim() == "" {
+            version = "".to_string();
+        } else if NOTE_HEADER_REGEX.is_match(line) {
+            let matches: Vec<String> = NOTE_HEADER_VALS_REGEX.find_iter(line)
+                .filter_map(|d| d.as_str().parse().ok())
+                .collect();
+            version = matches.get(0)?.clone();
+        }
+
+        let mut file = File {
+            path: "".to_string(),
+            time_total: 0,
+            timeline: HashMap::new(),
+            status: "".to_string(),
+            added_lines: 0,
+            deleted_lines: 0,
+        };
+
+        if version == "1" {
+            let field_groups: Vec<&str> = line.split(",").collect();
+            if field_groups.len() < 3 {
+                continue;
+            }
+            for i in 0..field_groups.len() {
+                let fields: Vec<&str> = field_groups.get(i)?.split(":").collect();
+                if i == 0 && fields.len() == 2 {
+                    file.path = fields.get(0)?.replace("->", ":");
+                    file.time_total = fields.get(1)?.parse().unwrap_or(0);
+                } else if i == field_groups.len() - 1 && fields.len() == 1 {
+                    file.status = fields.get(0)?.to_string();
+                } else if fields.len() == 2 {
+                    let epoch_timeline: i64 = fields.get(0)?.parse().unwrap_or(0);
+                    let epoch_total: i32 = fields.get(1)?.parse().unwrap_or(0);
+                    file.timeline.insert(epoch_timeline, epoch_total);
+                }
+            }
+        } else {
+            continue;
+        }
+
+        let mut found: bool = false;
+        for mut added_file in files.iter_mut() {
+            if added_file.path == file.path {
+                added_file.time_total += file.time_total;
+                for (epoch, secs) in &file.timeline {
+                    added_file.timeline.insert(*epoch, *secs);
+                }
+                found = true;
+            }
+        }
+        if !found {
+            files.push(file);
+        }
+    }
+    return Option::from(files);
+}
+
+fn diff_parents(files: &mut Vec<File>, commit: &git2::Commit, repo: &git2::Repository) -> Result<(), git2::Error> {
+    if commit.parent_count() == 0 {
+        // TODO: Figure out how to handle initial commit
+        return Ok(());
+    }
+
+    let parent = commit.parent(0)?;
+    let child_tree = commit.tree()?;
+    let parent_tree = parent.tree()?;
+
+    for mut file in files {
+        if file.path.ends_with(".app") {
+            continue; // Skip app events
+        }
+        let mut diff_options = DiffOptions::new();
+        diff_options.pathspec(&file.path);
+        let diff = repo.diff_tree_to_tree(
+            Option::from(&parent_tree),
+            Option::from(&child_tree),
+            Option::from(&mut diff_options))?;
+        let diff_stats = diff.stats()?;
+        file.added_lines = diff_stats.insertions() as i32;
+        file.deleted_lines = diff_stats.deletions() as i32;
+    }
+
+    return Ok(());
+}
+
+impl fmt::Display for Commit {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        let _ = writeln!(f, "Commit: {}", self.hash);
+        let _ = writeln!(f, "Author: {}", self.author_email);
+        let _ = writeln!(f, "Time {}", self.time);
+        let _ = writeln!(f, "{}", self.message);
+
+        for file in &self.files {
+            let _ = writeln!(f, "{}", &file);
+        }
+        Ok(())
+    }
+}
+
+impl fmt::Display for File {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "{:>2}h{:>3}m{:>3}s : {:<45} +{:<4} - {:<4} {}",
+               self.time_total / 3600,
+               (self.time_total % 3600) / 60,
+               self.time_total % 60,
+               self.path,
+               self.added_lines,
+               self.deleted_lines,
+               self.status,
+        )
+    }
+}

src/main.rs

@@ -2,11 +2,21 @@
 
 #[macro_use] extern crate rocket;
 
+#[path = "git/git.rs"] mod git;
+#[path = "config/config.rs"] mod config;
+
 #[get("/")]
 fn index() -> &'static str {
     "Hello, world!"
 }
 
 fn main() {
-    rocket::ignite().mount("/", routes![index]).launch();
+    let loc = "./example_config.toml".to_string();
+    let loaded_cfg = config::load(&loc);
+    println!("{}", &loaded_cfg.target_host);
+    let repo_to_clone = loaded_cfg.repositories.get(0).unwrap();
+    let repo = git::clone_or_open(&repo_to_clone).unwrap();
+    let _res = git::fetch(&repo, &repo_to_clone);
+    let _commits = git::read_commits(&repo);
+    //rocket::ignite().mount("/", routes![index]).launch();
 }