Skip to content

Commit

Permalink
Merge pull request #50 from semdiffdotnet/Reduce-GitHub-Overhead
Browse files Browse the repository at this point in the history 
Reduce GitHub and local file overhead
  • Loading branch information
@CodyRay
CodyRay committed Mar 8, 2016
2 parents 693cd22 + 4dd37ca commit a2ad87e
Show file tree
Hide file tree
Showing 4 changed files with 171 additions and 5 deletions.
68 changes: 66 additions & 2 deletions SemDiff.Core/GitHub.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public GitHub(string repoOwner, string repoName, string authUsername = null, str
AuthToken = authToken;
Client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{AuthUsername}:{AuthToken}")));
}
GetCurrentSaved();
}

public string AuthToken { get; set; }
Expand All @@ -52,7 +53,33 @@ public GitHub(string repoOwner, string repoName, string authUsername = null, str
public int RequestsLimit { get; private set; }
public HttpClient Client { get; private set; }
public string RepoFolder { get; set; }
public IList<PullRequest> CurrentSaved { get; set; }
public string EtagNoChanges { get; set; }
public string JsonFileName { get; } = "LocalList.json";

internal void GetCurrentSaved()
{
try
{
var path = RepoFolder.Replace('/', Path.DirectorySeparatorChar);
path = Path.Combine(path, JsonFileName);
var json = File.ReadAllText(path);
CurrentSaved = JsonConvert.DeserializeObject<IList<PullRequest>>(json);
}
catch(Exception ex)
{
Logger.Error($"{ex.GetType().Name}: Couldn't load {JsonFileName} because {ex.Message}");
}
}

public void UpdateLocalSavedList()
{
var path = RepoFolder.Replace('/', Path.DirectorySeparatorChar);
path = Path.Combine(path, JsonFileName);
new FileInfo(path).Directory.Create();
File.WriteAllText(path, JsonConvert.SerializeObject(CurrentSaved));
}


/// <summary>
/// Makes a request to github to update RequestsRemaining and RequestsLimit
Expand Down Expand Up @@ -143,6 +170,22 @@ private static string ParseNextLink(IEnumerable<string> links)
return null;
}

private void DeletePRsFromDisk(IEnumerable<PullRequest> prs)
{
foreach (var pr in prs)
{
var dir = Path.Combine(RepoFolder, $"{pr.Number}");
try
{
Directory.Delete(dir, true);
}
catch(Exception ex)
{
Logger.Error($"{ex.GetType().Name}: Couldn't load {JsonFileName} because {ex.Message}");
}
}
}

/// <summary>
/// Gets each page of the pull request list from GitHub.
/// Once the list is complete, get all the pull request files for each pull request.
Expand Down Expand Up @@ -173,6 +216,24 @@ public async Task<IList<PullRequest>> GetPullRequestsAsync()
{
return null;
}
if (CurrentSaved != null)
{
var removePRs = CurrentSaved;
foreach (var pr in pullRequests)
{
foreach (var prRemove in removePRs)
{
if (pr.Number == prRemove.Number)
{
pr.LastWrite = prRemove.LastWrite;
removePRs.Remove(prRemove);
break;
}
}
}
DeletePRsFromDisk(removePRs);
}
CurrentSaved = pullRequests;
return await Task.WhenAll(pullRequests.Select(async pr =>
{
var filePagination = Ref.Create<string>(null);
Expand Down Expand Up @@ -200,6 +261,8 @@ public async Task DownloadFilesAsync(PullRequest pr)
{
foreach (var current in pr.Files)
{
if (pr.LastWrite >= pr.Updated)
return;
var csFileTokens = current.Filename.Split('.');
if (csFileTokens.Last() == "cs")
{
Expand All @@ -214,14 +277,15 @@ public async Task DownloadFilesAsync(PullRequest pr)
Logger.Info($"The mythical 'changed' status has occured! {pr.Number}:{current.Filename}");
goto case Files.StatusEnum.Modified; //Effectivly falls through to the following

case Files.StatusEnum.Modified:
case Files.StatusEnum.Modified:
var headTsk = DownloadFileAsync(pr.Number, current.Filename, pr.Head.Sha);
var ancTsk = DownloadFileAsync(pr.Number, current.Filename, pr.Base.Sha, isAncestor: true);
await Task.WhenAll(headTsk, ancTsk);
break;
}
}
}
pr.LastWrite = DateTime.UtcNow;
}

private async Task DownloadFileAsync(int prNum, string path, string sha, bool isAncestor = false)
Expand Down Expand Up @@ -267,7 +331,7 @@ public class PullRequest

[JsonProperty("updated_at")]
public DateTime Updated { get; set; }

public DateTime LastWrite { get; set; } = DateTime.MinValue;
[JsonProperty("html_url")]
public string Url { get; set; }

Expand Down
103 changes: 100 additions & 3 deletions SemDiff.Test/PullRequestListTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json;
using SemDiff.Core;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;

Expand Down Expand Up @@ -39,7 +41,7 @@ public void PullRequestFromTestRepo()
}
var requests = github.GetPullRequestsAsync().Result;
Assert.AreEqual(5, requests.Count);
var r = requests.ElementAt(requests.Count-4);
var r = requests.ElementAt(requests.Count - 4);
if (r.Number == 4)
{
Assert.AreEqual(r.Locked, false);
Expand Down Expand Up @@ -90,10 +92,10 @@ public void FilesPagination()
}
var PRs = github.GetPullRequestsAsync().Result;
Assert.IsTrue(PRs.Count >= 5);
foreach(var pr in PRs)
foreach (var pr in PRs)
{
if (pr.Number == 5)
Assert.AreEqual(40,pr.Files.Count);
Assert.AreEqual(40, pr.Files.Count);
}
}
[TestMethod]
Expand Down Expand Up @@ -146,5 +148,100 @@ public void GetFilesFromGitHub()
}
Assert.AreEqual(fourWasFound, true);
}
[TestMethod]
public void UpdateLocalSaved()
{
github.GetPullRequestsAsync();
var path = github.RepoFolder.Replace('/', Path.DirectorySeparatorChar);
path = Path.Combine(path, github.JsonFileName);
new FileInfo(path).Directory.Create();
if (File.Exists(path))
File.Delete(path);
github.UpdateLocalSavedList();
Assert.IsTrue(File.Exists(path));
var json = File.ReadAllText(path);
var currentSaved = JsonConvert.DeserializeObject<IList<GitHub.PullRequest>>(json);
Assert.AreEqual(github.CurrentSaved.Count, currentSaved.Count);
var local = currentSaved.First();
var gPR = github.CurrentSaved.First();
Assert.AreEqual(local.Number, gPR.Number);
Assert.AreEqual(local.State, gPR.State);
Assert.AreEqual(local.Title, gPR.Title);
Assert.AreEqual(local.Locked, gPR.Locked);
Assert.AreEqual(local.Updated, gPR.Updated);
Assert.AreEqual(local.LastWrite, gPR.LastWrite);
Assert.AreEqual(local.Url, gPR.Url);
Assert.IsNotNull(local.User);
Assert.IsNotNull(local.Head);
Assert.IsNotNull(local.Base);
Assert.IsNotNull(local.Files);
}
[TestMethod]
public void RemoveUnusedLocalFiles()
{
var path = github.RepoFolder.Replace('/', Path.DirectorySeparatorChar);
path = Path.Combine(path, "0");
Directory.CreateDirectory(path);
var requests = github.GetPullRequestsAsync().Result;
var prZero = new GitHub.PullRequest
{
Number = 0,
State = requests.First().State,
Title = requests.First().Title,
Locked = requests.First().Locked,
Updated = requests.First().Updated,
LastWrite = requests.First().LastWrite,
Url = requests.First().Url,
User = requests.First().User,
Head = requests.First().Head,
Base = requests.First().Base,
Files = requests.First().Files
};
var currentSaved = github.CurrentSaved;
currentSaved.Add(prZero);
github.CurrentSaved = currentSaved;
path = github.RepoFolder.Replace('/', Path.DirectorySeparatorChar);
path = Path.Combine(path, github.JsonFileName);
new FileInfo(path).Directory.Create();
File.WriteAllText(path, JsonConvert.SerializeObject(currentSaved));
github.GetCurrentSaved();
requests = github.GetPullRequestsAsync().Result;
path = github.RepoFolder.Replace('/', Path.DirectorySeparatorChar);
path = Path.Combine(path, "0");
Assert.IsFalse(Directory.Exists(path));
}
[TestMethod]
public void LastSessionLocalFiles()
{
var requests = github.GetPullRequestsAsync().Result;
github.UpdateLocalSavedList();
var newgithub = new GitHub(owner, repository);
Assert.IsNotNull(newgithub.CurrentSaved);
}
[TestMethod]
public void NoUnnecessaryDownloading()
{
var requests = github.GetPullRequestsAsync().Result;
var path = "";
foreach (var r in requests)
{
github.DownloadFilesAsync(r).Wait();
if (r.Number == 1)
{
foreach (var files in r.Files)
{
path = files.Filename.Replace('/', Path.DirectorySeparatorChar);
}
}
}
var fileLastUpdated = File.GetLastWriteTimeUtc(path);
github.EtagNoChanges = null;
requests = github.GetPullRequestsAsync().Result;
foreach (var r in requests)
{
github.DownloadFilesAsync(r).Wait();
}
Assert.IsTrue(fileLastUpdated == File.GetLastWriteTimeUtc(path));
}
}
}
4 changes: 4 additions & 0 deletions SemDiff.Test/SemDiff.Test.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@
<HintPath>..\packages\Microsoft.CodeAnalysis.Workspaces.Common.1.1.1\lib\net45\Microsoft.CodeAnalysis.Workspaces.Desktop.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Newtonsoft.Json, Version=8.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.8.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Collections.Immutable, Version=1.1.37.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Collections.Immutable.1.1.37\lib\dotnet\System.Collections.Immutable.dll</HintPath>
Expand Down
1 change: 1 addition & 0 deletions SemDiff.Test/packages.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
<package id="Microsoft.CodeAnalysis.Workspaces.Common" version="1.1.1" targetFramework="net461" />
<package id="Microsoft.Composition" version="1.0.30" targetFramework="net461" />
<package id="Microsoft.Net.Http" version="2.2.29" targetFramework="net461" />
<package id="Newtonsoft.Json" version="8.0.2" targetFramework="net461" />
<package id="System.Collections" version="4.0.10" targetFramework="net461" />
<package id="System.Collections.Immutable" version="1.1.37" targetFramework="net461" />
<package id="System.Diagnostics.Debug" version="4.0.10" targetFramework="net461" />
Expand Down

0 comments on commit a2ad87e

Please sign in to comment.