Skip to content

Commit

Permalink
- GitHub: Removed paging WIP and add warning
Browse files Browse the repository at this point in the history
  • Loading branch information
KoalaBear84 committed Aug 8, 2022
1 parent b64c8e4 commit 90dc2aa
Showing 1 changed file with 34 additions and 36 deletions.
70 changes: 34 additions & 36 deletions src/OpenDirectoryDownloader/Site/GitHub/GitHubParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -142,55 +142,53 @@ private static async Task<WebDirectory> ScanAsync(HttpClient httpClient, WebDire

try
{
// TODO: Add paging if needed
bool hasNextPage = false;
// There is NO paging available. Probably also not needed, but still..

do
string sha = CurrentCommitSha;

if (webDirectory.Uri.Segments.Length == 7)
{
//https://api.github.com/repos/KoalaBear84/OpenDirectoryDownloader/git/trees/39a621f7664d439e0617256f0364483c86646d4f
sha = webDirectory.Uri.Segments.Last();
}

string url = $"{GetApiUrl(Owner, Repository)}/git/trees/{CurrentCommitSha}";
// Setting recursive to any value returns more than current limit (100.000)
string url = $"{GetApiUrl(Owner, Repository)}/git/trees/{sha}?recursive=true";

if (webDirectory.Uri.Segments.Length == 7)
{
string treeSha = webDirectory.Uri.Segments.Last();
url = $"{GetApiUrl(Owner, Repository)}/git/trees/{treeSha}";
}
HttpResponseMessage httpResponseMessage = await DoRequest(httpClient, url);

HttpResponseMessage httpResponseMessage = await DoRequest(httpClient, url);
httpResponseMessage.EnsureSuccessStatusCode();

httpResponseMessage.EnsureSuccessStatusCode();
string json = await httpResponseMessage.Content.ReadAsStringAsync();

string json = await httpResponseMessage.Content.ReadAsStringAsync();
GitHubResult gitHubResult = GitHubResult.FromJson(json);

GitHubResult gitHubResult = GitHubResult.FromJson(json);
Logger.Warn($"GitHub response is truncated with {gitHubResult.Tree.Length} items, sadly there is no paging available..");

foreach (Tree treeItem in gitHubResult.Tree)
foreach (Tree treeItem in gitHubResult.Tree)
{
// Like directories
if (treeItem.Type == "tree")
{
// Like directories
if (treeItem.Type == "tree")
webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
{
webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
{
Parser = Parser,
// Use real URL
Url = treeItem.Url,
Name = treeItem.Path
});
}
else
// Like files
Parser = Parser,
// Use real URL
Url = treeItem.Url,
Name = treeItem.Path
});
}
else
// Like files
{
webDirectory.Files.Add(new WebFile
{
webDirectory.Files.Add(new WebFile
{
// Use real URL
Url = treeItem.Url,
FileName = treeItem.Path,
FileSize = treeItem.Size
});
}
// Use real URL
Url = treeItem.Url,
FileName = treeItem.Path,
FileSize = treeItem.Size
});
}
} while (hasNextPage);
}
}
catch (Exception ex)
{
Expand Down

0 comments on commit 90dc2aa

Please sign in to comment.