Skip to content

Commit

Permalink
Adds cache warmer.
Browse files Browse the repository at this point in the history 
Adds mime type handling to format parameter.
Adds PDF output.
Handles cache only request with no response body.
Fixes downloading so it works with caching.
New not found image is better aligned.
  • Loading branch information
@langsamu
langsamu committed Sep 24, 2018
1 parent 1b330e1 commit 6cbe279
Show file tree
Hide file tree
Showing 12 changed files with 1,239 additions and 46 deletions.
2 changes: 2 additions & 0 deletions ConvertAll/ConvertAll.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
</PropertyGroup>

<ItemGroup>
<None Remove="GetPhotoUsage.query" />
<None Remove="SharePointExport.json" />
</ItemGroup>

<ItemGroup>
<EmbeddedResource Include="GetPhotoUsage.query" />
<EmbeddedResource Include="SharePointExport.json" />
</ItemGroup>

Expand Down
13 changes: 13 additions & 0 deletions ConvertAll/GetPhotoUsage.query
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
let formatEvents = customEvents
| where cloud_RoleName contains "photo"
| where isnotempty(customDimensions.["format"])
| extend format = customDimensions.["format"];
requests
| where success == true
| where url contains "/photo/"
| join formatEvents on $left.operation_Id == $right.operation_Id
| extend parameters = parseurl(url).["Query Parameters"]
| project format = tostring(format), crop = tostring(parameters.["crop"]), width = toint(parameters.["width"]), height = toint(parameters.["height"]), quality = toint(parameters.["quality"])
| summarize total = count() by format, crop, width, height, quality
| where total > 1000
| order by total desc
203 changes: 171 additions & 32 deletions ConvertAll/Program.cs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
namespace ConsoleApp1
{
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
Expand All @@ -18,46 +21,115 @@ internal class Program

private static void Main(string[] args)
{
var config = Program.Config;
Program.functionUrl = config.GetValue<string>("FunctionUrl");
Program.storage = config.GetValue<string>("Storage");
while (true)
{
var config = Program.Config;

Console.WriteLine("c: Convert");
Console.WriteLine("w: Warm");
Console.WriteLine("esc: Exit");
Console.WriteLine();

switch (Console.ReadKey(true).Key)
{
case ConsoleKey.C:
Console.WriteLine("Converting");
Console.WriteLine();

Program.functionUrl = config.GetValue<string>("FunctionUrl");
Program.storage = config.GetValue<string>("Storage");

Parallel.ForEach(Mappings, new ParallelOptions { MaxDegreeOfParallelism = 50 }, Convert);

Console.WriteLine();
Console.WriteLine("Done converting");
Console.WriteLine();

break;

case ConsoleKey.W:
Console.WriteLine("Warming cache");

var photoServiceUrl = config.GetValue<string>("PhotoServiceUrl");

Console.WriteLine("Getting usage data");

Program.Run().Wait();
var rows = QueryAI().Result;

Console.WriteLine($"Got {rows.Count()} rows of usage data");

var resources = Mappings.Where(m => !m.Value<bool>("Photo Taken Down")).Select(m => m.Value<string>("ImageResourceUri"));
var urls = resources.SelectMany(resource => rows.Select(row => $"{photoServiceUrl}{resource}{row}"));

Parallel.ForEach(urls, new ParallelOptions { MaxDegreeOfParallelism = 50 }, Warm);

Console.WriteLine();
Console.WriteLine("Done warming cache");
Console.WriteLine();

break;

case ConsoleKey.Escape:
return;

default:
Console.WriteLine("Unkown");
Console.WriteLine();

break;
}
}
}

private static async Task Run()
private static void Warm(string url, ParallelLoopState arg2, long arg3)
{
var config = Config;
var mappings = Mappings;
var functionUrl = config.GetValue<string>("FunctionUrl");
var storage = config.GetValue<string>("Storage");

Parallel.ForEach<JToken>(Mappings.Children(), new ParallelOptions { MaxDegreeOfParallelism = 50 }, X);
using (var client = new HttpClient())
{
var response = client.GetAsync(url).Result;

if (!response.IsSuccessStatusCode)
{
Console.WriteLine("{0:d} ({1})", response.StatusCode, url);
}
}
}

//for (var i = 0; i < mappings.Count; i++)
//{
// var mapping = mappings[i];
// mapping["Storage"] = storage;
private static async Task<IEnumerable<Row>> QueryAI()
{
var aiApp = "330aa411-57af-491f-b22a-1cab7d08a6a9";
var aiKey = "asi4quq3wdffokim0b2dpln8k8rya6gfl5n5g62g";
var aiPeriod = "100"; // days

// using (var client = new HttpClient())
// {
// Console.WriteLine("{0} / {1}", i, mappings.Count);
// Console.WriteLine(mapping["PhotoID"]);
// Console.WriteLine(mapping["ImageResourceUri"]);
var period = System.Xml.XmlConvert.ToString(TimeSpan.Parse(aiPeriod));
var aiApi = $"https://api.applicationinsights.io/v1/apps/{aiApp}/query?timespan={period}";
var query = JsonConvert.SerializeObject(new { query = PhotoUsageQuery });

// var timer = Stopwatch.StartNew();
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("x-api-key", aiKey);

// var response = await client.PostAsync(functionUrl, new StringContent(mapping.ToString(), Encoding.UTF8, "application/json"));
using (var content = new StringContent(query))
{
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");

// Console.WriteLine(response.StatusCode);
// Console.WriteLine(timer.Elapsed);
// Console.WriteLine();
// }
//}
using (var response = await client.PostAsync(aiApi, content))
{
using (var stream = await response.Content.ReadAsStreamAsync())
{
using (var reader = new StreamReader(stream))
{
using (var json = new JsonTextReader(reader))
{
return JObject.Load(json)["tables"][0]["rows"].Select(r => new Row(r));
}
}
}
}
}
}
}

private static void X(JToken mapping, ParallelLoopState arg2, long arg3)
private static void Convert(JToken mapping, ParallelLoopState arg2, long arg3)
{
mapping["Storage"] = Program.storage;

Expand All @@ -71,7 +143,6 @@ private static void X(JToken mapping, ParallelLoopState arg2, long arg3)

Console.WriteLine("Finish {0} ({1}) with {2} in {3}", mapping["PhotoID"], arg3, response.StatusCode, timer.Elapsed);
}

}

private static IConfigurationRoot Config
Expand All @@ -86,21 +157,89 @@ private static IConfigurationRoot Config
}
}

private static JArray Mappings
private static JEnumerable<JToken> Mappings
{
get
{
using (var stream = Assembly.GetEntryAssembly().GetManifestResourceStream("ConsoleApp1.SharePointExport.json"))
using (var stream = Assembly.GetEntryAssembly().GetManifestResourceStream("ConvertAll.SharePointExport.json"))
{
using (var reader = new StreamReader(stream))
{
using (var jsonReader = new JsonTextReader(reader))
{
return JArray.Load(jsonReader);
return JArray.Load(jsonReader).Children();
}
}
}
}
}

private static string PhotoUsageQuery
{
get
{
using (var stream = Assembly.GetEntryAssembly().GetManifestResourceStream("ConvertAll.GetPhotoUsage.query"))
{
using (var reader = new StreamReader(stream))
{
return reader.ReadToEnd();
}
}
}
}
}

internal class Row
{
private readonly JToken original;

public Row(JToken original)
{
this.original = original;
}

public string Format => this.original.Value<string>(0);

public string Crop => this.original.Value<string>(1);

public int? Width => this.original.Value<int?>(2);

public int? Height => this.original.Value<int?>(3);

public int? Quality => this.original.Value<int?>(4);

public override string ToString()
{
var parameters = new List<string>();

if (!string.IsNullOrEmpty(this.Format))
{
parameters.Add($"format={this.Format}");
}

if (!string.IsNullOrEmpty(this.Crop))
{
parameters.Add($"crop={this.Crop}");
}

if (!(this.Width is null))
{
parameters.Add($"width={this.Width}");
}

if (!(this.Height is null))
{
parameters.Add($"height={this.Height}");
}

if (!(this.Quality is null))
{
parameters.Add($"quality={this.Quality}");
}

parameters.Add("cacheonly=true");

return $"?{string.Join("&", parameters)}";
}
}
}
Loading

0 comments on commit 6cbe279

Please sign in to comment.