Skip to content

Commit 120f8ff

Browse files
meteatamelmeteatamel
committed
Added a new service Amathus.Converter to do feed transformations
1 parent 410004e commit 120f8ff

File tree

13 files changed

+381
-18
lines changed

13 files changed

+381
-18
lines changed

‎Amathus/Amathus.Common/FeedStore/CloudStorageFeedStore.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public async Task<SyndicationFeed> ReadAsync(string feedId)
6565

6666
var client = StorageClient.Create();
6767

68-
var objectName = feedId.ToLowerInvariant() + "2.xml";
68+
var objectName = feedId.ToLowerInvariant();
6969
_logger?.LogInformation($"Reading {objectName} from bucket {_bucketId}");
7070

7171
var stream = new MemoryStream();

‎Amathus/Amathus.Converter/Amathus.Converter.csproj

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
<Project Sdk="Microsoft.NET.Sdk.Web">
2+
3+
<PropertyGroup>
4+
<TargetFramework>netcoreapp3.1</TargetFramework>
5+
</PropertyGroup>
6+
7+
<PropertyGroup Condition=" '$(RunConfiguration)' == 'Amathus.Reader' ">
8+
<StartAction>Project</StartAction>
9+
<ApplicationURL>http://localhost:5000</ApplicationURL>
10+
</PropertyGroup>
11+
<PropertyGroup Condition=" '$(RunConfiguration)' == 'Amathus.Web' ">
12+
<StartAction>Project</StartAction>
13+
<ApplicationURL>http://localhost:5000</ApplicationURL>
14+
</PropertyGroup>
15+
<ItemGroup>
16+
<ProjectReference Include="..\Amathus.Common\Amathus.Common.csproj">
17+
<GlobalPropertiesToRemove></GlobalPropertiesToRemove>
18+
</ProjectReference>
19+
</ItemGroup>
20+
<ItemGroup>
21+
<PackageReference Include="Google.Cloud.Firestore" Version="1.1.0" />
22+
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" />
23+
</ItemGroup>
24+
<ItemGroup>
25+
<Content Include="..\Shared\amathussources.json">
26+
<Link>amathussources.json</Link>
27+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
28+
</Content>
29+
</ItemGroup>
30+
</Project>

‎Amathus/Amathus.Converter/Controllers/ConvertController.cs

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Copyright 2020 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
using System.Collections.Generic;
15+
using System.Diagnostics;
16+
using System.IO;
17+
using System.ServiceModel.Syndication;
18+
using System.Threading.Tasks;
19+
using Amathus.Common.Converter;
20+
using Amathus.Common.FeedStore;
21+
using Microsoft.AspNetCore.Mvc;
22+
using Microsoft.Extensions.Logging;
23+
using Newtonsoft.Json.Linq;
24+
25+
namespace Amathus.Converter.Controllers
26+
{
27+
[Route("")]
28+
public class ConvertController : ControllerBase
29+
{
30+
private readonly ILogger _logger;
31+
private readonly ISyndFeedStore _syndStore;
32+
private readonly IFeedConverter _converter;
33+
private readonly IFeedStore _store;
34+
35+
public ConvertController(ISyndFeedStore syndStore, IFeedConverter converter, IFeedStore store, ILogger<ConvertController> logger)
36+
{
37+
_syndStore = syndStore;
38+
_converter = converter;
39+
_store = store;
40+
_logger = logger;
41+
}
42+
43+
[HttpPost]
44+
public async Task<IActionResult> Convert()
45+
{
46+
_logger?.LogInformation("Converting feeds");
47+
var stopWatch = new Stopwatch();
48+
stopWatch.Start();
49+
50+
using (var reader = new StreamReader(HttpContext.Request.Body))
51+
{
52+
var content = await reader.ReadToEndAsync();
53+
_logger?.LogInformation("Converter received event: " + content);
54+
55+
dynamic json = JValue.Parse(content);
56+
57+
var attributes = json.message.attributes;
58+
var eventType = attributes.eventType;
59+
if (eventType != "OBJECT_FINALIZE")
60+
{
61+
return Ok();
62+
}
63+
64+
var feedId = (string)attributes.objectId;
65+
_logger.LogInformation($"Reading raw feed: {feedId}");
66+
var rawFeed = await _syndStore.ReadAsync(feedId);
67+
68+
_logger.LogInformation($"Converting raw feed: {feedId}");
69+
var feed = _converter.Convert(feedId, rawFeed);
70+
71+
_logger.LogInformation($"Inserting into backend: {feedId}");
72+
await _store.InsertAsync(feed);
73+
}
74+
75+
stopWatch.Stop();
76+
_logger?.LogInformation($"Converting feed finished in {stopWatch.Elapsed.Seconds} seconds.");
77+
78+
return Ok();
79+
}
80+
81+
private string ConstructStorageUrl(dynamic attributes)
82+
{
83+
return attributes == null ? null
84+
: string.Format("gs://{0}/{1}", attributes.bucketId, attributes.objectId);
85+
}
86+
}
87+
}

‎Amathus/Amathus.Converter/Dockerfile

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Use Microsoft's official build .NET image.
2+
# https://hub.docker.com/_/microsoft-dotnet-core-sdk/
3+
FROM mcr.microsoft.com/dotnet/core/sdk:3.1-alpine AS build
4+
WORKDIR /app
5+
6+
# Copy local code to the container image.
7+
COPY . ./
8+
9+
# Install production dependencies
10+
# Restore as distinct layers
11+
RUN dotnet restore Amathus.Common
12+
RUN dotnet restore Amathus.Converter
13+
14+
# Build a release artifact.
15+
RUN dotnet publish Amathus.Converter -c Release -o out
16+
17+
# Use Microsoft's official runtime .NET image.
18+
# https://hub.docker.com/_/microsoft-dotnet-core-aspnet/
19+
# Note: Not using 'alpine' here due to some Firebase dependency
20+
FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 AS runtime
21+
WORKDIR /app
22+
COPY --from=build /app/out .
23+
24+
# Run the web service on container startup.
25+
ENTRYPOINT ["dotnet", "Amathus.Converter.dll"]

‎Amathus/Amathus.Converter/Program.cs

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
using System;
15+
using System.IO;
16+
using Microsoft.AspNetCore.Hosting;
17+
using Microsoft.Extensions.Configuration;
18+
using Microsoft.Extensions.Hosting;
19+
20+
namespace Amathus.Converter
21+
{
22+
public class Program
23+
{
24+
public static void Main(string[] args)
25+
{
26+
CreateHostBuilder(args).Build().Run();
27+
}
28+
29+
public static IHostBuilder CreateHostBuilder(string[] args)
30+
{
31+
var port = Environment.GetEnvironmentVariable("PORT") ?? "8080";
32+
var url = string.Concat("http://0.0.0.0:", port);
33+
34+
return Host.CreateDefaultBuilder(args)
35+
.ConfigureAppConfiguration((hostingContext, config) =>
36+
{
37+
// When running locally
38+
var shared = Path.Combine(hostingContext.HostingEnvironment.ContentRootPath, "..", "Shared");
39+
config.AddJsonFile(Path.Combine(shared, "amathussources.json"), optional: true, reloadOnChange: false);
40+
41+
// When running published
42+
config.AddJsonFile("amathussources.json", optional: true, reloadOnChange: false);
43+
})
44+
.ConfigureWebHostDefaults(webBuilder =>
45+
{
46+
webBuilder.UseStartup<Startup>().UseUrls(url);
47+
});
48+
}
49+
}
50+
}

‎Amathus/Amathus.Converter/Startup.cs

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
using System.Collections.Generic;
15+
using Amathus.Common.FeedStore;
16+
using Amathus.Common.Sources;
17+
using Microsoft.AspNetCore.Builder;
18+
using Microsoft.AspNetCore.Hosting;
19+
using Microsoft.Extensions.Configuration;
20+
using Microsoft.Extensions.DependencyInjection;
21+
using Microsoft.Extensions.Hosting;
22+
using Microsoft.Extensions.Logging;
23+
using Amathus.Common.Converter;
24+
using System;
25+
26+
namespace Amathus.Converter
27+
{
28+
public class Startup
29+
{
30+
private FeedStoreBackend _backend;
31+
private List<Source> _sources;
32+
33+
public Startup(IConfiguration configuration)
34+
{
35+
Configuration = configuration;
36+
}
37+
38+
public IConfiguration Configuration { get; }
39+
40+
public void ConfigureServices(IServiceCollection services)
41+
{
42+
services.AddControllers();
43+
44+
45+
services.AddSingleton<ISyndFeedStore>(container =>
46+
{
47+
var projectId = Configuration["Amathus:ProjectId"];
48+
var bucketId = Configuration["Amathus:BucketId"];
49+
var logger = container.GetRequiredService<ILogger<ISyndFeedStore>>();
50+
return new CloudStorageSyndFeedStore(projectId, bucketId, logger);
51+
});
52+
53+
_sources = Configuration.GetSection("Amathus:Sources").Get<List<Source>>();
54+
55+
services.AddSingleton<IFeedConverter>(container =>
56+
{
57+
var logger = container.GetRequiredService<ILogger<IFeedConverter>>();
58+
return new FeedConverter(_sources, logger);
59+
});
60+
61+
_backend = Enum.Parse<FeedStoreBackend>(Configuration["Amathus:FeedStore"], ignoreCase: true);
62+
63+
switch (_backend)
64+
{
65+
case FeedStoreBackend.Firestore:
66+
services.AddSingleton<IFeedStore>(provider =>
67+
new FirestoreFeedStore(
68+
Configuration["Amathus:ProjectId"]));
69+
break;
70+
default:
71+
throw new ArgumentException("Backend cannot be initialized");
72+
}
73+
}
74+
75+
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILogger<Startup> logger)
76+
{
77+
if (env.IsDevelopment())
78+
{
79+
app.UseDeveloperExceptionPage();
80+
}
81+
82+
logger.LogInformation("Starting...");
83+
logger.LogInformation("Sources: " + _sources?.Count);
84+
logger.LogInformation("Backend: " + _backend);
85+
86+
app.UseRouting();
87+
88+
app.UseEndpoints(endpoints =>
89+
{
90+
endpoints.MapControllers();
91+
});
92+
}
93+
}
94+
}

‎Amathus/Amathus.Converter/appsettings.Development.json

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"Logging": {
3+
"LogLevel": {
4+
"Default": "Debug",
5+
"System": "Information",
6+
"Microsoft": "Information"
7+
}
8+
},
9+
"Amathus": {
10+
"ProjectId": "amathus-atamel",
11+
"BucketId": "amathus-atamel-bucket",
12+
"FeedStore": "Firestore"
13+
}
14+
}

‎Amathus/Amathus.Converter/appsettings.json

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"Logging": {
3+
"LogLevel": {
4+
"Default": "Information",
5+
"Microsoft": "Warning",
6+
"Microsoft.Hosting.Lifetime": "Information"
7+
}
8+
},
9+
"AllowedHosts": "*",
10+
"Amathus": {
11+
"ProjectId": "amathus-atamel",
12+
"BucketId": "amathus-atamel-bucket",
13+
"FeedStore": "Firestore"
14+
}
15+
}

‎Amathus/Amathus.Reader/Amathus.Reader.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<PropertyGroup Condition=" '$(RunConfiguration)' == 'Amathus.Web' ">
88
<StartAction>Project</StartAction>
9-
<ApplicationURL>http://localhost:5000</ApplicationURL>
9+
<ApplicationURL>http://localhost:5002</ApplicationURL>
1010
</PropertyGroup>
1111
<ItemGroup>
1212
</ItemGroup>

‎Amathus/Amathus.Web/Startup.cs

+13-14
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,18 @@ public void ConfigureServices(IServiceCollection services)
6060
// memory.
6161
case FeedStoreBackend.InMemory:
6262
services.AddSingleton<IFeedStore, InMemoryFeedStore>();
63-
services.AddHostedService<FeedReaderService>();
63+
services.AddHostedService<FeedReaderService>();
64+
_sources = Configuration.GetSection("Amathus:Sources").Get<List<Source>>();
65+
services.AddSingleton<IFeedReader>(container =>
66+
{
67+
var logger = container.GetRequiredService<ILogger<IFeedReader>>();
68+
return new FeedReader(_sources, logger);
69+
});
70+
services.AddSingleton<IFeedConverter>(container =>
71+
{
72+
var logger = container.GetRequiredService<ILogger<IFeedConverter>>();
73+
return new FeedConverter(_sources, logger);
74+
});
6475
break;
6576
// Firestore backedend. It assumes that the backend is already populated.
6677
case FeedStoreBackend.Firestore:
@@ -70,19 +81,7 @@ public void ConfigureServices(IServiceCollection services)
7081
break;
7182
default:
7283
throw new ArgumentException("Backend cannot be initialized");
73-
}
74-
75-
_sources = Configuration.GetSection("Amathus:Sources").Get<List<Source>>();
76-
services.AddSingleton<IFeedReader>(container =>
77-
{
78-
var logger = container.GetRequiredService<ILogger<IFeedReader>>();
79-
return new FeedReader(_sources, logger);
80-
});
81-
services.AddSingleton<IFeedConverter>(container =>
82-
{
83-
var logger = container.GetRequiredService<ILogger<IFeedConverter>>();
84-
return new FeedConverter(_sources, logger);
85-
});
84+
}
8685
}
8786

8887
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILogger<Startup> logger)

0 commit comments

Comments
 (0)