Skip to content

Commit 410004e

Browse files
meteatamelmeteatamel
committed
Amathus.Reader now simply reads and saves the feeds to Cloud Storage
1 parent 4e34f98 commit 410004e

11 files changed

+202
-36
lines changed

‎Amathus/Amathus.Common/Amathus.Common.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
1010
<PackageReference Include="System.ServiceModel.Syndication" Version="4.7.0" />
1111
<PackageReference Include="Microsoft.Extensions.Logging" Version="3.1.0" />
12+
<PackageReference Include="Google.Cloud.Storage.V1" Version="2.5.0" />
1213
</ItemGroup>
1314
<ItemGroup>
1415
<Folder Include="Reader\" />

‎Amathus/Amathus.Common/FeedStore/CloudStorageFeedStore.cs

+151
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
// Copyright 2020 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
using System;
15+
using System.IO;
16+
using System.ServiceModel.Syndication;
17+
using System.Threading.Tasks;
18+
using System.Xml;
19+
using Amathus.Common.Reader;
20+
using Google.Cloud.Storage.V1;
21+
using Microsoft.Extensions.Logging;
22+
23+
namespace Amathus.Common.FeedStore
24+
{
25+
public class CloudStorageSyndFeedStore : ISyndFeedStore
26+
{
27+
private readonly string _bucketId;
28+
private readonly string _projectId;
29+
private readonly ILogger _logger;
30+
31+
public CloudStorageSyndFeedStore(string projectId, string bucketId, ILogger logger = null)
32+
{
33+
_projectId = projectId;
34+
_bucketId = bucketId;
35+
_logger = logger;
36+
}
37+
38+
public async Task InsertAsync(SyndicationFeed feed)
39+
{
40+
var client = StorageClient.Create();
41+
42+
await CreateBucketIfNeeded(client);
43+
44+
var objectName = feed.Id.ToLowerInvariant() + ".xml";
45+
_logger?.LogInformation($"Uploading {objectName} to bucket {_bucketId}");
46+
47+
var stream = GetStream(feed);
48+
await client.UploadObjectAsync(_bucketId, objectName, "application/xml", stream);
49+
50+
_logger?.LogInformation($"Uploaded {objectName}");
51+
}
52+
53+
public async Task<SyndicationFeed> ReadAsync(string feedId)
54+
{
55+
if (string.IsNullOrEmpty(feedId))
56+
{
57+
throw new ArgumentNullException();
58+
}
59+
60+
var bucketExists = await BucketExists();
61+
if (!bucketExists)
62+
{
63+
throw new ArgumentException("Bucket does not exist yet");
64+
}
65+
66+
var client = StorageClient.Create();
67+
68+
var objectName = feedId.ToLowerInvariant() + "2.xml";
69+
_logger?.LogInformation($"Reading {objectName} from bucket {_bucketId}");
70+
71+
var stream = new MemoryStream();
72+
try
73+
{
74+
await client.DownloadObjectAsync(_bucketId, objectName, stream);
75+
_logger?.LogInformation($"Read {objectName}");
76+
}
77+
catch (Exception e)
78+
{
79+
_logger.LogError($"Error reading {objectName}: " + e.Message);
80+
throw e;
81+
}
82+
83+
_logger?.LogInformation($"Converting to SyndicationFeed");
84+
85+
try
86+
{
87+
stream.Position = 0; // Reset to read
88+
var reader = new DateInvariantXmlReader(stream);
89+
90+
var feed = SyndicationFeed.Load(reader);
91+
reader.Close();
92+
_logger?.LogInformation($"Converted to SyndicationFeed");
93+
return feed;
94+
}
95+
catch (Exception e)
96+
{
97+
_logger?.LogError($"Error converting {e.Message}");
98+
throw e;
99+
}
100+
}
101+
102+
103+
private async Task CreateBucketIfNeeded(StorageClient client)
104+
{
105+
var exists = await BucketExists();
106+
if (exists)
107+
{
108+
_logger?.LogInformation($"Bucket exists: {_bucketId}");
109+
110+
}
111+
else
112+
{
113+
_logger?.LogInformation($"Bucked does not exist, creating bucket: {_bucketId}");
114+
115+
await client.CreateBucketAsync(_projectId, _bucketId);
116+
}
117+
}
118+
119+
private async Task<bool> BucketExists()
120+
{
121+
if (string.IsNullOrEmpty(_bucketId))
122+
{
123+
throw new ArgumentNullException();
124+
}
125+
126+
try
127+
{
128+
var client = StorageClient.Create();
129+
130+
await client.GetBucketAsync(_bucketId);
131+
132+
return true;
133+
134+
}
135+
catch (Exception)
136+
{
137+
return false;
138+
}
139+
}
140+
141+
private static MemoryStream GetStream(SyndicationFeed feed)
142+
{
143+
var stream = new MemoryStream();
144+
var writer = XmlWriter.Create(stream);
145+
var atomFormatter = feed.GetAtom10Formatter();
146+
atomFormatter.WriteTo(writer);
147+
writer.Close();
148+
return stream;
149+
}
150+
}
151+
}

‎Amathus/Amathus.Common/FeedStore/ISyndFeedStore.cs

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright 2020 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
using System.ServiceModel.Syndication;
15+
using System.Threading.Tasks;
16+
17+
namespace Amathus.Common.FeedStore
18+
{
19+
public interface ISyndFeedStore
20+
{
21+
Task InsertAsync(SyndicationFeed feed);
22+
23+
Task<SyndicationFeed> ReadAsync(string feedId);
24+
}
25+
}

‎Amathus/Amathus.Common/Reader/DateInvariantXmlReader.cs

+6
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
// limitations under the License.
1515
using System;
1616
using System.Globalization;
17+
using System.IO;
1718
using System.Xml;
1819

1920
namespace Amathus.Common.Reader
@@ -29,6 +30,11 @@ public class DateInvariantXmlReader : XmlTextReader
2930

3031
private bool _readingDate;
3132

33+
public DateInvariantXmlReader(Stream input): base(input)
34+
{
35+
// No-op.
36+
}
37+
3238
public DateInvariantXmlReader(string inputUri): base(inputUri)
3339
{
3440
// No-op.

‎Amathus/Amathus.Reader/Controllers/ReadController.cs

+5-9
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
using Amathus.Common.FeedStore;
1919
using Microsoft.AspNetCore.Mvc;
2020
using Microsoft.Extensions.Logging;
21-
using Amathus.Common.Converter;
2221

2322
namespace Amathus.Reader.Controllers
2423
{
@@ -27,14 +26,12 @@ public class ReadController : ControllerBase
2726
{
2827
private readonly ILogger _logger;
2928
private readonly IFeedReader _reader;
30-
private readonly IFeedConverter _converter;
31-
private readonly IFeedStore _store;
29+
private readonly ISyndFeedStore _syncStore;
3230

33-
public ReadController(IFeedReader reader, IFeedConverter converter, IFeedStore store, ILogger<ReadController> logger)
31+
public ReadController(IFeedReader reader, ISyndFeedStore syncStore, ILogger<ReadController> logger)
3432
{
3533
_reader = reader;
36-
_converter = converter;
37-
_store = store;
34+
_syncStore = syncStore;
3835
_logger = logger;
3936
}
4037

@@ -46,10 +43,9 @@ public async Task<IActionResult> Read()
4643
stopWatch.Start();
4744

4845
var rawFeeds = (await _reader.ReadAll()).ToList();
49-
rawFeeds.ForEach(rawFeed =>
46+
rawFeeds.ForEach(async rawFeed =>
5047
{
51-
var feed = _converter.Convert(rawFeed.Id, rawFeed);
52-
_store.InsertAsync(feed);
48+
await _syncStore.InsertAsync(rawFeed);
5349
});
5450

5551
stopWatch.Stop();

‎Amathus/Amathus.Reader/Startup.cs

+7-20
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
14-
using System;
1514
using System.Collections.Generic;
1615
using Amathus.Common.Reader;
1716
using Amathus.Common.FeedStore;
@@ -22,13 +21,11 @@
2221
using Microsoft.Extensions.DependencyInjection;
2322
using Microsoft.Extensions.Hosting;
2423
using Microsoft.Extensions.Logging;
25-
using Amathus.Common.Converter;
2624

2725
namespace Amathus.Reader
2826
{
2927
public class Startup
3028
{
31-
private FeedStoreBackend _backend;
3229
private List<Source> _sources;
3330

3431
public Startup(IConfiguration configuration)
@@ -42,28 +39,19 @@ public void ConfigureServices(IServiceCollection services)
4239
{
4340
services.AddControllers();
4441

45-
_backend = Enum.Parse<FeedStoreBackend>(Configuration["Amathus:FeedStore"], ignoreCase: true);
46-
switch (_backend)
47-
{
48-
case FeedStoreBackend.Firestore:
49-
services.AddSingleton<IFeedStore>(provider =>
50-
new FirestoreFeedStore(
51-
Configuration["Amathus:FirestoreProjectId"]));
52-
break;
53-
default:
54-
throw new ArgumentException("Backend cannot be initialized");
55-
}
56-
5742
_sources = Configuration.GetSection("Amathus:Sources").Get<List<Source>>();
5843
services.AddSingleton<IFeedReader>(container =>
5944
{
6045
var logger = container.GetRequiredService<ILogger<IFeedReader>>();
6146
return new FeedReader(_sources, logger);
62-
});
63-
services.AddSingleton<IFeedConverter>(container =>
47+
});
48+
49+
services.AddSingleton<ISyndFeedStore>(container =>
6450
{
65-
var logger = container.GetRequiredService<ILogger<IFeedConverter>>();
66-
return new FeedConverter(_sources, logger);
51+
var projectId = Configuration["Amathus:ProjectId"];
52+
var bucketId = Configuration["Amathus:BucketId"];
53+
var logger = container.GetRequiredService<ILogger<ISyndFeedStore>>();
54+
return new CloudStorageSyndFeedStore(projectId, bucketId, logger);
6755
});
6856
}
6957

@@ -75,7 +63,6 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILogger<
7563
}
7664

7765
logger.LogInformation("Starting...");
78-
logger.LogInformation("Backend: " + _backend);
7966
logger.LogInformation("Sources: " + _sources?.Count);
8067

8168
app.UseRouting();

‎Amathus/Amathus.Reader/appsettings.Development.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
}
88
},
99
"Amathus": {
10-
"FeedStore": "Firestore",
11-
"FirestoreProjectId": "amathus-atamel"
10+
"ProjectId": "amathus-atamel",
11+
"BucketId": "amathus-atamel-bucket"
1212
}
1313
}

‎Amathus/Amathus.Reader/appsettings.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
},
99
"AllowedHosts": "*",
1010
"Amathus": {
11-
"FeedStore": "Firestore",
12-
"FirestoreProjectId": "amathus-atamel"
11+
"ProjectId": "amathus-atamel",
12+
"BucketId": "amathus-atamel-bucket"
1313
}
1414
}

‎Amathus/Amathus.Web/Startup.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void ConfigureServices(IServiceCollection services)
6666
case FeedStoreBackend.Firestore:
6767
services.AddSingleton<IFeedStore>(provider =>
6868
new FirestoreFeedStore(
69-
Configuration["Amathus:FirestoreProjectId"]));
69+
Configuration["Amathus:ProjectId"]));
7070
break;
7171
default:
7272
throw new ArgumentException("Backend cannot be initialized");

‎Amathus/Amathus.Web/appsettings.Development.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@
77
}
88
},
99
"Amathus": {
10-
"FeedStore": "InMemory",
10+
"FeedStore": "InMemory"
1111
}
1212
}

‎Amathus/Amathus.Web/appsettings.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
"AllowedHosts": "*",
1010
"Amathus": {
1111
"FeedStore": "Firestore",
12-
"FirestoreProjectId": "amathus-atamel"
12+
"ProjectId": "amathus-atamel"
1313
}
1414
}

0 commit comments

Comments
 (0)