Langtag download issue 987

CHANGELOG.md

@@ -174,6 +174,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - [SIL.Core] ConsoleErrorReporter logs exception if available
 - [SIL.Core, SIL.Windows.Forms] If WinFormsErrorReporter is set as the ErrorReporter, and ErrorReporter.NotifyUserOfProblem(IRepeatNoticePolicy, Exception, String, params object[]) is passed null for the exception, the "Details" button will no longer appear, making this consistent with the no-Exception overload of this method
 - [SIL.WritingSystems] Changed behavior of IetfLanguageTag to better handle zh-TW.
+- [SIL.WritingSystems] Download of langtag.json handled with etag instead of IF-MODIFIED-SINCE Header(#987)
 
 ### Fixed
 

SIL.WritingSystems.Tests/SldrTests.cs

@@ -15,9 +15,12 @@ public class SldrTests
 	{
 		private class TestEnvironment : IDisposable
 		{
-			public TestEnvironment(bool sldrOffline = true, DateTime? embeddedAllTagsTime = null)
+			public TestEnvironment(bool sldrOffline = true, string sldrCachePath = null,
+				DateTime? embeddedAllTagsTime = null)
 			{
-				var sldrCachePath = Sldr.SldrCachePath;
+				if (string.IsNullOrEmpty(sldrCachePath))
+					sldrCachePath = Sldr.DefaultSldrCachePath;
+
 				Sldr.Cleanup();
 				if (embeddedAllTagsTime == null)
 					Sldr.Initialize(sldrOffline, sldrCachePath);
@@ -424,7 +427,8 @@ public void MoveTmpToCache_DraftProposed_KeepsUid()
 		[Explicit]
 		public void LanguageTags_OlderEmbeddedLangTags_DownloadsNewLangTags()
 		{
-			using var testEnv = new TestEnvironment(false, new DateTime(2000, 1, 1, 12, 0, 0));
+			using var testEnv = new TestEnvironment(false, null,
+				new DateTime(2000, 1, 1, 12, 0, 0));
 			var langTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json");
 			if (File.Exists(langTagsPath))
 			{
@@ -509,5 +513,112 @@ public void GetUnicodeCategoryBasedOnICU_LowSurrogateAtStartOfString_ThrowsArgum
 			Assert.That(() => Sldr.GetUnicodeCategoryBasedOnICU(bogusString, 0),
 				Throws.ArgumentException);
 		}
+
+		[Test]
+		[Explicit]
+		public void DownloadLanguageTags_NoPreviousEtagNoFile()
+		{
+			// Setup
+			using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name);
+			using var environment = new TestEnvironment(false, sldrCachePath.Path);
+			var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json");
+			var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag");
+
+			// Execute
+			Sldr.DownloadLanguageTags();
+
+			// Verify
+			Assert.That(File.Exists(langtagsFile), Is.True);
+			Assert.That(File.Exists(langtagsEtagFile), Is.True);
+			Assert.That(File.ReadAllText(langtagsFile),
+				Contains.Substring("\"full\": \"aa-Latn-ET\""));
+		}
+
+		[Test]
+		[Explicit]
+		public void DownloadLanguageTags_NoPreviousEtagExistingFile()
+		{
+			// Setup
+			using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name);
+			using var environment = new TestEnvironment(false, sldrCachePath.Path);
+			var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json");
+			var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag");
+			Sldr.DownloadLanguageTags();
+			File.Delete(langtagsEtagFile);
+
+			// Execute
+			Sldr.DownloadLanguageTags();
+
+			// Verify
+			Assert.That(File.Exists(langtagsFile), Is.True);
+			Assert.That(File.Exists(langtagsEtagFile), Is.True);
+		}
+
+		[Test]
+		[Explicit]
+		public void DownloadLanguageTags_ExistingEtagNoFile()
+		{
+			// Setup
+			using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name);
+			using var environment = new TestEnvironment(false, sldrCachePath.Path);
+			var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json");
+			var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag");
+			Sldr.DownloadLanguageTags();
+			File.Delete(langtagsFile);
+
+			// Execute
+			Sldr.DownloadLanguageTags();
+
+			// Verify
+			Assert.That(File.Exists(langtagsFile), Is.True);
+			Assert.That(File.Exists(langtagsEtagFile), Is.True);
+		}
+
+		[Test]
+		[Explicit]
+		public void DownloadLanguageTags_NotModified()
+		{
+			// Setup
+			using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name);
+			using var environment = new TestEnvironment(false, sldrCachePath.Path);
+			Sldr.DownloadLanguageTags();
+			var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json");
+			var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag");
+			var langtagsFileDate = new FileInfo(langtagsFile).LastWriteTime;
+			var etagsFileDate = new FileInfo(langtagsEtagFile).LastWriteTime;
+
+			// Execute
+			Sldr.DownloadLanguageTags();
+
+			// Verify
+			Assert.That(File.Exists(langtagsFile), Is.True);
+			Assert.That(File.Exists(langtagsEtagFile), Is.True);
+			Assert.That(new FileInfo(langtagsFile).LastWriteTime, Is.EqualTo(langtagsFileDate));
+			Assert.That(new FileInfo(langtagsEtagFile).LastWriteTime, Is.EqualTo(etagsFileDate));
+		}
+
+		[Test]
+		[Explicit]
+		public void DownloadLanguageTags_Outdated()
+		{
+			// Setup
+			using var sldrCachePath = new TemporaryFolder(TestContext.CurrentContext.Test.Name);
+			using var environment = new TestEnvironment(false, sldrCachePath.Path);
+			Sldr.DownloadLanguageTags();
+			var langtagsFile = Path.Combine(sldrCachePath.Path, "langtags.json");
+			var langtagsEtagFile = Path.Combine(sldrCachePath.Path, "langtags.json.etag");
+			File.WriteAllText(langtagsEtagFile, @"""12345678-abcdef""");
+			var langtagsFileDate = new FileInfo(langtagsFile).LastWriteTime;
+			var etagsFileDate = new FileInfo(langtagsEtagFile).LastWriteTime;
+
+			// Execute
+			Sldr.DownloadLanguageTags();
+
+			// Verify
+			Assert.That(File.Exists(langtagsFile), Is.True);
+			Assert.That(File.Exists(langtagsEtagFile), Is.True);
+			Assert.That(new FileInfo(langtagsFile).LastWriteTime, Is.GreaterThan(langtagsFileDate));
+			Assert.That(new FileInfo(langtagsEtagFile).LastWriteTime, Is.GreaterThan(etagsFileDate));
+		}
 	}
 }

SIL.WritingSystems/Sldr.cs

@@ -451,8 +451,9 @@ public static IReadOnlyKeyedCollection<string, SldrLanguageTagInfo> LanguageTags
 			}
 		}
 
-		public static void InitializeLanguageTags()
+		public static void InitializeLanguageTags(bool downloadLangTags = true)
 		{
+			if (downloadLangTags) DownloadLanguageTags();
 			LoadLanguageTagsIfNecessary();
 		}
 
@@ -470,55 +471,92 @@ private static void LoadLanguageTagsIfNecessary()
 				CreateSldrCacheDirectory();
 
 				cachedAllTagsPath = Path.Combine(SldrCachePath, "langtags.json");
+				string etagPath;
+				etagPath = Path.Combine(SldrCachePath, "langtags.json.etag");
 				var sinceTime = _embeddedAllTagsTime.ToUniversalTime();
 				if (File.Exists(cachedAllTagsPath))
 				{
 					var fileTime = File.GetLastWriteTimeUtc(cachedAllTagsPath);
 					if (sinceTime > fileTime)
+					{
 						// delete the old langtags.json file if a newer embedded one is available.
 						// this can happen if the application is upgraded to use a newer version of SIL.WritingSystems
 						// that has an updated embedded langtags.json file.
 						File.Delete(cachedAllTagsPath);
+						File.Delete(etagPath);
+					}
 					else
 						sinceTime = fileTime;
 				}
-				sinceTime += TimeSpan.FromSeconds(1);
-				try
-				{
-					if (_offlineMode)
-						throw new WebException("Test mode: SLDR offline so accessing cache", WebExceptionStatus.ConnectFailure);
 
+			}
+			_languageTags = new ReadOnlyKeyedCollection<string, SldrLanguageTagInfo>(ParseAllTagsJson(cachedAllTagsPath));
+		}
 
-					// get SLDR langtags.json from the SLDR api compressed
-					// it will throw WebException or have status HttpStatusCode.NotModified if file is unchanged or not get it
-					var langtagsUrl =
-						$"{SldrRepository}index.html?query=langtags&ext=json{StagingParameter}";
-					var webRequest = (HttpWebRequest) WebRequest.Create(Uri.EscapeUriString(langtagsUrl));
-					webRequest.UserAgent = UserAgent;
-					webRequest.IfModifiedSince = sinceTime;
-					webRequest.Timeout = 10000;
-					webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
-					using var webResponse = (HttpWebResponse) webRequest.GetResponse();
+		public static void DownloadLanguageTags()
+		{
+			CreateSldrCacheDirectory();
+			string cachedAllTagsPath;
+			cachedAllTagsPath = Path.Combine(SldrCachePath, "langtags.json");
+			string etagPath;
+			etagPath = Path.Combine(SldrCachePath, "langtags.json.etag");
+			string etag;
+			string currentEtag;
+			try
+			{
+				if (_offlineMode)
+					throw new WebException("Test mode: SLDR offline so accessing cache", WebExceptionStatus.ConnectFailure);
+				// get SLDR langtags.json from the SLDR api compressed
+				// it will throw WebException or have status HttpStatusCode.NotModified if file is unchanged or not get it
+				var langtagsUrl =
+					$"{SldrRepository}index.html?query=langtags&ext=json{StagingParameter}";
+				var webRequest = (HttpWebRequest)WebRequest.Create(Uri.EscapeUriString(langtagsUrl));
+				webRequest.UserAgent = UserAgent;
+				webRequest.Timeout = 10000;
+				webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
+				using var webResponse = (HttpWebResponse)webRequest.GetResponse();
+				if (File.Exists(etagPath))
+				{
+					etag = File.ReadAllText(etagPath);
+					webRequest.Headers.Set(etag, "If-None-Match");
+					currentEtag = webResponse.Headers.Get("Etag");
+					if (!etag.Equals(currentEtag))
+					{
+						//File.WriteAllText(etagPath, currentEtag);
+						//webRequest.Headers.Set(etag, "If-None-Match");
+						if (webResponse.StatusCode != HttpStatusCode.NotModified)
+						{
+							File.WriteAllText(etagPath, currentEtag);
+							using Stream output = File.OpenWrite(cachedAllTagsPath);
+							using var input = webResponse.GetResponseStream();
+							input.CopyTo(output);
+						}
+					}
+				}
+				else
+				{
+					currentEtag = webResponse.Headers.Get("Etag");
+					File.WriteAllText(etagPath, currentEtag);
 					if (webResponse.StatusCode != HttpStatusCode.NotModified)
 					{
 						using Stream output = File.OpenWrite(cachedAllTagsPath);
 						using var input = webResponse.GetResponseStream();
 						input.CopyTo(output);
 					}
 				}
-				catch (WebException)
-				{
-				}
-				catch (UnauthorizedAccessException)
-				{
-				}
-				catch (IOException)
-				{
-				}
 			}
-			_languageTags = new ReadOnlyKeyedCollection<string, SldrLanguageTagInfo>(ParseAllTagsJson(cachedAllTagsPath));
+			catch (WebException)
+			{
+			}
+			catch (UnauthorizedAccessException)
+			{
+			}
+			catch (IOException)
+			{
+			}
 		}
 
+
 		private static IKeyedCollection<string, SldrLanguageTagInfo> ParseAllTagsJson(string cachedAllTagsPath)
 		{
 			// read in the json file