Skip to content

Commit b8c0c84

Browse files
seantleonardaaronpowell
seantleonard
and
aaronpowell
authored
[Cherry Pick] Handling missing environment variables cleaner for SWA (#1825) (#1834)
Cherry pick #1825 to 0.9 ## Why make this change? - Fixes #1820 ## What is this change? - New test coverage for #1820 - Added a way to control error handling when env vars aren't set - default is left to throw, override to ignore ## How was this tested? - [x] Integration Tests - [ ] Unit Tests Co-authored-by: Aaron Powell <[email protected]>
1 parent 34fa935 commit b8c0c84

12 files changed

+228
-38
lines changed

‎src/Cli.Tests/EnvironmentTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public class EnvironmentTests
1919
[TestInitialize]
2020
public void TestInitialize()
2121
{
22-
StringJsonConverterFactory converterFactory = new();
22+
StringJsonConverterFactory converterFactory = new(EnvironmentVariableReplacementFailureMode.Throw);
2323
_options = new()
2424
{
2525
PropertyNameCaseInsensitive = true

‎src/Config/Converters/EnvironmentVariableReplacementFailureMode.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Azure.DataApiBuilder.Config.Converters;
5+
6+
/// <summary>
7+
/// Control how to handle environment variable replacement failures when deserializing strings in the JSON config file.
8+
/// </summary>
9+
public enum EnvironmentVariableReplacementFailureMode
10+
{
11+
/// <summary>
12+
/// Ignore the missing environment variable and return the original value, eg: @env('schema').
13+
/// </summary>
14+
Ignore,
15+
/// <summary>
16+
/// Throw an exception when a missing environment variable is encountered. This is the default behavior.
17+
/// </summary>
18+
Throw
19+
}

‎src/Config/Converters/StringJsonConverterFactory.cs

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,21 @@ namespace Azure.DataApiBuilder.Config.Converters;
1414
/// </summary>
1515
public class StringJsonConverterFactory : JsonConverterFactory
1616
{
17+
private EnvironmentVariableReplacementFailureMode _replacementFailureMode;
18+
19+
public StringJsonConverterFactory(EnvironmentVariableReplacementFailureMode replacementFailureMode)
20+
{
21+
_replacementFailureMode = replacementFailureMode;
22+
}
23+
1724
public override bool CanConvert(Type typeToConvert)
1825
{
1926
return typeToConvert.IsAssignableTo(typeof(string));
2027
}
2128

2229
public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
2330
{
24-
return new StringJsonConverter();
31+
return new StringJsonConverter(_replacementFailureMode);
2532
}
2633

2734
class StringJsonConverter : JsonConverter<string>
@@ -42,6 +49,12 @@ class StringJsonConverter : JsonConverter<string>
4249
// within the name of the environment variable, but that ') is not
4350
// a valid environment variable name in certain shells.
4451
const string ENV_PATTERN = @"@env\('.*?(?='\))'\)";
52+
private EnvironmentVariableReplacementFailureMode _replacementFailureMode;
53+
54+
public StringJsonConverter(EnvironmentVariableReplacementFailureMode replacementFailureMode)
55+
{
56+
_replacementFailureMode = replacementFailureMode;
57+
}
4558

4659
public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
4760
{
@@ -64,7 +77,7 @@ public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOp
6477
writer.WriteStringValue(value);
6578
}
6679

67-
private static string ReplaceMatchWithEnvVariable(Match match)
80+
private string ReplaceMatchWithEnvVariable(Match match)
6881
{
6982
// [^@env\(] : any substring that is not @env(
7083
// .* : any char except newline any number of times
@@ -76,10 +89,17 @@ private static string ReplaceMatchWithEnvVariable(Match match)
7689
// strips first and last characters, ie: '''hello'' --> ''hello'
7790
string envName = Regex.Match(match.Value, innerPattern).Value[1..^1];
7891
string? envValue = Environment.GetEnvironmentVariable(envName);
79-
return envValue is not null ? envValue :
80-
throw new DataApiBuilderException(message: $"Environmental Variable, {envName}, not found.",
81-
statusCode: System.Net.HttpStatusCode.ServiceUnavailable,
82-
subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
92+
if (_replacementFailureMode == EnvironmentVariableReplacementFailureMode.Throw)
93+
{
94+
return envValue is not null ? envValue :
95+
throw new DataApiBuilderException(message: $"Environmental Variable, {envName}, not found.",
96+
statusCode: System.Net.HttpStatusCode.ServiceUnavailable,
97+
subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
98+
}
99+
else
100+
{
101+
return envValue ?? match.Value;
102+
}
83103
}
84104
}
85105
}

‎src/Config/Converters/Utf8JsonReaderExtensions.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,12 @@ static internal class Utf8JsonReaderExtensions
1515
/// <param name="reader">The reader that we want to pull the string from.</param>
1616
/// <param name="replaceEnvVar">Whether to replace environment variable with its
1717
/// value or not while deserializing.</param>
18+
/// <param name="replacementFailureMode">The failure mode to use when replacing environment variables.</param>
1819
/// <returns>The result of deserialization.</returns>
1920
/// <exception cref="JsonException">Thrown if the <see cref="JsonTokenType"/> is not String.</exception>
20-
public static string? DeserializeString(this Utf8JsonReader reader, bool replaceEnvVar)
21+
public static string? DeserializeString(this Utf8JsonReader reader,
22+
bool replaceEnvVar,
23+
EnvironmentVariableReplacementFailureMode replacementFailureMode = EnvironmentVariableReplacementFailureMode.Throw)
2124
{
2225
if (reader.TokenType is JsonTokenType.Null)
2326
{
@@ -33,7 +36,7 @@ static internal class Utf8JsonReaderExtensions
3336
JsonSerializerOptions options = new();
3437
if (replaceEnvVar)
3538
{
36-
options.Converters.Add(new StringJsonConverterFactory());
39+
options.Converters.Add(new StringJsonConverterFactory(replacementFailureMode));
3740
}
3841

3942
return JsonSerializer.Deserialize<string>(ref reader, options);

‎src/Config/RuntimeConfigLoader.cs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,10 @@ public static bool TryParseConfig(string json,
6060
string? connectionString = null,
6161
bool replaceEnvVar = false,
6262
string dataSourceName = "",
63-
Dictionary<string, string>? datasourceNameToConnectionString = null)
63+
Dictionary<string, string>? datasourceNameToConnectionString = null,
64+
EnvironmentVariableReplacementFailureMode replacementFailureMode = EnvironmentVariableReplacementFailureMode.Throw)
6465
{
65-
JsonSerializerOptions options = GetSerializationOptions(replaceEnvVar);
66+
JsonSerializerOptions options = GetSerializationOptions(replaceEnvVar, replacementFailureMode);
6667

6768
try
6869
{
@@ -171,7 +172,9 @@ public static bool TryParseConfig(string json,
171172
/// </summary>
172173
/// <param name="replaceEnvVar">Whether to replace environment variable with value or not while deserializing.
173174
/// By default, no replacement happens.</param>
174-
public static JsonSerializerOptions GetSerializationOptions(bool replaceEnvVar = false)
175+
public static JsonSerializerOptions GetSerializationOptions(
176+
bool replaceEnvVar = false,
177+
EnvironmentVariableReplacementFailureMode replacementFailureMode = EnvironmentVariableReplacementFailureMode.Throw)
175178
{
176179

177180
JsonSerializerOptions options = new()
@@ -193,7 +196,7 @@ public static JsonSerializerOptions GetSerializationOptions(bool replaceEnvVar =
193196

194197
if (replaceEnvVar)
195198
{
196-
options.Converters.Add(new StringJsonConverterFactory());
199+
options.Converters.Add(new StringJsonConverterFactory(replacementFailureMode));
197200
}
198201

199202
return options;

‎src/Core/Configurations/RuntimeConfigProvider.cs

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.Net;
77
using System.Text.Json;
88
using Azure.DataApiBuilder.Config;
9+
using Azure.DataApiBuilder.Config.Converters;
910
using Azure.DataApiBuilder.Config.NamingPolicies;
1011
using Azure.DataApiBuilder.Config.ObjectModel;
1112
using Azure.DataApiBuilder.Service.Exceptions;
@@ -136,7 +137,8 @@ public async Task<bool> Initialize(
136137
if (RuntimeConfigLoader.TryParseConfig(
137138
configuration,
138139
out RuntimeConfig? runtimeConfig,
139-
replaceEnvVar: true))
140+
replaceEnvVar: false,
141+
replacementFailureMode: EnvironmentVariableReplacementFailureMode.Ignore))
140142
{
141143
_runtimeConfig = runtimeConfig;
142144

@@ -170,7 +172,13 @@ public async Task<bool> Initialize(
170172
/// <param name="connectionString">The connection string to the database.</param>
171173
/// <param name="accessToken">The string representation of a managed identity access token</param>
172174
/// <returns>true if the initialization succeeded, false otherwise.</returns>
173-
public async Task<bool> Initialize(string jsonConfig, string? graphQLSchema, string connectionString, string? accessToken)
175+
public async Task<bool> Initialize(
176+
string jsonConfig,
177+
string? graphQLSchema,
178+
string connectionString,
179+
string? accessToken,
180+
bool replaceEnvVar = true,
181+
EnvironmentVariableReplacementFailureMode replacementFailureMode = EnvironmentVariableReplacementFailureMode.Throw)
174182
{
175183
if (string.IsNullOrEmpty(connectionString))
176184
{
@@ -184,7 +192,7 @@ public async Task<bool> Initialize(string jsonConfig, string? graphQLSchema, str
184192

185193
IsLateConfigured = true;
186194

187-
if (RuntimeConfigLoader.TryParseConfig(jsonConfig, out RuntimeConfig? runtimeConfig, replaceEnvVar: true))
195+
if (RuntimeConfigLoader.TryParseConfig(jsonConfig, out RuntimeConfig? runtimeConfig, replaceEnvVar: replaceEnvVar, replacementFailureMode: replacementFailureMode))
188196
{
189197
_runtimeConfig = runtimeConfig.DataSource.DatabaseType switch
190198
{

‎src/Service.Tests/Configuration/ConfigurationEndpoints.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Azure.DataApiBuilder.Service.Tests.Configuration;
5+
6+
internal static class ConfigurationEndpoints
7+
{
8+
// TODO: Remove the old endpoint once we've updated all callers to use the new one.
9+
public const string CONFIGURATION_ENDPOINT = "/configuration";
10+
public const string CONFIGURATION_ENDPOINT_V2 = "/configuration/v2";
11+
}

‎src/Service.Tests/Configuration/ConfigurationTests.cs

Lines changed: 3 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@
4343
using Npgsql;
4444
using VerifyMSTest;
4545
using static Azure.DataApiBuilder.Config.FileSystemRuntimeConfigLoader;
46+
using static Azure.DataApiBuilder.Service.Tests.Configuration.ConfigurationEndpoints;
47+
using static Azure.DataApiBuilder.Service.Tests.Configuration.TestConfigFileReader;
4648

4749
namespace Azure.DataApiBuilder.Service.Tests.Configuration
4850
{
@@ -67,10 +69,6 @@ public class ConfigurationTests
6769
private const int RETRY_COUNT = 5;
6870
private const int RETRY_WAIT_SECONDS = 1;
6971

70-
// TODO: Remove the old endpoint once we've updated all callers to use the new one.
71-
private const string CONFIGURATION_ENDPOINT = "/configuration";
72-
private const string CONFIGURATION_ENDPOINT_V2 = "/configuration/v2";
73-
7472
/// <summary>
7573
/// A valid REST API request body with correct parameter types for all the fields.
7674
/// </summary>
@@ -2232,7 +2230,7 @@ private static JsonContent GetJsonContentForCosmosConfigRequest(string endpoint,
22322230
if (CONFIGURATION_ENDPOINT == endpoint)
22332231
{
22342232
ConfigurationPostParameters configParams = GetCosmosConfigurationParameters();
2235-
if (config != null)
2233+
if (config is not null)
22362234
{
22372235
configParams = configParams with { Configuration = config };
22382236
}
@@ -2329,21 +2327,6 @@ private static ConfigurationPostParametersV2 GetCosmosConfigurationParametersV2(
23292327
AccessToken: null);
23302328
}
23312329

2332-
private static RuntimeConfig ReadCosmosConfigurationFromFile()
2333-
{
2334-
string cosmosFile = $"{CONFIGFILE_NAME}.{COSMOS_ENVIRONMENT}{CONFIG_EXTENSION}";
2335-
2336-
string configurationFileContents = File.ReadAllText(cosmosFile);
2337-
if (!RuntimeConfigLoader.TryParseConfig(configurationFileContents, out RuntimeConfig config))
2338-
{
2339-
throw new Exception("Failed to parse configuration file.");
2340-
}
2341-
2342-
// The Schema file isn't provided in the configuration file when going through the configuration endpoint so we're removing it.
2343-
config.DataSource.Options.Remove("Schema");
2344-
return config;
2345-
}
2346-
23472330
/// <summary>
23482331
/// Helper used to create the post-startup configuration payload sent to configuration controller.
23492332
/// Adds entity used to hydrate authorization resolver post-startup and validate that hydration succeeds.

‎src/Service.Tests/Configuration/LoadConfigViaEndpoint.cs

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Net;
6+
using System.Net.Http;
7+
using System.Net.Http.Json;
8+
using System.Threading.Tasks;
9+
using Azure.DataApiBuilder.Config.ObjectModel;
10+
using Azure.DataApiBuilder.Core.Configurations;
11+
using Azure.DataApiBuilder.Service.Controllers;
12+
using Microsoft.AspNetCore.TestHost;
13+
using Microsoft.VisualStudio.TestTools.UnitTesting;
14+
using static Azure.DataApiBuilder.Service.Tests.Configuration.ConfigurationEndpoints;
15+
using static Azure.DataApiBuilder.Service.Tests.Configuration.TestConfigFileReader;
16+
17+
namespace Azure.DataApiBuilder.Service.Tests.Configuration;
18+
19+
[TestClass]
20+
public class LoadConfigViaEndpointTests
21+
{
22+
[TestMethod("Testing that missing environment variables won't cause runtime failure."), TestCategory(TestCategory.COSMOSDBNOSQL)]
23+
[DataRow(CONFIGURATION_ENDPOINT)]
24+
[DataRow(CONFIGURATION_ENDPOINT_V2)]
25+
public async Task CanLoadConfigWithMissingEnvironmentVariables(string configurationEndpoint)
26+
{
27+
TestServer server = new(Program.CreateWebHostFromInMemoryUpdateableConfBuilder(Array.Empty<string>()));
28+
HttpClient httpClient = server.CreateClient();
29+
30+
(RuntimeConfig config, JsonContent content) = GetParameterContent(configurationEndpoint);
31+
32+
HttpResponseMessage postResult =
33+
await httpClient.PostAsync(configurationEndpoint, content);
34+
Assert.AreEqual(HttpStatusCode.OK, postResult.StatusCode);
35+
36+
RuntimeConfigProvider configProvider = server.Services.GetService(typeof(RuntimeConfigProvider)) as RuntimeConfigProvider;
37+
RuntimeConfig loadedConfig = configProvider.GetConfig();
38+
39+
Assert.AreEqual(config.Schema, loadedConfig.Schema);
40+
}
41+
42+
[TestMethod("Testing that environment variables can be replaced at runtime not only when config is loaded."), TestCategory(TestCategory.COSMOSDBNOSQL)]
43+
[DataRow(CONFIGURATION_ENDPOINT)]
44+
[DataRow(CONFIGURATION_ENDPOINT_V2)]
45+
[Ignore("We don't want to environment variable substitution in late configuration, but test is left in for if this changes.")]
46+
public async Task CanLoadConfigWithEnvironmentVariables(string configurationEndpoint)
47+
{
48+
Environment.SetEnvironmentVariable("schema", "schema.graphql");
49+
TestServer server = new(Program.CreateWebHostFromInMemoryUpdateableConfBuilder(Array.Empty<string>()));
50+
HttpClient httpClient = server.CreateClient();
51+
52+
(RuntimeConfig config, JsonContent content) = GetParameterContent(configurationEndpoint);
53+
54+
HttpResponseMessage postResult =
55+
await httpClient.PostAsync(configurationEndpoint, content);
56+
Assert.AreEqual(HttpStatusCode.OK, postResult.StatusCode);
57+
58+
RuntimeConfigProvider configProvider = server.Services.GetService(typeof(RuntimeConfigProvider)) as RuntimeConfigProvider;
59+
RuntimeConfig loadedConfig = configProvider.GetConfig();
60+
61+
Assert.AreNotEqual(config.Schema, loadedConfig.Schema);
62+
Assert.AreEqual(Environment.GetEnvironmentVariable("schema"), loadedConfig.Schema);
63+
}
64+
65+
[TestCleanup]
66+
public void Cleanup()
67+
{
68+
Environment.SetEnvironmentVariable("schema", null);
69+
}
70+
71+
private static (RuntimeConfig, JsonContent) GetParameterContent(string endpoint)
72+
{
73+
RuntimeConfig config = ReadCosmosConfigurationFromFile() with { Schema = "@env('schema')" };
74+
75+
if (endpoint == CONFIGURATION_ENDPOINT)
76+
{
77+
ConfigurationPostParameters @params = new(
78+
Configuration: config.ToJson(),
79+
Schema: @"
80+
type Entity {
81+
id: ID!
82+
name: String!
83+
}
84+
",
85+
ConnectionString: "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
86+
AccessToken: null
87+
);
88+
89+
return (config, JsonContent.Create(@params));
90+
}
91+
else if (endpoint == CONFIGURATION_ENDPOINT_V2)
92+
{
93+
ConfigurationPostParametersV2 @params = new(
94+
Configuration: config.ToJson(),
95+
ConfigurationOverrides: "{}",
96+
Schema: @"
97+
type Entity {
98+
id: ID!
99+
name: String!
100+
}
101+
",
102+
AccessToken: null
103+
);
104+
105+
return (config, JsonContent.Create(@params));
106+
}
107+
108+
throw new ArgumentException($"Unknown endpoint: {endpoint}");
109+
}
110+
}

‎src/Service.Tests/Configuration/TestConfigFileReader.cs

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.IO;
6+
using Azure.DataApiBuilder.Config;
7+
using Azure.DataApiBuilder.Config.ObjectModel;
8+
using static Azure.DataApiBuilder.Config.FileSystemRuntimeConfigLoader;
9+
10+
namespace Azure.DataApiBuilder.Service.Tests.Configuration;
11+
12+
/// <summary>
13+
/// Provides the methods to read the configuration files from disk for tests.
14+
/// </summary>
15+
internal static class TestConfigFileReader
16+
{
17+
public static RuntimeConfig ReadCosmosConfigurationFromFile()
18+
{
19+
string cosmosFile = $"{CONFIGFILE_NAME}.{TestCategory.COSMOSDBNOSQL}{CONFIG_EXTENSION}";
20+
21+
string configurationFileContents = File.ReadAllText(cosmosFile);
22+
if (!RuntimeConfigLoader.TryParseConfig(configurationFileContents, out RuntimeConfig config))
23+
{
24+
throw new Exception("Failed to parse configuration file.");
25+
}
26+
27+
// The Schema file isn't provided in the configuration file when going through the configuration endpoint so we're removing it.
28+
_ = config.DataSource.Options.Remove("Schema");
29+
return config;
30+
}
31+
}

0 commit comments

Comments
 (0)