Skip to content

Commit a12f39f

Browse files
aaronpowellaaronpowell
authored
Handling missing environment variables cleaner for SWA (#1825)
## Why make this change? - Fixes #1820 ## What is this change? - New test coverage for #1820 - Added a way to control error handling when env vars aren't set - default is left to throw, override to ignore ## How was this tested? - [x] Integration Tests - [ ] Unit Tests
1 parent a60b598 commit a12f39f

12 files changed

+228
-38
lines changed

‎src/Cli.Tests/EnvironmentTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public class EnvironmentTests
1919
[TestInitialize]
2020
public void TestInitialize()
2121
{
22-
StringJsonConverterFactory converterFactory = new();
22+
StringJsonConverterFactory converterFactory = new(EnvironmentVariableReplacementFailureMode.Throw);
2323
_options = new()
2424
{
2525
PropertyNameCaseInsensitive = true

‎src/Config/Converters/EnvironmentVariableReplacementFailureMode.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Azure.DataApiBuilder.Config.Converters;
5+
6+
/// <summary>
7+
/// Control how to handle environment variable replacement failures when deserializing strings in the JSON config file.
8+
/// </summary>
9+
public enum EnvironmentVariableReplacementFailureMode
10+
{
11+
/// <summary>
12+
/// Ignore the missing environment variable and return the original value, eg: @env('schema').
13+
/// </summary>
14+
Ignore,
15+
/// <summary>
16+
/// Throw an exception when a missing environment variable is encountered. This is the default behavior.
17+
/// </summary>
18+
Throw
19+
}

‎src/Config/Converters/StringJsonConverterFactory.cs

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,21 @@ namespace Azure.DataApiBuilder.Config.Converters;
1414
/// </summary>
1515
public class StringJsonConverterFactory : JsonConverterFactory
1616
{
17+
private EnvironmentVariableReplacementFailureMode _replacementFailureMode;
18+
19+
public StringJsonConverterFactory(EnvironmentVariableReplacementFailureMode replacementFailureMode)
20+
{
21+
_replacementFailureMode = replacementFailureMode;
22+
}
23+
1724
public override bool CanConvert(Type typeToConvert)
1825
{
1926
return typeToConvert.IsAssignableTo(typeof(string));
2027
}
2128

2229
public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
2330
{
24-
return new StringJsonConverter();
31+
return new StringJsonConverter(_replacementFailureMode);
2532
}
2633

2734
class StringJsonConverter : JsonConverter<string>
@@ -42,6 +49,12 @@ class StringJsonConverter : JsonConverter<string>
4249
// within the name of the environment variable, but that ') is not
4350
// a valid environment variable name in certain shells.
4451
const string ENV_PATTERN = @"@env\('.*?(?='\))'\)";
52+
private EnvironmentVariableReplacementFailureMode _replacementFailureMode;
53+
54+
public StringJsonConverter(EnvironmentVariableReplacementFailureMode replacementFailureMode)
55+
{
56+
_replacementFailureMode = replacementFailureMode;
57+
}
4558

4659
public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
4760
{
@@ -64,7 +77,7 @@ public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOp
6477
writer.WriteStringValue(value);
6578
}
6679

67-
private static string ReplaceMatchWithEnvVariable(Match match)
80+
private string ReplaceMatchWithEnvVariable(Match match)
6881
{
6982
// [^@env\(] : any substring that is not @env(
7083
// .* : any char except newline any number of times
@@ -76,10 +89,17 @@ private static string ReplaceMatchWithEnvVariable(Match match)
7689
// strips first and last characters, ie: '''hello'' --> ''hello'
7790
string envName = Regex.Match(match.Value, innerPattern).Value[1..^1];
7891
string? envValue = Environment.GetEnvironmentVariable(envName);
79-
return envValue is not null ? envValue :
80-
throw new DataApiBuilderException(message: $"Environmental Variable, {envName}, not found.",
81-
statusCode: System.Net.HttpStatusCode.ServiceUnavailable,
82-
subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
92+
if (_replacementFailureMode == EnvironmentVariableReplacementFailureMode.Throw)
93+
{
94+
return envValue is not null ? envValue :
95+
throw new DataApiBuilderException(message: $"Environmental Variable, {envName}, not found.",
96+
statusCode: System.Net.HttpStatusCode.ServiceUnavailable,
97+
subStatusCode: DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
98+
}
99+
else
100+
{
101+
return envValue ?? match.Value;
102+
}
83103
}
84104
}
85105
}

‎src/Config/Converters/Utf8JsonReaderExtensions.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,12 @@ static internal class Utf8JsonReaderExtensions
1515
/// <param name="reader">The reader that we want to pull the string from.</param>
1616
/// <param name="replaceEnvVar">Whether to replace environment variable with its
1717
/// value or not while deserializing.</param>
18+
/// <param name="replacementFailureMode">The failure mode to use when replacing environment variables.</param>
1819
/// <returns>The result of deserialization.</returns>
1920
/// <exception cref="JsonException">Thrown if the <see cref="JsonTokenType"/> is not String.</exception>
20-
public static string? DeserializeString(this Utf8JsonReader reader, bool replaceEnvVar)
21+
public static string? DeserializeString(this Utf8JsonReader reader,
22+
bool replaceEnvVar,
23+
EnvironmentVariableReplacementFailureMode replacementFailureMode = EnvironmentVariableReplacementFailureMode.Throw)
2124
{
2225
if (reader.TokenType is JsonTokenType.Null)
2326
{
@@ -33,7 +36,7 @@ static internal class Utf8JsonReaderExtensions
3336
JsonSerializerOptions options = new();
3437
if (replaceEnvVar)
3538
{
36-
options.Converters.Add(new StringJsonConverterFactory());
39+
options.Converters.Add(new StringJsonConverterFactory(replacementFailureMode));
3740
}
3841

3942
return JsonSerializer.Deserialize<string>(ref reader, options);

‎src/Config/RuntimeConfigLoader.cs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,10 @@ public static bool TryParseConfig(string json,
6060
string? connectionString = null,
6161
bool replaceEnvVar = false,
6262
string dataSourceName = "",
63-
Dictionary<string, string>? datasourceNameToConnectionString = null)
63+
Dictionary<string, string>? datasourceNameToConnectionString = null,
64+
EnvironmentVariableReplacementFailureMode replacementFailureMode = EnvironmentVariableReplacementFailureMode.Throw)
6465
{
65-
JsonSerializerOptions options = GetSerializationOptions(replaceEnvVar);
66+
JsonSerializerOptions options = GetSerializationOptions(replaceEnvVar, replacementFailureMode);
6667

6768
try
6869
{
@@ -174,7 +175,9 @@ ex is JsonException ||
174175
/// </summary>
175176
/// <param name="replaceEnvVar">Whether to replace environment variable with value or not while deserializing.
176177
/// By default, no replacement happens.</param>
177-
public static JsonSerializerOptions GetSerializationOptions(bool replaceEnvVar = false)
178+
public static JsonSerializerOptions GetSerializationOptions(
179+
bool replaceEnvVar = false,
180+
EnvironmentVariableReplacementFailureMode replacementFailureMode = EnvironmentVariableReplacementFailureMode.Throw)
178181
{
179182

180183
JsonSerializerOptions options = new()
@@ -197,7 +200,7 @@ public static JsonSerializerOptions GetSerializationOptions(bool replaceEnvVar =
197200

198201
if (replaceEnvVar)
199202
{
200-
options.Converters.Add(new StringJsonConverterFactory());
203+
options.Converters.Add(new StringJsonConverterFactory(replacementFailureMode));
201204
}
202205

203206
return options;

‎src/Core/Configurations/RuntimeConfigProvider.cs

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.Net;
77
using System.Text.Json;
88
using Azure.DataApiBuilder.Config;
9+
using Azure.DataApiBuilder.Config.Converters;
910
using Azure.DataApiBuilder.Config.NamingPolicies;
1011
using Azure.DataApiBuilder.Config.ObjectModel;
1112
using Azure.DataApiBuilder.Service.Exceptions;
@@ -136,7 +137,8 @@ public async Task<bool> Initialize(
136137
if (RuntimeConfigLoader.TryParseConfig(
137138
configuration,
138139
out RuntimeConfig? runtimeConfig,
139-
replaceEnvVar: true))
140+
replaceEnvVar: false,
141+
replacementFailureMode: EnvironmentVariableReplacementFailureMode.Ignore))
140142
{
141143
_runtimeConfig = runtimeConfig;
142144

@@ -198,7 +200,13 @@ public bool TrySetAccesstoken(
198200
/// <param name="connectionString">The connection string to the database.</param>
199201
/// <param name="accessToken">The string representation of a managed identity access token</param>
200202
/// <returns>true if the initialization succeeded, false otherwise.</returns>
201-
public async Task<bool> Initialize(string jsonConfig, string? graphQLSchema, string connectionString, string? accessToken)
203+
public async Task<bool> Initialize(
204+
string jsonConfig,
205+
string? graphQLSchema,
206+
string connectionString,
207+
string? accessToken,
208+
bool replaceEnvVar = true,
209+
EnvironmentVariableReplacementFailureMode replacementFailureMode = EnvironmentVariableReplacementFailureMode.Throw)
202210
{
203211
if (string.IsNullOrEmpty(connectionString))
204212
{
@@ -212,7 +220,7 @@ public async Task<bool> Initialize(string jsonConfig, string? graphQLSchema, str
212220

213221
IsLateConfigured = true;
214222

215-
if (RuntimeConfigLoader.TryParseConfig(jsonConfig, out RuntimeConfig? runtimeConfig, replaceEnvVar: true))
223+
if (RuntimeConfigLoader.TryParseConfig(jsonConfig, out RuntimeConfig? runtimeConfig, replaceEnvVar: replaceEnvVar, replacementFailureMode: replacementFailureMode))
216224
{
217225
_runtimeConfig = runtimeConfig.DataSource.DatabaseType switch
218226
{

‎src/Service.Tests/Configuration/ConfigurationEndpoints.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Azure.DataApiBuilder.Service.Tests.Configuration;
5+
6+
internal static class ConfigurationEndpoints
7+
{
8+
// TODO: Remove the old endpoint once we've updated all callers to use the new one.
9+
public const string CONFIGURATION_ENDPOINT = "/configuration";
10+
public const string CONFIGURATION_ENDPOINT_V2 = "/configuration/v2";
11+
}

‎src/Service.Tests/Configuration/ConfigurationTests.cs

Lines changed: 3 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
using Moq;
4242
using VerifyMSTest;
4343
using static Azure.DataApiBuilder.Config.FileSystemRuntimeConfigLoader;
44+
using static Azure.DataApiBuilder.Service.Tests.Configuration.ConfigurationEndpoints;
45+
using static Azure.DataApiBuilder.Service.Tests.Configuration.TestConfigFileReader;
4446

4547
namespace Azure.DataApiBuilder.Service.Tests.Configuration
4648
{
@@ -65,10 +67,6 @@ public class ConfigurationTests
6567
private const int RETRY_COUNT = 5;
6668
private const int RETRY_WAIT_SECONDS = 1;
6769

68-
// TODO: Remove the old endpoint once we've updated all callers to use the new one.
69-
private const string CONFIGURATION_ENDPOINT = "/configuration";
70-
private const string CONFIGURATION_ENDPOINT_V2 = "/configuration/v2";
71-
7270
/// <summary>
7371
/// A valid REST API request body with correct parameter types for all the fields.
7472
/// </summary>
@@ -2261,7 +2259,7 @@ private static JsonContent GetJsonContentForCosmosConfigRequest(string endpoint,
22612259
if (CONFIGURATION_ENDPOINT == endpoint)
22622260
{
22632261
ConfigurationPostParameters configParams = GetCosmosConfigurationParameters();
2264-
if (config != null)
2262+
if (config is not null)
22652263
{
22662264
configParams = configParams with { Configuration = config };
22672265
}
@@ -2358,21 +2356,6 @@ private static ConfigurationPostParametersV2 GetCosmosConfigurationParametersV2(
23582356
AccessToken: null);
23592357
}
23602358

2361-
private static RuntimeConfig ReadCosmosConfigurationFromFile()
2362-
{
2363-
string cosmosFile = $"{CONFIGFILE_NAME}.{COSMOS_ENVIRONMENT}{CONFIG_EXTENSION}";
2364-
2365-
string configurationFileContents = File.ReadAllText(cosmosFile);
2366-
if (!RuntimeConfigLoader.TryParseConfig(configurationFileContents, out RuntimeConfig config))
2367-
{
2368-
throw new Exception("Failed to parse configuration file.");
2369-
}
2370-
2371-
// The Schema file isn't provided in the configuration file when going through the configuration endpoint so we're removing it.
2372-
config.DataSource.Options.Remove("Schema");
2373-
return config;
2374-
}
2375-
23762359
/// <summary>
23772360
/// Helper used to create the post-startup configuration payload sent to configuration controller.
23782361
/// Adds entity used to hydrate authorization resolver post-startup and validate that hydration succeeds.

‎src/Service.Tests/Configuration/LoadConfigViaEndpoint.cs

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Net;
6+
using System.Net.Http;
7+
using System.Net.Http.Json;
8+
using System.Threading.Tasks;
9+
using Azure.DataApiBuilder.Config.ObjectModel;
10+
using Azure.DataApiBuilder.Core.Configurations;
11+
using Azure.DataApiBuilder.Service.Controllers;
12+
using Microsoft.AspNetCore.TestHost;
13+
using Microsoft.VisualStudio.TestTools.UnitTesting;
14+
using static Azure.DataApiBuilder.Service.Tests.Configuration.ConfigurationEndpoints;
15+
using static Azure.DataApiBuilder.Service.Tests.Configuration.TestConfigFileReader;
16+
17+
namespace Azure.DataApiBuilder.Service.Tests.Configuration;
18+
19+
[TestClass]
20+
public class LoadConfigViaEndpointTests
21+
{
22+
[TestMethod("Testing that missing environment variables won't cause runtime failure."), TestCategory(TestCategory.COSMOSDBNOSQL)]
23+
[DataRow(CONFIGURATION_ENDPOINT)]
24+
[DataRow(CONFIGURATION_ENDPOINT_V2)]
25+
public async Task CanLoadConfigWithMissingEnvironmentVariables(string configurationEndpoint)
26+
{
27+
TestServer server = new(Program.CreateWebHostFromInMemoryUpdateableConfBuilder(Array.Empty<string>()));
28+
HttpClient httpClient = server.CreateClient();
29+
30+
(RuntimeConfig config, JsonContent content) = GetParameterContent(configurationEndpoint);
31+
32+
HttpResponseMessage postResult =
33+
await httpClient.PostAsync(configurationEndpoint, content);
34+
Assert.AreEqual(HttpStatusCode.OK, postResult.StatusCode);
35+
36+
RuntimeConfigProvider configProvider = server.Services.GetService(typeof(RuntimeConfigProvider)) as RuntimeConfigProvider;
37+
RuntimeConfig loadedConfig = configProvider.GetConfig();
38+
39+
Assert.AreEqual(config.Schema, loadedConfig.Schema);
40+
}
41+
42+
[TestMethod("Testing that environment variables can be replaced at runtime not only when config is loaded."), TestCategory(TestCategory.COSMOSDBNOSQL)]
43+
[DataRow(CONFIGURATION_ENDPOINT)]
44+
[DataRow(CONFIGURATION_ENDPOINT_V2)]
45+
[Ignore("We don't want to environment variable substitution in late configuration, but test is left in for if this changes.")]
46+
public async Task CanLoadConfigWithEnvironmentVariables(string configurationEndpoint)
47+
{
48+
Environment.SetEnvironmentVariable("schema", "schema.graphql");
49+
TestServer server = new(Program.CreateWebHostFromInMemoryUpdateableConfBuilder(Array.Empty<string>()));
50+
HttpClient httpClient = server.CreateClient();
51+
52+
(RuntimeConfig config, JsonContent content) = GetParameterContent(configurationEndpoint);
53+
54+
HttpResponseMessage postResult =
55+
await httpClient.PostAsync(configurationEndpoint, content);
56+
Assert.AreEqual(HttpStatusCode.OK, postResult.StatusCode);
57+
58+
RuntimeConfigProvider configProvider = server.Services.GetService(typeof(RuntimeConfigProvider)) as RuntimeConfigProvider;
59+
RuntimeConfig loadedConfig = configProvider.GetConfig();
60+
61+
Assert.AreNotEqual(config.Schema, loadedConfig.Schema);
62+
Assert.AreEqual(Environment.GetEnvironmentVariable("schema"), loadedConfig.Schema);
63+
}
64+
65+
[TestCleanup]
66+
public void Cleanup()
67+
{
68+
Environment.SetEnvironmentVariable("schema", null);
69+
}
70+
71+
private static (RuntimeConfig, JsonContent) GetParameterContent(string endpoint)
72+
{
73+
RuntimeConfig config = ReadCosmosConfigurationFromFile() with { Schema = "@env('schema')" };
74+
75+
if (endpoint == CONFIGURATION_ENDPOINT)
76+
{
77+
ConfigurationPostParameters @params = new(
78+
Configuration: config.ToJson(),
79+
Schema: @"
80+
type Entity {
81+
id: ID!
82+
name: String!
83+
}
84+
",
85+
ConnectionString: "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
86+
AccessToken: null
87+
);
88+
89+
return (config, JsonContent.Create(@params));
90+
}
91+
else if (endpoint == CONFIGURATION_ENDPOINT_V2)
92+
{
93+
ConfigurationPostParametersV2 @params = new(
94+
Configuration: config.ToJson(),
95+
ConfigurationOverrides: "{}",
96+
Schema: @"
97+
type Entity {
98+
id: ID!
99+
name: String!
100+
}
101+
",
102+
AccessToken: null
103+
);
104+
105+
return (config, JsonContent.Create(@params));
106+
}
107+
108+
throw new ArgumentException($"Unknown endpoint: {endpoint}");
109+
}
110+
}

‎src/Service.Tests/Configuration/TestConfigFileReader.cs

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.IO;
6+
using Azure.DataApiBuilder.Config;
7+
using Azure.DataApiBuilder.Config.ObjectModel;
8+
using static Azure.DataApiBuilder.Config.FileSystemRuntimeConfigLoader;
9+
10+
namespace Azure.DataApiBuilder.Service.Tests.Configuration;
11+
12+
/// <summary>
13+
/// Provides the methods to read the configuration files from disk for tests.
14+
/// </summary>
15+
internal static class TestConfigFileReader
16+
{
17+
public static RuntimeConfig ReadCosmosConfigurationFromFile()
18+
{
19+
string cosmosFile = $"{CONFIGFILE_NAME}.{TestCategory.COSMOSDBNOSQL}{CONFIG_EXTENSION}";
20+
21+
string configurationFileContents = File.ReadAllText(cosmosFile);
22+
if (!RuntimeConfigLoader.TryParseConfig(configurationFileContents, out RuntimeConfig config))
23+
{
24+
throw new Exception("Failed to parse configuration file.");
25+
}
26+
27+
// The Schema file isn't provided in the configuration file when going through the configuration endpoint so we're removing it.
28+
_ = config.DataSource.Options.Remove("Schema");
29+
return config;
30+
}
31+
}

0 commit comments

Comments
 (0)