Add support for Azure Managed Identity (elastic#111344)

With this commit, if no key or SAS token is supplied for an Azure
repository then Elasticsearch will use the `DefaultAzureCredential`
chain defined in the Azure SDK, which will obtain credentials from the
instance metadata service when running on an Azure VM.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java

@@ -102,7 +102,7 @@ public class LicenseAnalyzer {
             AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
             LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
             OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-            SOFTWARE\\.
+            SOFTWARE\\.?
             """).replaceAll("\\s+", "\\\\s*"), Pattern.DOTALL)),
         new LicenseMatcher(
             "MIT-0",

docs/changelog/111344.yaml

@@ -0,0 +1,5 @@
+pr: 111344
+summary: Add support for Azure Managed Identity
+area: Snapshot/Restore
+type: enhancement
+issues: []

gradle/verification-metadata.xml

@@ -129,6 +129,11 @@
             <sha256 value="e07002835364c67a9267fd019ee1019dc9ea865491b6b1d4f9b8c8809fc1bae7" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.azure" name="azure-identity" version="1.13.1">
+         <artifact name="azure-identity-1.13.1.jar">
+            <sha256 value="b479a7a1b6143212faaca3f4edc62d2929b14ea9aa48c80fe7cbe418928d6bcd" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.azure" name="azure-json" version="1.1.0">
          <artifact name="azure-json-1.1.0.jar">
             <sha256 value="114af9b1459c9c93190b2a82f427c0e93fbbb28896fc57f2585a9b821275ca56" origin="Generated by Gradle"/>
@@ -536,6 +541,11 @@
             <sha256 value="0cdd163ce3598a20fc04eee71b140b24f6f2a3b35f0a499dbbdd9852e83fbfaf" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.code.gson" name="gson" version="2.11.0">
+         <artifact name="gson-2.11.0.jar">
+            <sha256 value="57928d6e5a6edeb2abd3770a8f95ba44dce45f3b23b7a9dc2b309c581552a78b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.code.gson" name="gson" version="2.2.4">
          <artifact name="gson-2.2.4.jar">
             <sha256 value="c0328cd07ca9e363a5acd00c1cf4afe8cf554bd6d373834981ba05cebec687fb" origin="Generated by Gradle"/>
@@ -546,6 +556,11 @@
             <sha256 value="c8fb4839054d280b3033f800d1f5a97de2f028eb8ba2eb458ad287e536f3f25f" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.crypto.tink" name="tink" version="1.14.0">
+         <artifact name="tink-1.14.0.jar">
+            <sha256 value="47b2248705e0c9771bc259f22465a79655c1296e2d47aaee852adb7cdacb6198" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.errorprone" name="error_prone_annotations" version="2.1.3">
          <artifact name="error_prone_annotations-2.1.3.jar">
             <sha256 value="03d0329547c13da9e17c634d1049ea2ead093925e290567e1a364fd6b1fc7ff8" origin="Generated by Gradle"/>
@@ -726,6 +741,11 @@
             <sha256 value="1b78b4a76a71512debfdff8f8fc5aef6bfd459f65758fecf7aff245e6e6301e4" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protobuf-java" version="4.27.0">
+         <artifact name="protobuf-java-4.27.0.jar">
+            <sha256 value="9072e60fe66cff5d6c0f11a1df21d8f3e4b29b5ee782b45c3fc75f59fbe2b839" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.protobuf" name="protobuf-java-util" version="3.21.9">
          <artifact name="protobuf-java-util-3.21.9.jar">
             <sha256 value="a78226d02551ca2cb34b24b4168a75b4a84041cc3333f3725bb5ad9806a80b77" origin="Generated by Gradle"/>
@@ -861,6 +881,16 @@
             <sha256 value="f1eae7a4e14944940e54605d7b512bdd3296efe6353af6bfb15c1b97ed06d8ec" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.microsoft.azure" name="msal4j" version="1.16.1">
+         <artifact name="msal4j-1.16.1.jar">
+            <sha256 value="871e50e97d61685d77480fc021b048eecb94d4a005b08f89e7f5d0a6a8ab1dc0" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.microsoft.azure" name="msal4j-persistence-extension" version="1.3.0">
+         <artifact name="msal4j-persistence-extension-1.3.0.jar">
+            <sha256 value="dfc41c817fbfa76057af6ffe4379dbca6a5e16b8e87df8bdda23f371756c2d09" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.microsoft.sqlserver" name="mssql-jdbc" version="6.2.1.jre7">
          <artifact name="mssql-jdbc-6.2.1.jre7.jar">
             <sha256 value="9cfa259450ae3471d2e6e2c3d8aefcce236e3daef8b3734d21dc93c3a5bbe806" origin="Generated by Gradle"/>
@@ -896,11 +926,21 @@
             <sha256 value="260647c459cc7de269b2a8f576e5693e434df888fd4801e92365fe18b9992cbc" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.nimbusds" name="content-type" version="2.3">
+         <artifact name="content-type-2.3.jar">
+            <sha256 value="60349793e006fba96b532cb0c21e10e969fe0db8d87f91c3b9eaf82ba2998895" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.nimbusds" name="lang-tag" version="1.4.4">
          <artifact name="lang-tag-1.4.4.jar">
             <sha256 value="e49d2c694bb80c7036c177f2aabf53b7156061a68bd19dfd60e2bd370709e0c5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.nimbusds" name="lang-tag" version="1.7">
+         <artifact name="lang-tag-1.7.jar">
+            <sha256 value="e8c1c594e2425bdbea2d860de55c69b69fc5d59454452449a0f0913c2a5b8a31" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.nimbusds" name="nimbus-jose-jwt" version="4.41.1">
          <artifact name="nimbus-jose-jwt-4.41.1.jar">
             <sha256 value="fbfd0d5f2b2f86758b821daa5e79b5d7c965edd9dc1b2cc80b515df1c6ddc22d" origin="Generated by Gradle"/>
@@ -911,11 +951,21 @@
             <sha256 value="33ab8084fdae1d75be1b061b1489d4a12045bd7b50c2e24ff152911e4551ec07" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.nimbusds" name="nimbus-jose-jwt" version="9.37.3">
+         <artifact name="nimbus-jose-jwt-9.37.3.jar">
+            <sha256 value="12ae4a3a260095d7aeba2adea7ae396e8b9570db8b7b409e09a824c219cc0444" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.nimbusds" name="nimbus-jose-jwt" version="9.8.1">
          <artifact name="nimbus-jose-jwt-9.8.1.jar">
             <sha256 value="7664cf8c6f2adadf600287812b32878277beda54912eab9d4c2932cd50cb704a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.nimbusds" name="oauth2-oidc-sdk" version="11.9.1">
+         <artifact name="oauth2-oidc-sdk-11.9.1.jar">
+            <sha256 value="0820c9690966304d075347b88e81ae490213440fc4d2c84f3d370d41941b2b9c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.nimbusds" name="oauth2-oidc-sdk" version="9.37">
          <artifact name="oauth2-oidc-sdk-9.37.jar">
             <sha256 value="44a04bbed5ae3f6d198aa73ee6b545c476e528ec1a267ef3e9f7033f886dd6fe" origin="Generated by Gradle"/>
@@ -1639,6 +1689,11 @@
             <sha256 value="0271ae7fc162a5e69c337f36d86fdb94a8a232c5c42a80d8a7424071addd1fdc" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="net.java.dev.jna" name="jna-platform" version="5.12.1">
+         <artifact name="jna-platform-5.12.1.jar">
+            <sha256 value="8ce969116cac95bd61b07a8d5e07174b352e63301473caac72c395e3c08488d2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="net.java.dev.jna" name="jna-platform" version="5.7.0">
          <artifact name="jna-platform-5.7.0.jar">
             <sha256 value="42e020705692eddbd285e2b72ef0ff468f51a926382569c45f4e9cea4602ad1e" origin="Generated by Gradle"/>
@@ -1664,6 +1719,11 @@
             <sha256 value="0972bbc99437c4163acd09b630e6c77eab4cfab8a9594621c95466c0c6645396" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="net.minidev" name="accessors-smart" version="2.5.0">
+         <artifact name="accessors-smart-2.5.0.jar">
+            <sha256 value="12314fc6881d66a413fd66370787adba16e504fbf7e138690b0f3952e3fbd321" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="net.minidev" name="json-smart" version="2.3">
          <artifact name="json-smart-2.3.jar">
             <sha256 value="903f48c8aa4c3f6426440b8d32de89fa1dc23b1169abde25e4e1d068aa67708b" origin="Generated by Gradle"/>
@@ -1679,6 +1739,11 @@
             <sha256 value="64072f56d9dff5040b2acec477c5d5e6bcebfc88c508f12acb26072d07942146" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="net.minidev" name="json-smart" version="2.5.0">
+         <artifact name="json-smart-2.5.0.jar">
+            <sha256 value="432b9e545848c4141b80717b26e367f83bf33f19250a228ce75da6e967da2bc7" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="net.nextencia" name="rrdiagram" version="0.9.4">
          <artifact name="rrdiagram-0.9.4.jar">
             <sha256 value="8f0855addca5320cfadedbf7d3d46b681f3f308b6e87d5d82f32637ba72256b6" origin="Generated by Gradle"/>
@@ -3312,6 +3377,11 @@
             <sha256 value="c600d1ae61b5b0ff1391e00eb6fb390201e4612c3aaf2dc1b94050c8784840be" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.cryptomator" name="siv-mode" version="1.5.2">
+         <artifact name="siv-mode-1.5.2.jar">
+            <sha256 value="81a33da58e5e878ffe8972ef93cd0bfc3a9daa0f5e3af4dd151be2b1d73018b4" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.eclipse.jdt" name="ecj" version="3.33.0">
          <artifact name="ecj-3.33.0.jar">
             <sha256 value="f7686c4960cf70c2ebc5c500a73a8cfc04541b730c18f1c5c21329889b137f45" origin="Generated by Gradle"/>

modules/repository-azure/build.gradle

@@ -1,5 +1,6 @@
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
+import org.elasticsearch.gradle.internal.test.RestIntegTestTask
 
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
@@ -24,11 +25,14 @@ dependencies {
   // Microsoft
   api "com.azure:azure-core-http-netty:1.15.1"
   api "com.azure:azure-core:1.50.0"
+  api "com.azure:azure-identity:1.13.1"
   api "com.azure:azure-json:1.1.0"
   api "com.azure:azure-storage-blob:12.26.1"
   api "com.azure:azure-storage-common:12.26.0"
   api "com.azure:azure-storage-internal-avro:12.11.1"
   api "com.azure:azure-xml:1.0.0"
+  api "com.microsoft.azure:msal4j-persistence-extension:1.3.0"
+  api "com.microsoft.azure:msal4j:1.16.1"
 
   // Jackson
   api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
@@ -53,9 +57,24 @@ dependencies {
 
   // Others
   api "com.fasterxml.woodstox:woodstox-core:6.4.0"
+  api "com.github.stephenc.jcip:jcip-annotations:1.0-1"
+  api "com.nimbusds:content-type:2.3"
+  api "com.nimbusds:lang-tag:1.7"
+  api "com.nimbusds:nimbus-jose-jwt:9.37.3"
+  api "com.nimbusds:oauth2-oidc-sdk:11.9.1"
   api "jakarta.activation:jakarta.activation-api:1.2.1"
   api "jakarta.xml.bind:jakarta.xml.bind-api:2.3.3"
+  api "net.java.dev.jna:jna-platform:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version
+  api "net.java.dev.jna:jna:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version
+  api "net.minidev:accessors-smart:2.5.0"
+  api "net.minidev:json-smart:2.5.0"
   api "org.codehaus.woodstox:stax2-api:4.2.1"
+  api "org.ow2.asm:asm:9.3"
+
+  runtimeOnly "com.google.crypto.tink:tink:1.14.0"
+  runtimeOnly "com.google.protobuf:protobuf-java:4.27.0"
+  runtimeOnly "com.google.code.gson:gson:2.11.0"
+  runtimeOnly "org.cryptomator:siv-mode:1.5.2"
 
   implementation project(":modules:transport-netty4")
   implementation("org.slf4j:slf4j-api:${versions.slf4j}")
@@ -151,12 +170,94 @@ tasks.named("thirdPartyAudit").configure {
     'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
     // [missing classes] SLF4j includes an optional class that depends on an extension class. see Log4jLogger#createConverter
     //    'org.slf4j.ext.EventData' - bring back when https://github.com/elastic/elasticsearch/issues/93714 is done
+
+    // Optional dependency of tink
+    'com.google.api.client.http.HttpHeaders',
+    'com.google.api.client.http.HttpRequest',
+    'com.google.api.client.http.HttpRequestFactory',
+    'com.google.api.client.http.HttpResponse',
+    'com.google.api.client.http.HttpTransport',
+    'com.google.api.client.http.javanet.NetHttpTransport',
+    'com.google.api.client.http.javanet.NetHttpTransport$Builder',
+
+    // Optional dependency of nimbus-jose-jwt and oauth2-oidc-sdk
+    'org.bouncycastle.asn1.pkcs.PrivateKeyInfo',
+    'org.bouncycastle.asn1.x509.AlgorithmIdentifier',
+    'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo',
+    'org.bouncycastle.cert.X509CertificateHolder',
+    'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder',
+    'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
+    'org.bouncycastle.crypto.InvalidCipherTextException',
+    'org.bouncycastle.crypto.engines.AESEngine',
+    'org.bouncycastle.crypto.modes.GCMBlockCipher',
+    'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider',
+    'org.bouncycastle.jce.provider.BouncyCastleProvider',
+    'org.bouncycastle.openssl.PEMKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+
+    // OAuth servlet support is optional and not required
+    'jakarta.servlet.ServletRequest',
+    'jakarta.servlet.http.HttpServletRequest',
+    'jakarta.servlet.http.HttpServletResponse',
+    'javax.servlet.ServletRequest',
+    'javax.servlet.http.HttpServletRequest',
+    'javax.servlet.http.HttpServletResponse',
+
+    // OpenSAML support is optional
+    'org.joda.time.DateTime',
+    'net.shibboleth.utilities.java.support.xml.SerializeSupport',
+    'org.opensaml.core.config.InitializationException',
+    'org.opensaml.core.config.InitializationService',
+    'org.opensaml.core.xml.XMLObject',
+    'org.opensaml.core.xml.XMLObjectBuilder',
+    'org.opensaml.core.xml.XMLObjectBuilderFactory',
+    'org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport',
+    'org.opensaml.core.xml.io.Marshaller',
+    'org.opensaml.core.xml.io.MarshallerFactory',
+    'org.opensaml.core.xml.io.MarshallingException',
+    'org.opensaml.core.xml.io.Unmarshaller',
+    'org.opensaml.core.xml.io.UnmarshallerFactory',
+    'org.opensaml.core.xml.schema.XSString',
+    'org.opensaml.core.xml.schema.impl.XSStringBuilder',
+    'org.opensaml.saml.saml2.core.Assertion',
+    'org.opensaml.saml.saml2.core.Attribute',
+    'org.opensaml.saml.saml2.core.AttributeStatement',
+    'org.opensaml.saml.saml2.core.AttributeValue',
+    'org.opensaml.saml.saml2.core.Audience',
+    'org.opensaml.saml.saml2.core.AudienceRestriction',
+    'org.opensaml.saml.saml2.core.AuthnContext',
+    'org.opensaml.saml.saml2.core.AuthnContextClassRef',
+    'org.opensaml.saml.saml2.core.AuthnStatement',
+    'org.opensaml.saml.saml2.core.Conditions',
+    'org.opensaml.saml.saml2.core.Issuer',
+    'org.opensaml.saml.saml2.core.NameID',
+    'org.opensaml.saml.saml2.core.Subject',
+    'org.opensaml.saml.saml2.core.SubjectConfirmation',
+    'org.opensaml.saml.saml2.core.SubjectConfirmationData',
+    'org.opensaml.saml.security.impl.SAMLSignatureProfileValidator',
+    'org.opensaml.security.credential.BasicCredential',
+    'org.opensaml.security.credential.Credential',
+    'org.opensaml.security.credential.UsageType',
+    'org.opensaml.xmlsec.signature.Signature',
+    'org.opensaml.xmlsec.signature.support.SignatureException',
+    'org.opensaml.xmlsec.signature.support.SignatureValidator',
+    'org.opensaml.xmlsec.signature.support.Signer',
   )
 
   ignoreViolations(
     'javax.activation.MailcapCommandMap',
     'javax.activation.MimetypesFileTypeMap',
     'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException',
+
+    'com.google.protobuf.MessageSchema',
+    'com.google.protobuf.UnsafeUtil',
+    'com.google.protobuf.UnsafeUtil$1',
+    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$MemoryAccessor',
   )
 }
 
@@ -189,6 +290,7 @@ tasks.named("processYamlRestTestResources") {
 tasks.named("internalClusterTest") {
   // this is tested explicitly in a separate test task
   exclude '**/AzureStorageCleanupThirdPartyTests.class'
+  systemProperty "AZURE_POD_IDENTITY_AUTHORITY_HOST", "127.0.0.1:1" // ensure a fast failure
 }
 
 tasks.named("yamlRestTest") {
@@ -199,6 +301,18 @@ tasks.named("yamlRestTest") {
   systemProperty 'test.azure.sas_token', azureSasToken
 }
 
+tasks.register("managedIdentityYamlRestTest", RestIntegTestTask) {
+  testClassesDirs = sourceSets.yamlRestTest.output.classesDirs
+  classpath = sourceSets.yamlRestTest.runtimeClasspath
+}
+
+tasks.named("managedIdentityYamlRestTest") {
+  systemProperty 'test.azure.fixture', Boolean.toString(useFixture)
+  systemProperty 'test.azure.account', azureAccount
+  systemProperty 'test.azure.container', azureContainer
+  // omitting key and sas_token so that we use a bearer token from the metadata service
+}
+
 tasks.register("azureThirdPartyUnitTest", Test) {
   SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
   SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
@@ -219,4 +333,5 @@ tasks.register('azureThirdPartyTest') {
 
 tasks.named("check") {
   dependsOn("azureThirdPartyUnitTest")
+  dependsOn("managedIdentityYamlRestTest")
 }