Segmentation fault in aws_event_loop_group_get_next_loop+0x12

[souravagrawal29]

Describe the bug

We are seeing the following segmentation fault when using the S3AsyncClient built using S3CrtAsyncClientBuilder to delete an object on S3.

#
# A fatal error has been detected by the Java Runtime Environment:
#
#  SIGSEGV (0xb) at pc=0x00007f3482340b22, pid=294274, tid=0x00007f346a1fe640
#
# JRE version: OpenJDK Runtime Environment (8.0_432-b06) (build 1.8.0_432-b06)
# Java VM: OpenJDK 64-Bit Server VM (25.432-b06 mixed mode linux-amd64 compressed oops)
# Problematic frame:
# C  [AWSCRT_9130152621248699454libaws-crt-jni.so+0x140b22]  aws_event_loop_group_get_next_loop+0x12
#
# Core dump written. Default location: /home/kompuser/core or core.294274
#
# An error report file with more information is saved as:
# /home/kompuser/log/hs_err_pid_cloud5886_294274.log
[thread 139863156319808 also had an error]
#
# If you would like to submit a bug report, please visit:
#   https://access.redhat.com/support/cases/
# The crash happened outside the Java Virtual Machine in native code.
# See problematic frame for where to report the bug.
#

Stack trace from hs_err_pid.log file

Stack: [0x00007f346a0ff000,0x00007f346a1ff000],  sp=0x00007f346a1fb5c0,  free space=1009k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
C  [AWSCRT_9130152621248699454libaws-crt-jni.so+0x140b22]  aws_event_loop_group_get_next_loop+0x12
C  [AWSCRT_9130152621248699454libaws-crt-jni.so+0x12795f]  aws_s3_meta_request_init_base+0x27f
C  [AWSCRT_9130152621248699454libaws-crt-jni.so+0x1242eb]  aws_s3_meta_request_default_new+0x7b
C  [AWSCRT_9130152621248699454libaws-crt-jni.so+0x11ecfb]
C  [AWSCRT_9130152621248699454libaws-crt-jni.so+0x1204a3]  aws_s3_client_make_meta_request+0x1b3
C  [AWSCRT_9130152621248699454libaws-crt-jni.so+0xa3e36]  Java_software_amazon_awssdk_crt_s3_S3Client_s3ClientMakeMetaRequest+0x706
j  software.amazon.awssdk.crt.s3.S3Client.s3ClientMakeMetaRequest(JLsoftware/amazon/awssdk/crt/s3/S3MetaRequest;[BIIIZ[I[BLsoftware/amazon/awssdk/crt/http/HttpRequestBodyStream;[BLsoftware/amazon/awssdk/crt/auth/signing/AwsSigningConfig;Lsoftware/amazon/awssdk/crt/s3/S3MetaRequestResponseHandlerNativeAdapter;[BLsoftware/amazon/awssdk/crt/s3/ResumeToken;)J+0
j  software.amazon.awssdk.crt.s3.S3Client.makeMetaRequest(Lsoftware/amazon/awssdk/crt/s3/S3MetaRequestOptions;)Lsoftware/amazon/awssdk/crt/s3/S3MetaRequest;+295
j  software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncHttpClient.execute(Lsoftware/amazon/awssdk/http/async/AsyncExecuteRequest;)Ljava/util/concurrent/CompletableFuture;+215
j  software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.doExecuteHttpRequest(Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;Lsoftware/amazon/awssdk/http/async/AsyncExecuteRequest$Builder;Lsoftware/amazon/awssdk/core/internal/http/TransformingAsyncResponseHandler;)Ljava/util/concurrent/CompletableFuture;+45
j  software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.executeHttpRequest(Lsoftware/amazon/awssdk/http/SdkHttpFullRequest;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/util/concurrent/CompletableFuture;+147
j  software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$execute$1(Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;Ljava/util/concurrent/CompletableFuture;Lsoftware/amazon/awssdk/http/SdkHttpFullRequest;)V+3
j  software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage$$Lambda$894.accept(Ljava/lang/Object;)V+16
j  java.util.concurrent.CompletableFuture.uniAccept(Ljava/util/concurrent/CompletableFuture;Ljava/util/function/Consumer;Ljava/util/concurrent/CompletableFuture$UniAccept;)Z+84
j  java.util.concurrent.CompletableFuture.uniAcceptStage(Ljava/util/concurrent/Executor;Ljava/util/function/Consumer;)Ljava/util/concurrent/CompletableFuture;+28
j  java.util.concurrent.CompletableFuture.thenAccept(Ljava/util/function/Consumer;)Ljava/util/concurrent/CompletableFuture;+3
j  software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.execute(Ljava/util/concurrent/CompletableFuture;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/util/concurrent/CompletableFuture;+29
j  software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object;+6
J 6840 C2 software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object; (22 bytes) @ 0x00007f34c21d3408 [0x00007f34c21d33a0+0x68]
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallAttemptMetricCollectionStage.execute(Lsoftware/amazon/awssdk/http/SdkHttpFullRequest;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/util/concurrent/CompletableFuture;+26
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallAttemptMetricCollectionStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object;+6
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.attemptExecute(Ljava/util/concurrent/CompletableFuture;)V+25
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.maybeAttemptExecute(Ljava/util/concurrent/CompletableFuture;)V+181
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.execute()Ljava/util/concurrent/CompletableFuture;+10
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage.execute(Lsoftware/amazon/awssdk/http/SdkHttpFullRequest;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/util/concurrent/CompletableFuture;+11
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object;+6
J 6840 C2 software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object; (22 bytes) @ 0x00007f34c21d33ec [0x00007f34c21d33a0+0x4c]
J 6840 C2 software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object; (22 bytes) @ 0x00007f34c21d33ec [0x00007f34c21d33a0+0x4c]
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncExecutionFailureExceptionReportingStage.execute(Lsoftware/amazon/awssdk/http/SdkHttpFullRequest;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/util/concurrent/CompletableFuture;+6
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncExecutionFailureExceptionReportingStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object;+6
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallTimeoutTrackingStage.execute(Lsoftware/amazon/awssdk/http/SdkHttpFullRequest;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/util/concurrent/CompletableFuture;+67
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallTimeoutTrackingStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object;+6
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallMetricCollectionStage.execute(Lsoftware/amazon/awssdk/http/SdkHttpFullRequest;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/util/concurrent/CompletableFuture;+33
j  software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallMetricCollectionStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object;+6
J 6840 C2 software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(Ljava/lang/Object;Lsoftware/amazon/awssdk/core/internal/http/RequestExecutionContext;)Ljava/lang/Object; (22 bytes) @ 0x00007f34c21d3408 [0x00007f34c21d33a0+0x68]
j  software.amazon.awssdk.core.internal.http.AmazonAsyncHttpClient$RequestExecutionBuilderImpl.execute(Lsoftware/amazon/awssdk/core/internal/http/TransformingAsyncResponseHandler;)Ljava/util/concurrent/CompletableFuture;+202
j  software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.invoke(Lsoftware/amazon/awssdk/core/client/config/SdkClientConfiguration;Lsoftware/amazon/awssdk/http/SdkHttpFullRequest;Lsoftware/amazon/awssdk/core/async/AsyncRequestBody;Lsoftware/amazon/awssdk/core/SdkRequest;Lsoftware/amazon/awssdk/core/http/ExecutionContext;Lsoftware/amazon/awssdk/core/internal/http/TransformingAsyncResponseHandler;)Ljava/util/concurrent/CompletableFuture;+46
j  software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.doExecute(Lsoftware/amazon/awssdk/core/client/handler/ClientExecutionParams;Lsoftware/amazon/awssdk/core/http/ExecutionContext;Lsoftware/amazon/awssdk/core/internal/http/TransformingAsyncResponseHandler;)Ljava/util/concurrent/CompletableFuture;+146
j  software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.lambda$execute$1(Lsoftware/amazon/awssdk/core/client/handler/ClientExecutionParams;)Ljava/util/concurrent/CompletableFuture;+17
j  software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler$$Lambda$846.get()Ljava/lang/Object;+8
j  software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.measureApiCallSuccess(Lsoftware/amazon/awssdk/core/client/handler/ClientExecutionParams;Ljava/util/function/Supplier;)Ljava/util/concurrent/CompletableFuture;+1
j  software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.execute(Lsoftware/amazon/awssdk/core/client/handler/ClientExecutionParams;)Ljava/util/concurrent/CompletableFuture;+9
j  software.amazon.awssdk.awscore.client.handler.AwsAsyncClientHandler.execute(Lsoftware/amazon/awssdk/core/client/handler/ClientExecutionParams;)Ljava/util/concurrent/CompletableFuture;+2
j  software.amazon.awssdk.services.s3.DefaultS3AsyncClient.deleteObject(Lsoftware/amazon/awssdk/services/s3/model/DeleteObjectRequest;)Ljava/util/concurrent/CompletableFuture;+151
j  software.amazon.awssdk.services.s3.DelegatingS3AsyncClient.lambda$deleteObject$19(Lsoftware/amazon/awssdk/services/s3/model/DeleteObjectRequest;)Ljava/util/concurrent/CompletableFuture;+5
j  software.amazon.awssdk.services.s3.DelegatingS3AsyncClient$$Lambda$843.apply(Ljava/lang/Object;)Ljava/lang/Object;+8
j  software.amazon.awssdk.services.s3.DelegatingS3AsyncClient.invokeOperation(Lsoftware/amazon/awssdk/services/s3/model/S3Request;Ljava/util/function/Function;)Ljava/util/concurrent/CompletableFuture;+2
j  software.amazon.awssdk.services.s3.DelegatingS3AsyncClient.deleteObject(Lsoftware/amazon/awssdk/services/s3/model/DeleteObjectRequest;)Ljava/util/concurrent/CompletableFuture;+8
j  kdc.cloudadapters.adapters.AbstractAmazonS3Adapter.deleteObject(Ljava/lang/String;)V+32
j  kdc.cloudadapters.CloudAdapterWrapper$17.handle()Ljava/lang/Void;+11
j  kdc.cloudadapters.CloudAdapterWrapper$17.handle()Ljava/lang/Object;+1
J 6191 C1 kdc.cloudadapters.CloudAdapterWrapper$CloudRetry.call()Ljava/lang/Object; (280 bytes) @ 0x00007f34c1f31bdc [0x00007f34c1f31a00+0x1dc]
J 6185 C1 kdc.util.Repeater.callHandleException()Ljava/lang/Object; (78 bytes) @ 0x00007f34c1f2e684 [0x00007f34c1f2e560+0x124]
J 6184 C1 kdc.util.Repeater.run()Ljava/lang/Object; (101 bytes) @ 0x00007f34c1f2db34 [0x00007f34c1f2d7c0+0x374]
J 6498 C1 kdc.cloudadapters.CloudAdapterWrapper$CloudRetry.execute()Ljava/lang/Object; (30 bytes) @ 0x00007f34c173ef44 [0x00007f34c173eec0+0x84]
j  kdc.cloudadapters.CloudAdapterWrapper.deleteObject(Ljava/lang/String;)V+13
j  kcfs.KcfsCloud.deleteObjInCloud(ILjava/lang/String;ZLjava/lang/String;Lkdc/file/EventLogBase$Operation;)Lkcfs/ChunkMigrationStatus;+89
j  kcfs.ChunkMigrator.performFileDelete(Ljava/lang/String;)Lkcfs/ChunkMigrationStatus;+150
J 3519 C1 kcfs.ChunkMigrator.performOperation()V (651 bytes) @ 0x00007f34c19df11c [0x00007f34c19ddfc0+0x115c]
J 3516 C1 kcfs.RequestProcessor$Task.call()Ljava/lang/Void; (154 bytes) @ 0x00007f34c19db86c [0x00007f34c19db280+0x5ec]
J 3434 C1 kcfs.RequestProcessor$Task.call()Ljava/lang/Object; (5 bytes) @ 0x00007f34c19aadcc [0x00007f34c19aad40+0x8c]
J 3076 C1 java.util.concurrent.FutureTask.run()V (126 bytes) @ 0x00007f34c18c24fc [0x00007f34c18c22a0+0x25c]
j  java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V+95
j  java.util.concurrent.ThreadPoolExecutor$Worker.run()V+5
j  java.lang.Thread.run()V+11
v  ~StubRoutines::call_stub
V  [libjvm.so+0x6d5a35]  JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)+0xcb5
V  [libjvm.so+0x6d315f]  JavaCalls::call_virtual(JavaValue*, KlassHandle, Symbol*, Symbol*, JavaCallArguments*, Thread*)+0x27f
V  [libjvm.so+0x6d374f]  JavaCalls::call_virtual(JavaValue*, Handle, KlassHandle, Symbol*, Symbol*, Thread*)+0x5f
V  [libjvm.so+0x78b9d2]  thread_entry(JavaThread*, Thread*)+0xb2
V  [libjvm.so+0xb9673e]  JavaThread::thread_main_inner()+0x22e
V  [libjvm.so+0x9cc080]  java_start(Thread*)+0x150
C  [libc.so.6+0x89c02]  start_thread+0x2d2

Regression Issue

• Select this option if this issue appears to be a regression.

Expected Behavior

No segmentation fault

Current Behavior

This leads to the crash of the JVM

Reproduction Steps

Not sure if it is reproducible. Attaching the entire hs_err_pid.log file. The issue was observed when the sdk was running as part of a jar on a Rocky Linux VM.

hs_err_pid_cloud5886_294274.log

Possible Solution

No response

Additional Information/Context

No response

AWS Java SDK version used

2.25.26

JDK version used

openjdk version "1.8.0_432"

Operating System and version

Rocky Linux release 9.5 (Blue Onyx)

[debora-ito]

Hi @souravagrawal29

Can you provide some additional info so we can have more context?

• I understand the issue is not easy to reproduce, but can you share a code sample showing how you create the s3 client and request?
• Can you provide the CRT logs of the moment of the error? Instructions can be found in our Dev Guide
• Does the issue only happen with DeleteObject?

[debora-ito]

Another question: do you have the same code running in a different JDK version? Does it throw SIGSEGV too? We're trying to narrow down if this can be a problem with JDK 8.

[souravagrawal29]

Hi @debora-ito , we only have the code running in JDK 8 and not in any other version of the JDK.

To answer your previous questions, following is a small snippet of how we initialize our async s3 client and the snippet used to delete an object

s3CrtAsyncClientBuilder = S3AsyncClient.crtBuilder()
                                                   .httpConfiguration(httpConfiguration)
                                                   .credentialsProvider(credentialsProvider)
                                                   .forcePathStyle(pathStyleAccess)
                                                   .checksumValidationEnabled(false)
                                                   .minimumPartSizeInBytes(CLOUD_ADAPTER_MULTIPART_COPY_PARTSIZE)
                                                   .thresholdInBytes(CLOUD_ADAPTER_MULTIPART_COPY_THRESHOLD);

if (adapterConfiguration.getNumThreads() != null) {
     s3CrtAsyncClientBuilder.maxConcurrency(adapterConfiguration.getNumThreads());
 }
s3CrtAsyncClientBuilder.crossRegionAccessEnabled(targetConfiguration.getStorageType().hasRegions());
s3CrtAsyncClientBuilder.region(regionType);
s3CrtAsyncClientBuilder.endpointOverride(endpointOverride);


    public void connect() {
        if (s3AsyncClient == null) {
            this.s3AsyncClient = s3CrtAsyncClientBuilder.build();
            this.transferManager = getTransferManager(s3AsyncClient);
            this.lastConnectionTimeMillis = System.currentTimeMillis();
        }
        if (s3Client == null) {
            this.s3Client = s3ClientBuilder.build();
        }
        if (s3Presigner == null) {
            this.s3Presigner = s3PresignerBuilder.s3Client(s3Client).build();
        }
        if (stsClient == null) {
            this.stsClient = stsClientBuilder.build();
        }
    }

Some of the configuration as part of the crt builder is conditional based on what configuration is used but this is all of it.

DeleteObject method

public void deleteObject(String key) throws CloudException {
        try {
            DeleteObjectRequest request = DeleteObjectRequest.builder()
                                                             .bucket(bucketName)
                                                             .key(key)
                                                             .build();
            s3AsyncClient.deleteObject(request).get();
        } catch (ExecutionException e) {
            throwCloudException(e, key, "deleteObject");
        } catch (Exception e) {
            log().error("Unable to delete object for key={}; bucket={}", key, bucketName, e);
            throw new CloudException(e);
        }
    }