From af4822f4335133c09289c543506942fbc18ea1ff Mon Sep 17 00:00:00 2001 From: Josh Humphries <2035234+jhump@users.noreply.github.com> Date: Mon, 29 Jan 2024 21:08:30 -0500 Subject: [PATCH] add VerbosePrinter and verbose output up to verbosity level 5, including okhttp event logger and logging decorator for http client --- .../conformance/client/java/JavaHelpers.kt | 6 +- .../conformance/client/java/Main.kt | 2 +- .../client/javalite/JavaLiteHelpers.kt | 6 +- .../conformance/client/javalite/Main.kt | 2 +- .../connectrpc/conformance/client/Client.kt | 25 +++- .../conformance/client/ClientArgs.kt | 17 ++- .../client/ConformanceClientLoop.kt | 40 +++--- .../conformance/client/OkHttpEventTracer.kt | 83 ++++++++++++ .../conformance/client/TracingHTTPClient.kt | 118 ++++++++++++++++++ .../conformance/client/VerbosePrinter.kt | 112 +++++++++++++++++ .../client/adapt/ClientCompatRequest.kt | 2 + .../client/adapt/ClientResponseResult.kt | 5 + 12 files changed, 395 insertions(+), 23 deletions(-) create mode 100644 conformance/client/src/main/kotlin/com/connectrpc/conformance/client/OkHttpEventTracer.kt create mode 100644 conformance/client/src/main/kotlin/com/connectrpc/conformance/client/TracingHTTPClient.kt create mode 100644 conformance/client/src/main/kotlin/com/connectrpc/conformance/client/VerbosePrinter.kt diff --git a/conformance/client/google-java/src/main/kotlin/com/connectrpc/conformance/client/java/JavaHelpers.kt b/conformance/client/google-java/src/main/kotlin/com/connectrpc/conformance/client/java/JavaHelpers.kt index 32a8892a..4233fa1e 100644 --- a/conformance/client/google-java/src/main/kotlin/com/connectrpc/conformance/client/java/JavaHelpers.kt +++ b/conformance/client/google-java/src/main/kotlin/com/connectrpc/conformance/client/java/JavaHelpers.kt @@ -84,7 +84,9 @@ class JavaHelpers { if (err != null) { respBuilder.setError(toProtoError(err)) } - builder.setResponse(respBuilder) + val respMsg = respBuilder.build() + result.response.raw = respMsg + builder.setResponse(respMsg) } is ClientCompatResponse.Result.ErrorResult -> { builder.setError( @@ -166,6 +168,8 @@ class JavaHelpers { private class ClientCompatRequestImpl( private val msg: com.connectrpc.conformance.v1.ClientCompatRequest, ) : ClientCompatRequest { + override val raw: kotlin.Any + get() = msg override val testName: String get() = msg.testName override val service: String diff --git a/conformance/client/google-java/src/main/kotlin/com/connectrpc/conformance/client/java/Main.kt b/conformance/client/google-java/src/main/kotlin/com/connectrpc/conformance/client/java/Main.kt index 12be5f59..57d99859 100644 --- a/conformance/client/google-java/src/main/kotlin/com/connectrpc/conformance/client/java/Main.kt +++ b/conformance/client/google-java/src/main/kotlin/com/connectrpc/conformance/client/java/Main.kt @@ -23,7 +23,7 @@ fun main(args: Array) { val loop = ConformanceClientLoop( JavaHelpers::unmarshalRequest, JavaHelpers::marshalResponse, - clientArgs.verbosity, + clientArgs.verbose, ) val client = Client( args = clientArgs, diff --git a/conformance/client/google-javalite/src/main/kotlin/com/connectrpc/conformance/client/javalite/JavaLiteHelpers.kt b/conformance/client/google-javalite/src/main/kotlin/com/connectrpc/conformance/client/javalite/JavaLiteHelpers.kt index 62da7e72..84c9d5fb 100644 --- a/conformance/client/google-javalite/src/main/kotlin/com/connectrpc/conformance/client/javalite/JavaLiteHelpers.kt +++ b/conformance/client/google-javalite/src/main/kotlin/com/connectrpc/conformance/client/javalite/JavaLiteHelpers.kt @@ -76,7 +76,9 @@ class JavaLiteHelpers { if (err != null) { respBuilder.setError(toProtoError(err)) } - builder.setResponse(respBuilder) + val respMsg = respBuilder.build() + result.response.raw = respMsg + builder.setResponse(respMsg) } is ClientCompatResponse.Result.ErrorResult -> { builder.setError( @@ -148,6 +150,8 @@ class JavaLiteHelpers { private class ClientCompatRequestImpl( private val msg: com.connectrpc.conformance.v1.ClientCompatRequest, ) : ClientCompatRequest { + override val raw: kotlin.Any + get() = msg override val testName: String get() = msg.testName override val service: String diff --git a/conformance/client/google-javalite/src/main/kotlin/com/connectrpc/conformance/client/javalite/Main.kt b/conformance/client/google-javalite/src/main/kotlin/com/connectrpc/conformance/client/javalite/Main.kt index edb11619..1a6c76ad 100644 --- a/conformance/client/google-javalite/src/main/kotlin/com/connectrpc/conformance/client/javalite/Main.kt +++ b/conformance/client/google-javalite/src/main/kotlin/com/connectrpc/conformance/client/javalite/Main.kt @@ -23,7 +23,7 @@ fun main(args: Array) { val loop = ConformanceClientLoop( JavaLiteHelpers::unmarshalRequest, JavaLiteHelpers::marshalResponse, - clientArgs.verbosity, + clientArgs.verbose, ) val client = Client( args = clientArgs, diff --git a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/Client.kt b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/Client.kt index c461088f..a10820ce 100644 --- a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/Client.kt +++ b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/Client.kt @@ -36,6 +36,7 @@ import com.connectrpc.conformance.client.adapt.Invoker import com.connectrpc.conformance.client.adapt.ResponseStream import com.connectrpc.conformance.client.adapt.ServerStreamClient import com.connectrpc.conformance.client.adapt.UnaryClient +import com.connectrpc.http.HTTPClientInterface import com.connectrpc.impl.ProtocolClient import com.connectrpc.okhttp.ConnectOkHttpClient import com.connectrpc.protocols.GETConfiguration @@ -45,6 +46,7 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.launch import okhttp3.OkHttpClient +import okhttp3.Protocol import okhttp3.tls.HandshakeCertificates import okhttp3.tls.HeldCertificate import java.security.KeyFactory @@ -172,6 +174,10 @@ class Client( try { stream.send(msg) } catch (ex: Exception) { + args.verbose.verbosity(2) { + println("Failed to send request message:") + indent().println(ex.stackTraceToString()) + } numUnsent = req.requestMessages.size - i break } @@ -277,6 +283,10 @@ class Client( try { stream.requests.send(msg) } catch (ex: Exception) { + args.verbose.verbosity(2) { + println("Failed to send request message:") + indent().println(ex.stackTraceToString()) + } numUnsent = req.requestMessages.size - i break } @@ -318,6 +328,10 @@ class Client( try { stream.requests.send(msg) } catch (ex: Exception) { + args.verbose.verbosity(2) { + println("Failed to send request message:") + indent().println(ex.stackTraceToString()) + } // Ignore. We should see it again below when we receive the response. } @@ -447,6 +461,11 @@ class Client( var clientBuilder = OkHttpClient.Builder() .protocols(asOkHttpProtocols(req.httpVersion, useTls)) .connectTimeout(Duration.ofMinutes(1)) + + args.verbose.withPrefix("okhttp3 events: ").verbosity(4) { + clientBuilder = clientBuilder.eventListener(OkHttpEventTracer(this)) + } + if (useTls) { val certs = certs(req) clientBuilder = clientBuilder.sslSocketFactory(certs.sslSocketFactory(), certs.trustManager) @@ -469,10 +488,14 @@ class Client( emptyList() } val httpClient = clientBuilder.build() + var connectHttpClient: HTTPClientInterface = ConnectOkHttpClient(httpClient) + args.verbose.withPrefix("http client interface: ").verbosity(3) { + connectHttpClient = TracingHTTPClient(connectHttpClient, this) + } return Pair( httpClient, ProtocolClient( - httpClient = ConnectOkHttpClient(httpClient), + httpClient = connectHttpClient, ProtocolClientConfig( host = host, serializationStrategy = serializationStrategy, diff --git a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/ClientArgs.kt b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/ClientArgs.kt index f5662e8a..5c8c2310 100644 --- a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/ClientArgs.kt +++ b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/ClientArgs.kt @@ -18,7 +18,7 @@ import com.connectrpc.conformance.client.adapt.UnaryClient.InvokeStyle data class ClientArgs( val invokeStyle: InvokeStyle, - val verbosity: Int, + val verbose: VerbosePrinter, ) { companion object { fun parseArgs(args: Array): ClientArgs { @@ -53,7 +53,18 @@ data class ClientArgs( } } "-v" -> { - verbosity = 1 + // see if there's a value + if (i < args.size - 1 && !args[i + 1].startsWith("-")) { + skip = true // consuming next string now + val v = args[i + 1] + val intVal = v.toIntOrNull() + if (intVal == null || intVal < 1 || intVal > 5) { + throw RuntimeException("value for $arg option should be an integer between 1 and 5; instead got '$v'") + } + verbosity = intVal + } else { + verbosity = 1 + } } "-vv" -> { verbosity = 2 @@ -70,7 +81,7 @@ data class ClientArgs( } } } - return ClientArgs(invokeStyle, verbosity) + return ClientArgs(invokeStyle, VerbosePrinter(verbosity, "* client: ")) } } } diff --git a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/ConformanceClientLoop.kt b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/ConformanceClientLoop.kt index 8c0d148f..6ea90e1a 100644 --- a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/ConformanceClientLoop.kt +++ b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/ConformanceClientLoop.kt @@ -30,39 +30,43 @@ import java.io.OutputStream class ConformanceClientLoop( private val requestUnmarshaller: (ByteArray) -> ClientCompatRequest, private val responseMarshaller: (ClientCompatResponse) -> ByteArray, - private val verbosity: Int = 0, + private val verbose: VerbosePrinter, ) { fun run(input: InputStream, output: OutputStream, client: Client) = runBlocking { // TODO: issue RPCs in parallel while (true) { var result: ClientCompatResponse.Result val req = readRequest(input) ?: return@runBlocking // end of stream - if (verbosity > 0) { - System.err.println("* client: read request for test ${req.testName}") + verbose.verbosity(1) { + println("read request for test ${req.testName}") + verbose.verbosity(3) { + println("RPC request:") + indent().println("${req.raw}") + } } try { val resp = client.handle(req) result = ClientCompatResponse.Result.ResponseResult(resp) - if (verbosity > 0) { - System.err.println("* client: RPC completed for test ${req.testName}") + verbose.verbosity(1) { + println("RPC completed for test ${req.testName}") } - } catch (e: Exception) { - if (verbosity > 0) { - System.err.println("* client: RPC could not be issued for test ${req.testName}") - e.printStackTrace() + } catch (ex: Exception) { + verbose.verbosity(1) { + println("RPC could not be issued for test ${req.testName}") + indent().println(ex.stackTraceToString()) } - val msg = if (e.message.orEmpty() == "") { - e::class.qualifiedName.orEmpty() + val msg = if (ex.message.orEmpty() == "") { + ex::class.qualifiedName.orEmpty() } else { - "${e::class.qualifiedName}: ${e.message}" + "${ex::class.qualifiedName}: ${ex.message}" } result = ClientCompatResponse.Result.ErrorResult(msg) } if (result is ClientCompatResponse.Result.ResponseResult && result.response.error != null) { - if (verbosity > 2) { + verbose.verbosity(2) { val ex = result.response.error!! - System.err.println("* client: RPC failed with code ${ex.code}") - ex.printStackTrace() + println("RPC failed with code ${ex.code} for test ${req.testName}:") + indent().println(ex.stackTraceToString()) } } writeResponse( @@ -72,6 +76,12 @@ class ConformanceClientLoop( result = result, ), ) + if (result is ClientCompatResponse.Result.ResponseResult && result.response.raw != null) { + verbose.verbosity(3) { + println("RPC result:") + indent().println("${result.response.raw}") + } + } } } diff --git a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/OkHttpEventTracer.kt b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/OkHttpEventTracer.kt new file mode 100644 index 00000000..8a050272 --- /dev/null +++ b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/OkHttpEventTracer.kt @@ -0,0 +1,83 @@ +// Copyright 2022-2023 The Connect Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.connectrpc.conformance.client + +import okhttp3.Call +import okhttp3.Connection +import okhttp3.EventListener +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import java.io.IOException +import java.net.InetSocketAddress +import java.net.Proxy + +internal class OkHttpEventTracer( + private val printer: VerbosePrinter.Printer, +) : EventListener() { + override fun connectStart(call: Call, inetSocketAddress: InetSocketAddress, proxy: Proxy) { + printer.printlnWithStackTrace("connecting to $inetSocketAddress...") + } + override fun connectEnd( + call: Call, + inetSocketAddress: InetSocketAddress, + proxy: Proxy, + protocol: Protocol?, + ) { + printer.printlnWithStackTrace("connected to $inetSocketAddress") + } + override fun connectFailed( + call: Call, + inetSocketAddress: InetSocketAddress, + proxy: Proxy, + protocol: Protocol?, + ioe: IOException, + ) { + printer.printlnWithStackTrace("connect to $inetSocketAddress failed") + } + override fun connectionAcquired(call: Call, connection: Connection) { + printer.printlnWithStackTrace("connection to ${connection.socket().remoteSocketAddress} acquired") + } + override fun requestHeadersStart(call: Call) { + printer.printlnWithStackTrace("writing request headers...") + } + override fun requestHeadersEnd(call: Call, request: Request) { + printer.printlnWithStackTrace("request headers written") + } + override fun requestBodyStart(call: Call) { + printer.printlnWithStackTrace("writing request body...") + } + override fun requestBodyEnd(call: Call, byteCount: Long) { + printer.printlnWithStackTrace("request body written: $byteCount bytes") + } + override fun requestFailed(call: Call, ioe: IOException) { + printer.printlnWithStackTrace("request failed: ${ioe.message}") + } + override fun responseHeadersStart(call: Call) { + printer.printlnWithStackTrace("reading response headers...") + } + override fun responseHeadersEnd(call: Call, response: Response) { + printer.printlnWithStackTrace("response headers read: status code = ${response.code}") + } + override fun responseBodyStart(call: Call) { + printer.printlnWithStackTrace("reading response body...") + } + override fun responseBodyEnd(call: Call, byteCount: Long) { + printer.printlnWithStackTrace("response body read: $byteCount bytes") + } + override fun responseFailed(call: Call, ioe: IOException) { + printer.printlnWithStackTrace("response failed: ${ioe.message}") + } +} diff --git a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/TracingHTTPClient.kt b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/TracingHTTPClient.kt new file mode 100644 index 00000000..f20acd70 --- /dev/null +++ b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/TracingHTTPClient.kt @@ -0,0 +1,118 @@ +// Copyright 2022-2023 The Connect Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.connectrpc.conformance.client + +import com.connectrpc.StreamResult +import com.connectrpc.http.Cancelable +import com.connectrpc.http.HTTPClientInterface +import com.connectrpc.http.HTTPRequest +import com.connectrpc.http.HTTPResponse +import com.connectrpc.http.Stream +import com.connectrpc.http.UnaryHTTPRequest +import okio.Buffer + +internal class TracingHTTPClient( + private val delegate: HTTPClientInterface, + private val printer: VerbosePrinter.Printer, +) : HTTPClientInterface { + override fun unary(request: UnaryHTTPRequest, onResult: (HTTPResponse) -> Unit): Cancelable { + printer.printlnWithStackTrace("Sending unary request (${request.message.size} bytes): ${request.httpMethod} ${request.url}") + val cancel = delegate.unary(request) { response -> + val buffer = Buffer() + buffer.writeAll(response.message) + if (response.cause != null) { + printer.println("Failed to receive HTTP response (${buffer.size} bytes): ${response.cause!!.message.orEmpty()}") + printer.indent().println(response.cause!!.stackTraceToString()) + } else { + printer.println("Received HTTP response (${buffer.size} bytes): ${response.tracingInfo?.httpStatus ?: "???"}") + } + onResult( + HTTPResponse( + code = response.code, + headers = response.headers, + message = buffer, + trailers = response.trailers, + tracingInfo = response.tracingInfo, + cause = response.cause, + ), + ) + } + return { + printer.println("Canceling HTTP request...") + cancel() + } + } + + override fun stream( + request: HTTPRequest, + duplex: Boolean, + onResult: suspend (StreamResult) -> Unit, + ): Stream { + printer.printlnWithStackTrace("Sending HTTP stream request: POST ${request.url}") + val stream = delegate.stream(request, duplex) { result -> + when (result) { + is StreamResult.Headers -> { + printer.printlnWithStackTrace("Received HTTP response headers") + } + is StreamResult.Message -> { + printer.printlnWithStackTrace("Received HTTP response data (${result.message.size} bytes)") + } + is StreamResult.Complete -> { + if (result.cause != null) { + printer.printlnWithStackTrace("Failed to complete HTTP response (code=${result.code}): ${result.cause!!.message.orEmpty()}") + } else { + printer.printlnWithStackTrace("Received HTTP response completion: code=${result.code}") + } + } + } + onResult(result) + } + return TracingStream(stream, printer) + } + + private class TracingStream( + private val delegate: Stream, + private val printer: VerbosePrinter.Printer, + ) : Stream { + override suspend fun send(buffer: Buffer): Result { + val size = buffer.size + val res = delegate.send(buffer) + if (res.isFailure) { + printer.printlnWithStackTrace("Failed to send HTTP request data ($size bytes): ${res.exceptionOrNull()!!.message}") + } else { + printer.printlnWithStackTrace("Sent HTTP request data ($size bytes)") + } + return res + } + + override fun sendClose() { + printer.printlnWithStackTrace("Half-closing stream") + delegate.sendClose() + } + + override fun receiveClose() { + printer.printlnWithStackTrace("Closing stream") + delegate.receiveClose() + } + + override fun isSendClosed(): Boolean { + return delegate.isSendClosed() + } + + override fun isReceiveClosed(): Boolean { + return delegate.isReceiveClosed() + } + } +} diff --git a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/VerbosePrinter.kt b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/VerbosePrinter.kt new file mode 100644 index 00000000..93bc2d45 --- /dev/null +++ b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/VerbosePrinter.kt @@ -0,0 +1,112 @@ +// Copyright 2022-2023 The Connect Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.connectrpc.conformance.client + +/** + * Helper for printing verbose output. This can be useful + * for troubleshooting and debugging. + */ +class VerbosePrinter( + private val verbosity: Int, + private val prefix: String, +) { + companion object { + /** + * Verbosity required to log optional stack traces. + */ + private const val STACK_TRACE_VERBOSITY = 5 + + /** + * Lock used to synchronize all output so that concurrent + * threads printing don't interleave optional stack traces + * (which makes them much harder to read) + */ + private val lock = Object() + } + + private val output = PrinterImpl(verbosity, prefix) + + /** + * Runs the given block with the given verbosity. If the + * given verbosity is higher than the currently configured + * output, the block will not be executed. + */ + fun verbosity(v: Int, block: Printer.() -> Unit) { + if (v > verbosity) return + block(output) + } + + /** + * Returns a new printer with the given additional prefix + * added to each printed line. + */ + fun withPrefix(prefix: String): VerbosePrinter { + return VerbosePrinter(verbosity, this.prefix + prefix) + } + + /** + * Prints verbose messages. + */ + interface Printer { + fun println(s: String) + + /** + * Like `println` but will also record a stack trace + * of the caller if verbosity is sufficiently high. + */ + fun printlnWithStackTrace(s: String) + + /** + * Returns a printer whose output is prefixed with an + * extra indentation (tab stop). + */ + fun indent(): Printer + } + + private class PrinterImpl( + private val verbosity: Int, + private val prefix: String, + ) : Printer { + override fun println(s: String) { + synchronized(lock) { + s.splitToSequence("\n").forEach { + System.err.println("${prefix}$it") + } + } + } + override fun printlnWithStackTrace(s: String) { + synchronized(lock) { + println(s) + if (verbosity < STACK_TRACE_VERBOSITY) { + return + } + println( + RuntimeException() + .stackTraceToString() + // Skip first two lines. First one is the exception + // type and message. Second one is stack-frame for + // this function. + .substringAfter("\n") + .substringAfter("\n") + // Trim trailing newline. + .trimEnd('\n'), + ) + } + } + override fun indent(): Printer { + return PrinterImpl(verbosity, prefix + "\t") + } + } +} diff --git a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/adapt/ClientCompatRequest.kt b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/adapt/ClientCompatRequest.kt index ab8f3002..219a7115 100644 --- a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/adapt/ClientCompatRequest.kt +++ b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/adapt/ClientCompatRequest.kt @@ -56,6 +56,8 @@ interface ClientCompatRequest { val requestMessages: List val cancel: Cancel? + val raw: Any // the underlying message or object + interface TlsCreds { val cert: ByteString val key: ByteString diff --git a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/adapt/ClientResponseResult.kt b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/adapt/ClientResponseResult.kt index 1d8c5724..4680276a 100644 --- a/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/adapt/ClientResponseResult.kt +++ b/conformance/client/src/main/kotlin/com/connectrpc/conformance/client/adapt/ClientResponseResult.kt @@ -31,4 +31,9 @@ class ClientResponseResult( val trailers: Headers = emptyMap(), val error: ConnectException? = null, val numUnsentRequests: Int = 0, + + /** + * Set to underlying message or object during marshal. + */ + var raw: Any? = null, )