diff --git a/fhevm-engine/Cargo.lock b/fhevm-engine/Cargo.lock index f57160fb..e2931421 100644 --- a/fhevm-engine/Cargo.lock +++ b/fhevm-engine/Cargo.lock @@ -104,7 +104,7 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio", + "mio 1.0.2", "socket2", "tokio", "tracing", @@ -1770,6 +1770,9 @@ dependencies = [ "lazy_static", "log", "lru", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "prometheus", "prost", "rand", @@ -1786,6 +1789,7 @@ dependencies = [ "tonic", "tonic-build", "tonic-health", + "tonic-tracing-opentelemetry", "tonic-types", "tonic-web", ] @@ -3224,6 +3228,17 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "mio" version = "1.0.2" @@ -3446,6 +3461,71 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803801d3d3b71cd026851a53f974ea03df3d179cb758b260136a6c9e22e196af" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "596b1719b3cab83addb20bcbffdf21575279d9436d9ccccfe651a3bf0ab5ab06" +dependencies = [ + "async-trait", + "futures-core", + "http 1.1.0", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "thiserror", + "tokio", + "tonic", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c43620e8f93359eb7e627a3b16ee92d8585774986f24f2ab010817426c5ce61" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0da0d6b47a3dbc6e9c9e36a0520e25cf943e046843818faaa3f87365a548c82" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "percent-encoding", + "rand", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -4579,6 +4659,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -5285,6 +5374,16 @@ dependencies = [ "syn 2.0.75", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "threadpool" version = "1.8.1" @@ -5351,27 +5450,28 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.3" +version = "1.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" +checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" dependencies = [ "backtrace", "bytes", "libc", - "mio", + "mio 0.8.11", + "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", @@ -5500,6 +5600,26 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-tracing-opentelemetry" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6189e979896bf663c2478b4b4d04278c356a02dc8d80228d002e5535fe976e68" +dependencies = [ + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "hyper", + "opentelemetry", + "pin-project-lite", + "tonic", + "tower 0.5.1", + "tracing", + "tracing-opentelemetry", + "tracing-opentelemetry-instrumentation-sdk", +] + [[package]] name = "tonic-types" version = "0.12.1" @@ -5635,6 +5755,59 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eabc56d23707ad55ba2a0750fc24767125d5a0f51993ba41ad2c441cc7b8dea" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-opentelemetry-instrumentation-sdk" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612243becba145498d15c8ff9d41f333ca27b343ee874909d964cc2aed3a013f" +dependencies = [ + "http 1.1.0", + "opentelemetry", + "tracing", + "tracing-opentelemetry", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", ] [[package]] @@ -5949,6 +6122,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/fhevm-engine/coprocessor/Cargo.toml b/fhevm-engine/coprocessor/Cargo.toml index 9ae9f7ce..d8e30fd3 100644 --- a/fhevm-engine/coprocessor/Cargo.toml +++ b/fhevm-engine/coprocessor/Cargo.toml @@ -35,6 +35,10 @@ prometheus.workspace = true log.workspace = true structured-logger = "1.0.3" actix-web = "4.9.0" +opentelemetry = "0.25" +opentelemetry-otlp = "0.25" +opentelemetry_sdk = { version = "0.25", features = ["rt-tokio"] } +tonic-tracing-opentelemetry = "0.21" [dev-dependencies] testcontainers = "0.21" diff --git a/fhevm-engine/coprocessor/src/main.rs b/fhevm-engine/coprocessor/src/main.rs index f262482d..1c2c501d 100644 --- a/fhevm-engine/coprocessor/src/main.rs +++ b/fhevm-engine/coprocessor/src/main.rs @@ -10,6 +10,7 @@ mod tfhe_worker; mod types; mod utils; mod metrics; +mod tracing; fn main() { let args = crate::cli::parse_args(); @@ -64,6 +65,10 @@ async fn async_main( .with_default_writer(structured_logger::async_json::new_writer(tokio::io::stdout())) .init(); + if let Err(err) = tracing::setup_tracing() { + panic!("Error while initializing tracing: {:?}", err); + } + let mut set = JoinSet::new(); if args.run_server { log::info!(target: "async_main", "Initializing api server"); diff --git a/fhevm-engine/coprocessor/src/server.rs b/fhevm-engine/coprocessor/src/server.rs index bafcdead..abebdb41 100644 --- a/fhevm-engine/coprocessor/src/server.rs +++ b/fhevm-engine/coprocessor/src/server.rs @@ -132,6 +132,7 @@ pub async fn run_server_iteration( }; Server::builder() + .layer(tonic_tracing_opentelemetry::middleware::server::OtelGrpcLayer::default()) .add_service( crate::server::coprocessor::fhevm_coprocessor_server::FhevmCoprocessorServer::new( service, diff --git a/fhevm-engine/coprocessor/src/tfhe_worker.rs b/fhevm-engine/coprocessor/src/tfhe_worker.rs index 68c7a9d0..4055d470 100644 --- a/fhevm-engine/coprocessor/src/tfhe_worker.rs +++ b/fhevm-engine/coprocessor/src/tfhe_worker.rs @@ -5,6 +5,8 @@ use fhevm_engine_common::{ types::SupportedFheOperations, }; use lazy_static::lazy_static; +use opentelemetry::KeyValue; +use opentelemetry::trace::{Span, TraceContextExt, Tracer}; use prometheus::{register_int_counter, IntCounter}; use sqlx::{postgres::PgListener, query, Acquire}; use std::{ @@ -58,6 +60,8 @@ pub async fn run_tfhe_worker( async fn tfhe_worker_cycle( args: &crate::cli::Args, ) -> Result<(), Box> { + let tracer = opentelemetry::global::tracer("tfhe_worker"); + let tenant_key_cache: std::sync::Arc>> = std::sync::Arc::new(tokio::sync::RwLock::new(lru::LruCache::new( NonZeroUsize::new(args.tenant_key_cache_size as usize).unwrap(), @@ -88,10 +92,17 @@ async fn tfhe_worker_cycle( }; } + let loop_span = tracer.start("worker_iteration"); + let loop_ctx = opentelemetry::Context::current_with_span(loop_span); + let mut s = tracer.start_with_context("acquire_connection", &loop_ctx); let mut conn = pool.acquire().await?; + s.end(); + let mut s = tracer.start_with_context("begin_transaction", &loop_ctx); let mut trx = conn.begin().await?; + s.end(); // this query locks our work items so other worker doesn't select them + let mut s = tracer.start_with_context("query_work_items", &loop_ctx); let mut the_work = query!( " SELECT tenant_id, output_handle, dependencies, fhe_operation, is_scalar @@ -117,6 +128,8 @@ async fn tfhe_worker_cycle( ) .fetch_all(trx.as_mut()) .await?; + s.set_attribute(KeyValue::new("count", the_work.len() as i64)); + s.end(); immedially_poll_more_work = !the_work.is_empty(); @@ -127,10 +140,13 @@ async fn tfhe_worker_cycle( WORK_ITEMS_FOUND_COUNTER.inc_by(the_work.len() as u64); log::info!(target: "tfhe_worker", count = the_work.len(); "Processing work items"); + // make sure we process each tenant sequentially not to // load different keys in cache by different tenants the_work.sort_by_key(|k| k.tenant_id); + let mut s = tracer.start_with_context("populate_key_cache", &loop_ctx); + let mut cts_to_query: BTreeSet<&[u8]> = BTreeSet::new(); let mut tenants_to_query: BTreeSet = BTreeSet::new(); let mut keys_to_query: BTreeSet = BTreeSet::new(); @@ -153,8 +169,14 @@ async fn tfhe_worker_cycle( let tenants_to_query = tenants_to_query.into_iter().collect::>(); let keys_to_query = keys_to_query.into_iter().collect::>(); + s.set_attribute(KeyValue::new("keys_to_query", keys_to_query.len() as i64)); + s.set_attribute(KeyValue::new("tenants_to_query", tenants_to_query.len() as i64)); + populate_cache_with_tenant_keys(keys_to_query, trx.as_mut(), &tenant_key_cache).await?; + s.end(); + let mut s = tracer.start_with_context("query_ciphertext_batch", &loop_ctx); + s.set_attribute(KeyValue::new("cts_to_query", cts_to_query.len() as i64)); // TODO: select all the ciphertexts where they're contained in the tuples let ciphertexts_rows = query!( " @@ -169,6 +191,8 @@ async fn tfhe_worker_cycle( .fetch_all(trx.as_mut()) .await?; + s.end(); + // index ciphertexts in hashmap let mut ciphertext_map: HashMap<(i32, &[u8]), _> = HashMap::with_capacity(ciphertexts_rows.len()); @@ -176,9 +200,13 @@ async fn tfhe_worker_cycle( let _ = ciphertext_map.insert((row.tenant_id, &row.handle), row); } + let mut s = tracer.start_with_context("schedule_fhe_work", &loop_ctx); + s.set_attribute(KeyValue::new("work_items", the_work.len() as i64)); + let mut tfhe_work_set = tokio::task::JoinSet::new(); // process every tenant by tenant id because we must switch keys for each tenant for w in the_work { + let tenant_key_cache = tenant_key_cache.clone(); let fhe_op: SupportedFheOperations = w .fhe_operation @@ -202,6 +230,7 @@ async fn tfhe_worker_cycle( } // copy for setting error in database + let mut s = tracer.start_with_context("tfhe_computation", &loop_ctx); tfhe_work_set.spawn_blocking( move || -> Result<_, (Box<(dyn std::error::Error + Send + Sync)>, i32, Vec)> { // set the server key if not set @@ -252,15 +281,27 @@ async fn tfhe_worker_cycle( })?; let (db_type, db_bytes) = res.compress(); + s.set_attribute(KeyValue::new("fhe_operation", w.fhe_operation as i64)); + s.set_attribute(KeyValue::new("handle", format!("0x{}", hex::encode(&w.output_handle)))); + s.set_attribute(KeyValue::new("output_type", db_type as i64)); + let input_types = deserialized_cts.iter().map(|i| i.type_num().to_string()).collect::>().join(","); + s.set_attribute(KeyValue::new("input_types", input_types)); + s.end(); Ok((w, db_type, db_bytes)) }, ); } + s.end(); + let mut s_outer = tracer.start_with_context("wait_and_update_fhe_work", &loop_ctx); while let Some(output) = tfhe_work_set.join_next().await { let finished_work_unit = output?; match finished_work_unit { Ok((w, db_type, db_bytes)) => { + let mut s = tracer.start_with_context("insert_ct_into_db", &loop_ctx); + s.set_attribute(KeyValue::new("tenant_id", w.tenant_id as i64)); + s.set_attribute(KeyValue::new("handle", format!("0x{}", hex::encode(&w.output_handle)))); + s.set_attribute(KeyValue::new("ciphertext_type", db_type as i64)); let _ = query!(" INSERT INTO ciphertexts(tenant_id, handle, ciphertext, ciphertext_version, ciphertext_type) VALUES($1, $2, $3, $4, $5) @@ -268,6 +309,11 @@ async fn tfhe_worker_cycle( ", w.tenant_id, w.output_handle, &db_bytes, current_ciphertext_version(), db_type) .execute(trx.as_mut()) .await?; + s.end(); + let mut s = tracer.start_with_context("update_computation", &loop_ctx); + s.set_attribute(KeyValue::new("tenant_id", w.tenant_id as i64)); + s.set_attribute(KeyValue::new("handle", format!("0x{}", hex::encode(&w.output_handle)))); + s.set_attribute(KeyValue::new("ciphertext_type", db_type as i64)); let _ = query!( " UPDATE computations @@ -280,6 +326,7 @@ async fn tfhe_worker_cycle( ) .execute(trx.as_mut()) .await?; + s.end(); WORK_ITEMS_PROCESSED_COUNTER.inc(); } Err((err, tenant_id, output_handle)) => { @@ -290,6 +337,11 @@ async fn tfhe_worker_cycle( error = err.to_string(); "error while processing work item" ); + let mut s = tracer.start_with_context("set_computation_error_in_db", &loop_ctx); + s.set_attribute(KeyValue::new("tenant_id", tenant_id as i64)); + s.set_attribute(KeyValue::new("handle", format!("0x{}", hex::encode(&output_handle)))); + let err_string = err.to_string(); + s.set_status(opentelemetry::trace::Status::Error { description: err_string.clone().into() }); let _ = query!( " UPDATE computations @@ -297,16 +349,20 @@ async fn tfhe_worker_cycle( WHERE tenant_id = $2 AND output_handle = $3 ", - err.to_string(), + err_string, tenant_id, output_handle ) .execute(trx.as_mut()) .await?; + s.end(); } } } + s_outer.end(); trx.commit().await?; + + let _guard = loop_ctx.attach(); } } diff --git a/fhevm-engine/fhevm-engine-common/src/types.rs b/fhevm-engine/fhevm-engine-common/src/types.rs index 9ae9f552..ebe7f21c 100644 --- a/fhevm-engine/fhevm-engine-common/src/types.rs +++ b/fhevm-engine/fhevm-engine-common/src/types.rs @@ -390,7 +390,8 @@ impl SupportedFheCiphertexts { SupportedFheCiphertexts::FheBytes128(_) => 10, SupportedFheCiphertexts::FheBytes256(_) => 11, SupportedFheCiphertexts::Scalar(_) => { - panic!("we should never need to serialize scalar") + // need this for tracing as we join types of computation for a trace + 200 } } } diff --git a/fhevm-engine/fhevm-go-coproc/fhevm/api.go b/fhevm-engine/fhevm-go-coproc/fhevm/api.go index 6b935615..7a1c7a0d 100644 --- a/fhevm-engine/fhevm-go-coproc/fhevm/api.go +++ b/fhevm-engine/fhevm-go-coproc/fhevm/api.go @@ -442,7 +442,7 @@ func scheduleCoprocessorFlushes(impl *ApiImpl) { itemsComputed, err := flushWorkItemsToCoprocessor(impl.store) if err != nil { fmt.Printf("error flushing work items to coprocessor: %s\n", err) - } else { + } else if itemsComputed > 0 { fmt.Printf("successfully sent %d work items to the coprocessor\n", itemsComputed) } } diff --git a/fhevm-engine/fhevm-go-coproc/fhevm/coprocessor.pb.go b/fhevm-engine/fhevm-go-coproc/fhevm/coprocessor.pb.go index 2b63e264..148039a3 100644 --- a/fhevm-engine/fhevm-go-coproc/fhevm/coprocessor.pb.go +++ b/fhevm-engine/fhevm-go-coproc/fhevm/coprocessor.pb.go @@ -493,7 +493,7 @@ type InputToUpload struct { InputPayload []byte `protobuf:"bytes,1,opt,name=input_payload,json=inputPayload,proto3" json:"input_payload,omitempty"` ContractAddress string `protobuf:"bytes,2,opt,name=contract_address,json=contractAddress,proto3" json:"contract_address,omitempty"` - CallerAddress string `protobuf:"bytes,3,opt,name=caller_address,json=callerAddress,proto3" json:"caller_address,omitempty"` + UserAddress string `protobuf:"bytes,3,opt,name=user_address,json=userAddress,proto3" json:"user_address,omitempty"` Signature []byte `protobuf:"bytes,4,opt,name=signature,proto3" json:"signature,omitempty"` } @@ -543,9 +543,9 @@ func (x *InputToUpload) GetContractAddress() string { return "" } -func (x *InputToUpload) GetCallerAddress() string { +func (x *InputToUpload) GetUserAddress() string { if x != nil { - return x.CallerAddress + return x.UserAddress } return "" } @@ -617,11 +617,13 @@ type InputCiphertextResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - InputHandles []*InputCiphertextResponseHandle `protobuf:"bytes,1,rep,name=input_handles,json=inputHandles,proto3" json:"input_handles,omitempty"` - Eip712ContractAddress string `protobuf:"bytes,2,opt,name=eip712ContractAddress,proto3" json:"eip712ContractAddress,omitempty"` - Eip712CallerAddress string `protobuf:"bytes,3,opt,name=eip712CallerAddress,proto3" json:"eip712CallerAddress,omitempty"` - Eip712SignerAddress string `protobuf:"bytes,4,opt,name=eip712SignerAddress,proto3" json:"eip712SignerAddress,omitempty"` - Eip712Signature []byte `protobuf:"bytes,5,opt,name=eip712Signature,proto3" json:"eip712Signature,omitempty"` + AclAddress string `protobuf:"bytes,1,opt,name=aclAddress,proto3" json:"aclAddress,omitempty"` + HashOfCiphertext []byte `protobuf:"bytes,2,opt,name=hashOfCiphertext,proto3" json:"hashOfCiphertext,omitempty"` + InputHandles []*InputCiphertextResponseHandle `protobuf:"bytes,3,rep,name=input_handles,json=inputHandles,proto3" json:"input_handles,omitempty"` + ContractAddress string `protobuf:"bytes,5,opt,name=contractAddress,proto3" json:"contractAddress,omitempty"` + UserAddress string `protobuf:"bytes,6,opt,name=userAddress,proto3" json:"userAddress,omitempty"` + SignerAddress string `protobuf:"bytes,7,opt,name=signerAddress,proto3" json:"signerAddress,omitempty"` + Eip712Signature []byte `protobuf:"bytes,8,opt,name=eip712Signature,proto3" json:"eip712Signature,omitempty"` } func (x *InputCiphertextResponse) Reset() { @@ -656,6 +658,20 @@ func (*InputCiphertextResponse) Descriptor() ([]byte, []int) { return file_coprocessor_proto_rawDescGZIP(), []int{10} } +func (x *InputCiphertextResponse) GetAclAddress() string { + if x != nil { + return x.AclAddress + } + return "" +} + +func (x *InputCiphertextResponse) GetHashOfCiphertext() []byte { + if x != nil { + return x.HashOfCiphertext + } + return nil +} + func (x *InputCiphertextResponse) GetInputHandles() []*InputCiphertextResponseHandle { if x != nil { return x.InputHandles @@ -663,23 +679,23 @@ func (x *InputCiphertextResponse) GetInputHandles() []*InputCiphertextResponseHa return nil } -func (x *InputCiphertextResponse) GetEip712ContractAddress() string { +func (x *InputCiphertextResponse) GetContractAddress() string { if x != nil { - return x.Eip712ContractAddress + return x.ContractAddress } return "" } -func (x *InputCiphertextResponse) GetEip712CallerAddress() string { +func (x *InputCiphertextResponse) GetUserAddress() string { if x != nil { - return x.Eip712CallerAddress + return x.UserAddress } return "" } -func (x *InputCiphertextResponse) GetEip712SignerAddress() string { +func (x *InputCiphertextResponse) GetSignerAddress() string { if x != nil { - return x.Eip712SignerAddress + return x.SignerAddress } return "" } @@ -1039,41 +1055,43 @@ var file_coprocessor_proto_rawDesc = []byte{ 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x18, 0x0a, 0x06, 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x06, 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x42, 0x07, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, - 0xa4, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x55, 0x70, 0x6c, 0x6f, 0x61, + 0xa0, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x61, 0x6c, 0x6c, 0x65, - 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x60, 0x0a, 0x1d, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, - 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x61, 0x6e, 0x64, 0x6c, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x12, - 0x27, 0x0a, 0x0f, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, - 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0xb4, 0x02, 0x0a, 0x17, 0x49, 0x6e, 0x70, - 0x75, 0x74, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x0d, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, - 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x66, 0x68, - 0x65, 0x76, 0x6d, 0x2e, 0x63, 0x6f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x2e, - 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x0c, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x65, - 0x69, 0x70, 0x37, 0x31, 0x32, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x41, 0x64, 0x64, - 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x65, 0x69, 0x70, 0x37, - 0x31, 0x32, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x12, 0x30, 0x0a, 0x13, 0x65, 0x69, 0x70, 0x37, 0x31, 0x32, 0x43, 0x61, 0x6c, 0x6c, 0x65, - 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, - 0x65, 0x69, 0x70, 0x37, 0x31, 0x32, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x65, 0x69, 0x70, 0x37, 0x31, 0x32, 0x53, 0x69, 0x67, - 0x6e, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x13, 0x65, 0x69, 0x70, 0x37, 0x31, 0x32, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x41, 0x64, + 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x22, 0x60, 0x0a, 0x1d, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x69, 0x70, 0x68, 0x65, + 0x72, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x61, 0x6e, + 0x64, 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x06, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x63, + 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x22, 0xd8, 0x02, 0x0a, 0x17, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x69, + 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x63, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x63, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x12, 0x2a, 0x0a, 0x10, 0x68, 0x61, 0x73, 0x68, 0x4f, 0x66, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, + 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x10, 0x68, 0x61, 0x73, 0x68, + 0x4f, 0x66, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, 0x12, 0x55, 0x0a, 0x0d, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x66, 0x68, 0x65, 0x76, 0x6d, 0x2e, 0x63, 0x6f, 0x70, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x69, 0x70, + 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, + 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x0c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x48, 0x61, 0x6e, 0x64, + 0x6c, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x20, 0x0a, + 0x0b, 0x75, 0x73, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, + 0x24, 0x0a, 0x0d, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x65, 0x69, 0x70, 0x37, 0x31, 0x32, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x65, 0x69, 0x70, 0x37, 0x31, 0x32, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x6c, 0x0a, 0x13, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x10, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64,