Make some codeprobes rare (#11607)

* Make BlobGranule code probes rare

* Make encryption related code probes rare

* fixup! Fix formatting

fdbclient/BackupAgentBase.actor.cpp

@@ -402,7 +402,7 @@ ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
 					    .detail("TenantId", domainId);
 					if (e.code() == error_code_encrypt_keys_fetch_failed ||
 					    e.code() == error_code_encrypt_key_not_found) {
-						CODE_PROBE(true, "mutation log restore encrypt keys not found");
+						CODE_PROBE(true, "mutation log restore encrypt keys not found", probe::decoration::rare);
 						consumed += BackupAgentBase::logHeaderSize + len1 + len2;
 						continue;
 					} else {

fdbclient/BlobCipher.cpp

@@ -441,7 +441,7 @@ void BlobCipherKey::initKey(const EncryptCipherDomainId& domainId,
 		TraceEvent(SevWarnAlways, "MaxBaseCipherKeyLimit")
 		    .detail("MaxAllowed", MAX_BASE_CIPHER_LEN)
 		    .detail("BaseCipherLen", baseCiphLen);
-		CODE_PROBE(true, "Encryption max base cipher len violation");
+		CODE_PROBE(true, "Encryption max base cipher len violation", probe::decoration::rare);
 		throw encrypt_max_base_cipher_len();
 	}
 
@@ -1241,7 +1241,7 @@ Reference<EncryptBuf> EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plainte
 		*encryptTime = timer_monotonic() - startTime;
 	}
 
-	CODE_PROBE(true, "BlobCipher data encryption");
+	CODE_PROBE(true, "BlobCipher data encryption", probe::decoration::rare);
 	CODE_PROBE(header->flags.authTokenAlgo == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE,
 	           "Encryption authentication disabled");
 	CODE_PROBE(header->flags.authTokenAlgo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
@@ -1287,7 +1287,7 @@ void EncryptBlobCipherAes265Ctr::encryptInplace(uint8_t* plaintext,
 		*encryptTime = timer_monotonic() - startTime;
 	}
 
-	CODE_PROBE(true, "encryptInplace: BlobCipher data encryption");
+	CODE_PROBE(true, "encryptInplace: BlobCipher data encryption", probe::decoration::rare);
 	CODE_PROBE(header->flags.authTokenAlgo == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE,
 	           "encryptInplace: Encryption authentication disabled");
 	CODE_PROBE(header->flags.authTokenAlgo == EncryptAuthTokenAlgo::ENCRYPT_HEADER_AUTH_TOKEN_ALGO_HMAC_SHA,
@@ -1411,7 +1411,7 @@ void DecryptBlobCipherAes256Ctr::validateEncryptHeaderFlagsV1(const uint32_t hea
 		    .detail("ExpectedCipherMode", EncryptCipherMode::ENCRYPT_CIPHER_MODE_AES_256_CTR)
 		    .detail("EncryptHeaderAuthTokenMode", flags.authTokenMode);
 
-		CODE_PROBE(true, "ConfigurableEncryption: Encryption header metadata mismatch");
+		CODE_PROBE(true, "ConfigurableEncryption: Encryption header metadata mismatch", probe::decoration::rare);
 
 		throw encrypt_header_metadata_mismatch();
 	}
@@ -1559,7 +1559,7 @@ void DecryptBlobCipherAes256Ctr::verifyEncryptHeaderMetadata(const BlobCipherEnc
 		    .detail("ExpectedCipherMode", EncryptCipherMode::ENCRYPT_CIPHER_MODE_AES_256_CTR)
 		    .detail("EncryptHeaderAuthTokenMode", header.flags.authTokenMode);
 
-		CODE_PROBE(true, "Encryption header metadata mismatch");
+		CODE_PROBE(true, "Encryption header metadata mismatch", probe::decoration::rare);
 
 		throw encrypt_header_metadata_mismatch();
 	}

fdbclient/BlobGranuleFiles.cpp

@@ -372,7 +372,7 @@ struct IndexBlockRef {
 			// Compressing indexBlock will need offset recalculation (circular dependency). IndexBlock size is bounded
 			// by number of chunks and sizeof(KeyPrefix), 'not' compressing IndexBlock shouldn't cause significant file
 			// size bloat.
-			CODE_PROBE(true, "encrypting index block");
+			CODE_PROBE(true, "encrypting index block", probe::decoration::rare);
 			ASSERT(cipherKeysCtx.present());
 			encrypt(cipherKeysCtx.get(), arena);
 		} else {
@@ -513,7 +513,7 @@ struct IndexBlobGranuleFileChunkRef {
 		}
 
 		if (cipherKeysCtx.present()) {
-			CODE_PROBE(true, "encrypting granule chunk");
+			CODE_PROBE(true, "encrypting granule chunk", probe::decoration::rare);
 			IndexBlobGranuleFileChunkRef::encrypt(cipherKeysCtx.get(), chunkRef, arena);
 		}
 
@@ -743,8 +743,8 @@ Value serializeChunkedSnapshot(const Standalone<StringRef>& fileNameRef,
 		    .detail("Compressed", compressFilter.present());
 	}
 
-	CODE_PROBE(compressFilter.present(), "serializing compressed snapshot file");
-	CODE_PROBE(cipherKeysCtx.present(), "serializing encrypted snapshot file");
+	CODE_PROBE(compressFilter.present(), "serializing compressed snapshot file", probe::decoration::rare);
+	CODE_PROBE(cipherKeysCtx.present(), "serializing encrypted snapshot file", probe::decoration::rare);
 	Standalone<IndexedBlobGranuleFile> file;
 
 	file.init(SNAPSHOT_FILE_TYPE, cipherKeysCtx);
@@ -903,7 +903,7 @@ void updateMutationBoundary(Standalone<DeltaBoundaryRef>& boundary, const ValueA
 			// duplicate same set even if it's the same as the last one, so beginVersion reads still get updates
 			boundary.values.push_back(boundary.arena(), update);
 		} else {
-			CODE_PROBE(true, "multiple boundary updates at same version (set)");
+			CODE_PROBE(true, "multiple boundary updates at same version (set)", probe::decoration::rare);
 			// preserve inter-mutation order by replacing this one
 			boundary.values.back() = update;
 		}
@@ -914,12 +914,12 @@ void updateMutationBoundary(Standalone<DeltaBoundaryRef>& boundary, const ValueA
 			// with beginVersion
 			boundary.values.push_back(boundary.arena(), update);
 		} else if (!boundary.values.empty() && boundary.values.back().version == update.version) {
-			CODE_PROBE(true, "multiple boundary updates at same version (clear)");
+			CODE_PROBE(true, "multiple boundary updates at same version (clear)", probe::decoration::rare);
 			if (boundary.values.back().isSet()) {
 				// if the last 2 updates were clear @ v1 and set @ v2, and we now have a clear at v2, just pop off the
 				// set and leave the previous clear. Otherwise, just set the last set to a clear
 				if (boundary.values.size() >= 2 && boundary.values[boundary.values.size() - 2].isClear()) {
-					CODE_PROBE(true, "clear then set/clear at same version optimization");
+					CODE_PROBE(true, "clear then set/clear at same version optimization", probe::decoration::rare);
 					boundary.values.pop_back();
 				} else {
 					boundary.values.back() = update;
@@ -1008,8 +1008,8 @@ Value serializeChunkedDeltaFile(const Standalone<StringRef>& fileNameRef,
 		    .detail("Compressed", compressFilter.present());
 	}
 
-	CODE_PROBE(compressFilter.present(), "serializing compressed delta file");
-	CODE_PROBE(cipherKeysCtx.present(), "serializing encrypted delta file");
+	CODE_PROBE(compressFilter.present(), "serializing compressed delta file", probe::decoration::rare);
+	CODE_PROBE(cipherKeysCtx.present(), "serializing encrypted delta file", probe::decoration::rare);
 	Standalone<IndexedBlobGranuleFile> file;
 
 	file.init(DELTA_FILE_TYPE, cipherKeysCtx);

fdbclient/FileBackupAgent.actor.cpp

@@ -1206,7 +1206,7 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
 		if (file_version == BACKUP_AGENT_SNAPSHOT_FILE_VERSION) {
 			wait(decodeKVPairs(&reader, &results, false, encryptMode, Optional<int64_t>(), tenantCache));
 		} else if (file_version == BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION) {
-			CODE_PROBE(true, "decoding encrypted block");
+			CODE_PROBE(true, "decoding encrypted block", probe::decoration::rare);
 			// read header size
 			state uint32_t headerLen = reader.consume<uint32_t>();
 			// read the encryption header
@@ -1241,11 +1241,11 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
 		if (e.code() == error_code_encrypt_keys_fetch_failed || e.code() == error_code_encrypt_key_not_found) {
 			ASSERT(!isReservedEncryptDomain(blockDomainId));
 			TraceEvent(SevWarnAlways, "SnapshotRestoreEncryptKeyFetchFailed").detail("TenantId", blockDomainId);
-			CODE_PROBE(true, "Snapshot restore encrypt keys not found");
+			CODE_PROBE(true, "Snapshot restore encrypt keys not found", probe::decoration::rare);
 		} else if (e.code() == error_code_tenant_not_found) {
 			ASSERT(!isReservedEncryptDomain(blockDomainId));
 			TraceEvent(SevWarnAlways, "EncryptedSnapshotRestoreTenantNotFound").detail("TenantId", blockDomainId);
-			CODE_PROBE(true, "Encrypted Snapshot restore tenant not found");
+			CODE_PROBE(true, "Encrypted Snapshot restore tenant not found", probe::decoration::rare);
 		}
 		TraceEvent(SevWarn, "FileRestoreDecodeRangeFileBlockFailed")
 		    .error(e)
@@ -1928,7 +1928,7 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 				TraceEvent(SevDebug, "EncryptionMode").detail("EncryptMode", encryptMode.toString());
 				// Initialize range file writer and write begin key
 				if (encryptMode.mode != EncryptionAtRestMode::DISABLED) {
-					CODE_PROBE(true, "using encrypted snapshot file writer");
+					CODE_PROBE(true, "using encrypted snapshot file writer", probe::decoration::rare);
 					rangeFile = std::make_unique<EncryptedRangeFileWriter>(
 					    cx, &arena, encryptMode, tenantCache, outFile, blockSize);
 				} else {

fdbserver/ApplyMetadataMutation.cpp

@@ -205,8 +205,8 @@ class ApplyMetadataMutationsImpl {
 		} else {
 			ASSERT(cipherKeys != nullptr);
 			Arena arena;
-			CODE_PROBE(!forResolver, "encrypting metadata mutations");
-			CODE_PROBE(forResolver, "encrypting resolver mutations");
+			CODE_PROBE(!forResolver, "encrypting metadata mutations", probe::decoration::rare);
+			CODE_PROBE(forResolver, "encrypting resolver mutations", probe::decoration::rare);
 			toCommit->writeTypedMessage(m.encryptMetadata(*cipherKeys, arena, BlobCipherMetrics::TLOG));
 		}
 	}

fdbserver/BlobGranuleServerCommon.actor.cpp

@@ -516,7 +516,7 @@ ACTOR Future<Void> loadBlobMetadataForTenants(BGTenantMap* self, std::vector<Blo
 					TraceEvent(SevWarn, "BlobMetadataFetchMissingTenants")
 					    .suppressFor(30.0)
 					    .detail("Count", missingIds.size());
-					CODE_PROBE(true, "blob metadata fetch missing tenants");
+					CODE_PROBE(true, "blob metadata fetch missing tenants", probe::decoration::rare);
 
 					req.domainIds.clear();
 					for (auto& id : missingIds) {
@@ -537,7 +537,7 @@ ACTOR Future<Void> loadBlobMetadataForTenants(BGTenantMap* self, std::vector<Blo
 			if (e.code() == error_code_operation_cancelled) {
 				throw e;
 			}
-			CODE_PROBE(true, "blob metadata fetch error");
+			CODE_PROBE(true, "blob metadata fetch error", probe::decoration::rare);
 			TraceEvent(SevWarn, "BlobMetadataFetchError").errorUnsuppressed(e).suppressFor(30.0);
 			// need to reset request on error
 			prevEKPID = UID();
@@ -598,7 +598,7 @@ ACTOR Future<Reference<GranuleTenantData>> getDataForGranuleActor(BGTenantMap* s
 			return tenant;
 		} else if (!tenant->startedLoadingBStore || (tenant->bstore.isValid() && tenant->bstore->isExpired())) {
 			tenant->startedLoadingBStore = true;
-			CODE_PROBE(true, "re-fetching expired blob metadata");
+			CODE_PROBE(true, "re-fetching expired blob metadata", probe::decoration::rare);
 
 			// even if this actor gets cancelled, we marked it as startedLoading, so finish the load in the actor
 			// collection
@@ -659,7 +659,7 @@ ACTOR Future<Reference<BlobConnectionProvider>> loadBStoreForTenant(BGTenantMap*
 				wait(delay(0));
 				return data->bstore;
 			} else {
-				CODE_PROBE(true, "bstore for unknown tenant");
+				CODE_PROBE(true, "bstore for unknown tenant", probe::decoration::rare);
 				// Assume not loaded yet, just wait a bit. Could do sophisticated mechanism but will redo tenant
 				// loading to be versioned anyway. 10 retries means it's likely not a transient race with
 				// loading tenants, and instead a persistent issue.

fdbserver/BlobGranuleValidation.actor.cpp

@@ -200,7 +200,7 @@ ACTOR Future<Void> clearAndAwaitMerge(Database cx, KeyRange range) {
 			if (ranges.size() == 1) {
 				return Void();
 			}
-			CODE_PROBE(true, "ClearAndAwaitMerge doing clear");
+			CODE_PROBE(true, "ClearAndAwaitMerge doing clear", probe::decoration::rare);
 			reClearCount--;
 			if (reClearCount <= 0) {
 				tr.clear(range);
@@ -284,7 +284,7 @@ ACTOR Future<Void> validateGranuleSummaries(Database cx,
 			}
 
 			if (lastSummaryVersion != invalidVersion) {
-				CODE_PROBE(true, "comparing multiple summaries");
+				CODE_PROBE(true, "comparing multiple summaries", probe::decoration::rare);
 				// diff with last summary ranges to ensure versions never decreased for any range
 				for (auto& it : nextSummary) {
 					auto lastSummaries = lastSummary.intersectingRanges(it.keyRange);

fdbserver/BlobWorker.actor.cpp

@@ -5603,7 +5603,7 @@ ACTOR Future<Void> checkUpdateEncryptionAtRestMode(Reference<BlobWorkerData> sel
 	} else {
 		self->persistedEncryptMode = encryptionAtRestMode;
 		if (self->storage) {
-			CODE_PROBE(true, "BlobWorker: Persisting encryption at rest mode");
+			CODE_PROBE(true, "BlobWorker: Persisting encryption at rest mode", probe::decoration::rare);
 			self->storage->set(KeyValueRef(persistEncryptionAtRestModeKey, self->persistedEncryptMode.get().toValue()));
 			wait(self->storage->commit());
 			TraceEvent("BlobWorkerPersistEncryptionAtRestMode", self->id)
@@ -5702,7 +5702,7 @@ ACTOR Future<UID> restorePersistentState(Reference<BlobWorkerData> self) {
 	ASSERT(recoveredID != self->id);
 
 	if (fEncryptionAtRestMode.get().present()) {
-		CODE_PROBE(true, "BlobWorker: Retrieved persisted encryption at rest mode");
+		CODE_PROBE(true, "BlobWorker: Retrieved persisted encryption at rest mode", probe::decoration::rare);
 		self->persistedEncryptMode =
 		    Optional<EncryptionAtRestMode>(EncryptionAtRestMode::fromValue(fEncryptionAtRestMode.get()));
 		TraceEvent("BlobWorkerPersistEncryptionAtRestModeRead", self->id)

fdbserver/ClusterRecovery.actor.cpp

@@ -1099,7 +1099,7 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 		EncryptionAtRestMode encryptMode = wait(self->controllerData->encryptionAtRestMode.getFuture());
 		enableEncryptionForTxnStateStore = encryptMode.isEncryptionEnabled();
 	}
-	CODE_PROBE(enableEncryptionForTxnStateStore, "Enable encryption for txnStateStore");
+	CODE_PROBE(enableEncryptionForTxnStateStore, "Enable encryption for txnStateStore", probe::decoration::rare);
 	if (self->txnStateStore)
 		self->txnStateStore->close();
 	self->txnStateLogAdapter = openDiskQueueAdapter(oldLogSystem, myLocality, txsPoppedVersion);

fdbserver/CommitProxyServer.actor.cpp

@@ -1044,14 +1044,14 @@ EncryptCipherDomainId getEncryptDetailsFromMutationRef(ProxyCommitData* commitDa
 		// 2. Transaction can be a multi-key transaction spawning multiple tenants
 		// For now fallback to 'default encryption domain'
 
-		CODE_PROBE(true, "ClearRange mutation encryption");
+		CODE_PROBE(true, "ClearRange mutation encryption", probe::decoration::rare);
 	}
 
 	// Unknown tenant, fallback to fdb default encryption domain
 	if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
 		domainId = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
 
-		CODE_PROBE(true, "Default domain mutation encryption");
+		CODE_PROBE(true, "Default domain mutation encryption", probe::decoration::rare);
 	}
 
 	return domainId;
@@ -1817,7 +1817,7 @@ ACTOR Future<WriteMutationRefVar> writeMutationEncryptedMutation(CommitBatchCont
 	ASSERT(decryptedMutation.param1 == mutation->param1);
 	ASSERT(decryptedMutation.param2 == mutation->param2);
 
-	CODE_PROBE(true, "encrypting non-metadata mutations");
+	CODE_PROBE(true, "encrypting non-metadata mutations", probe::decoration::rare);
 	self->toCommit.writeTypedMessage(encryptedMutation);
 	return encryptedMutation;
 }
@@ -1852,7 +1852,7 @@ Future<WriteMutationRefVar> writeMutation(CommitBatchContext* self,
 		CODE_PROBE(self->pProxyCommitData->getTenantMode() == TenantMode::REQUIRED, "using required tenant mode");
 
 		if (encryptedMutationOpt && encryptedMutationOpt->present()) {
-			CODE_PROBE(true, "using already encrypted mutation");
+			CODE_PROBE(true, "using already encrypted mutation", probe::decoration::rare);
 			encryptedMutation = encryptedMutationOpt->get();
 			ASSERT(encryptedMutation.isEncrypted());
 			// During simulation check whether the encrypted mutation matches the decrpyted mutation
@@ -1862,15 +1862,15 @@ Future<WriteMutationRefVar> writeMutation(CommitBatchContext* self,
 		} else {
 			if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
 				domainId = getEncryptDetailsFromMutationRef(self->pProxyCommitData, *mutation);
-				CODE_PROBE(true, "Raw access mutation encryption");
+				CODE_PROBE(true, "Raw access mutation encryption", probe::decoration::rare);
 			}
 			ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID);
 			ASSERT(self->cipherKeys.count(domainId) > 0);
 			encryptedMutation =
 			    mutation->encrypt(self->cipherKeys, domainId, *arena, BlobCipherMetrics::TLOG, encryptTime);
 		}
 		ASSERT(encryptedMutation.isEncrypted());
-		CODE_PROBE(true, "encrypting non-metadata mutations");
+		CODE_PROBE(true, "encrypting non-metadata mutations", probe::decoration::rare);
 		self->toCommit.writeTypedMessage(encryptedMutation);
 		return std::variant<MutationRef, VectorRef<MutationRef>>{ encryptedMutation };
 	} else {
@@ -1918,7 +1918,7 @@ double pushToBackupMutations(CommitBatchContext* self,
 		// Add the mutation to the relevant backup tag
 		for (auto backupName : pProxyCommitData->vecBackupKeys[m.param1]) {
 			// If encryption is enabled make sure the mutation we are writing is also encrypted
-			CODE_PROBE(writtenMutation.isEncrypted(), "using encrypted backup mutation");
+			CODE_PROBE(writtenMutation.isEncrypted(), "using encrypted backup mutation", probe::decoration::rare);
 			self->logRangeMutations[backupName].push_back_deep(self->logRangeMutationsArena, writtenMutation);
 		}
 
@@ -1938,7 +1938,7 @@ double pushToBackupMutations(CommitBatchContext* self,
 			MutationRef backupMutation(MutationRef::Type::ClearRange, intersectionRange.begin, intersectionRange.end);
 
 			if (pProxyCommitData->encryptMode.isEncryptionEnabled()) {
-				CODE_PROBE(true, "encrypting clear range backup mutation");
+				CODE_PROBE(true, "encrypting clear range backup mutation", probe::decoration::rare);
 				if (backupMutation.param1 == m.param1 && backupMutation.param2 == m.param2 &&
 				    encryptedMutation.present()) {
 					backupMutation = encryptedMutation.get();
@@ -2318,7 +2318,7 @@ ACTOR Future<Void> postResolution(CommitBatchContext* self) {
 		    auto& tags = pProxyCommitData->tagsForKey(kv.key);
 		    self->toCommit.addTags(tags);
 		    if (self->pProxyCommitData->encryptMode.isEncryptionEnabled()) {
-			    CODE_PROBE(true, "encrypting idempotency mutation");
+			    CODE_PROBE(true, "encrypting idempotency mutation", probe::decoration::rare);
 			    EncryptCipherDomainId domainId =
 			        getEncryptDetailsFromMutationRef(self->pProxyCommitData, idempotencyIdSet);
 			    MutationRef encryptedMutation =

fdbserver/SimulatedCluster.actor.cpp

@@ -1703,7 +1703,7 @@ void SimulationConfig::setEncryptionAtRestMode(const TestConfig& testConfig) {
 	    deterministicRandom()->random01() < SERVER_KNOBS->DISABLED_ENCRYPTION_PROBABILITY_SIM) {
 		EncryptionAtRestMode encryptionMode = EncryptionAtRestMode::DISABLED;
 		TraceEvent("SimulatedClusterEncryptionMode").detail("Mode", encryptionMode.toString());
-		CODE_PROBE(true, "Enforce to disable encryption in simulation");
+		CODE_PROBE(true, "Enforce to disable encryption in simulation", probe::decoration::rare);
 		set_config("encryption_at_rest_mode=" + encryptionMode.toString());
 		return;
 	}
@@ -1757,9 +1757,14 @@ void SimulationConfig::setEncryptionAtRestMode(const TestConfig& testConfig) {
 		r -= probability[mode];
 	}
 	TraceEvent("SimulatedClusterEncryptionMode").detail("Mode", encryptionMode.toString());
-	CODE_PROBE(encryptionMode == EncryptionAtRestMode::DISABLED, "Disabled encryption in simulation");
-	CODE_PROBE(encryptionMode == EncryptionAtRestMode::CLUSTER_AWARE, "Enabled cluster-aware encryption in simulation");
-	CODE_PROBE(encryptionMode == EncryptionAtRestMode::DOMAIN_AWARE, "Enabled domain-aware encryption in simulation");
+	CODE_PROBE(
+	    encryptionMode == EncryptionAtRestMode::DISABLED, "Disabled encryption in simulation", probe::decoration::rare);
+	CODE_PROBE(encryptionMode == EncryptionAtRestMode::CLUSTER_AWARE,
+	           "Enabled cluster-aware encryption in simulation",
+	           probe::decoration::rare);
+	CODE_PROBE(encryptionMode == EncryptionAtRestMode::DOMAIN_AWARE,
+	           "Enabled domain-aware encryption in simulation",
+	           probe::decoration::rare);
 	set_config("encryption_at_rest_mode=" + encryptionMode.toString());
 }