From b8801919d4747f9dd103dbb20f74767619e5f7c7 Mon Sep 17 00:00:00 2001 From: Caleb Jaeger Date: Mon, 27 Mar 2023 17:06:49 -0700 Subject: [PATCH] Move PCF2-Lift Options to a shared location (#2198) Summary: Pull Request resolved: https://github.com/facebookresearch/fbpcs/pull/2198 This is moving the Lift options out of main so that they can be used in other parts of the code. This is especially important for feature flagging within the cpp code. Reviewed By: ajinkya-ghonge, haochenuw Differential Revision: D44062784 fbshipit-source-id: 46b4dc7fc95f14d24eeed32462569f4c3ffad9fe --- .../calculator/test/CalculatorAppTest.cpp | 2 - .../lift/common/CommonLiftOptions.cpp | 75 +++++++++++++ .../emp_games/lift/common/CommonLiftOptions.h | 31 +++++ .../lift/metadata_compaction/MainUtil.h | 2 +- .../MetadataCompactionOptions.cpp | 106 +++++++++--------- .../MetadataCompactionOptions.h | 31 +---- .../lift/metadata_compaction/main.cpp | 2 + .../lift/pcf2_calculator/LiftOptions.cpp | 46 ++++++++ .../lift/pcf2_calculator/LiftOptions.h | 21 ++++ fbpcs/emp_games/lift/pcf2_calculator/main.cpp | 106 +----------------- 10 files changed, 230 insertions(+), 192 deletions(-) create mode 100644 fbpcs/emp_games/lift/common/CommonLiftOptions.cpp create mode 100644 fbpcs/emp_games/lift/common/CommonLiftOptions.h create mode 100644 fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp create mode 100644 fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h diff --git a/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp b/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp index 7b25b7133..3cbd89c78 100644 --- a/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp +++ b/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp @@ -24,8 +24,6 @@ constexpr int32_t tsOffset = 10; DEFINE_bool(is_conversion_lift, true, "is conversion lift"); -DEFINE_int32(num_conversions_per_user, 4, "num of conversions per user"); -DEFINE_int64(epoch, 1546300800, "epoch"); namespace private_lift { class CalculatorAppTest : public ::testing::Test { diff --git a/fbpcs/emp_games/lift/common/CommonLiftOptions.cpp b/fbpcs/emp_games/lift/common/CommonLiftOptions.cpp new file mode 100644 index 000000000..d83913173 --- /dev/null +++ b/fbpcs/emp_games/lift/common/CommonLiftOptions.cpp @@ -0,0 +1,75 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" +#include + +DEFINE_bool( + compute_publisher_breakdowns, + true, + "To enable or disable computing publisher breakdown for result validation"); +DEFINE_bool( + log_cost, + false, + "Log cost info into cloud which will be used for dashboard"); +DEFINE_bool( + use_tls, + false, + "Whether to use TLS when communicating with other parties."); +DEFINE_bool( + use_xor_encryption, + true, + "Reveal output with XOR secret shares instead of in the clear to both parties"); +DEFINE_int32(concurrency, 1, "max number of games that will run concurrently"); +DEFINE_int32( + file_start_index, + 0, + "First file that will be read with base path"); +DEFINE_int32( + num_conversions_per_user, + 4, + "Cap and pad to this many conversions per user"); +DEFINE_int32(num_files, 0, "Number of files that should be read"); +DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); +DEFINE_int32( + port, + 10000, + "Network port for establishing connection to other player"); +DEFINE_int64( + epoch, + 1546300800, + "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps"); +DEFINE_string( + ca_cert_path, + "", + "Relative file path where root CA cert is stored. It will be prefixed with $HOME."); +DEFINE_string( + input_base_path, + "", + "Local or s3 base path for the sharded input files"); +DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); +DEFINE_string( + log_cost_s3_region, + ".s3.us-west-2.amazonaws.com/", + "s3 region name"); +DEFINE_string( + pc_feature_flags, + "", + "A String of PC Feature Flags passing from PCS, separated by comma"); +DEFINE_string( + private_key_path, + "", + "Relative file path where private key is stored. It will be prefixed with $HOME."); +DEFINE_string( + run_name, + "", + "A user given run name that will be used in s3 filename"); +DEFINE_string( + server_cert_path, + "", + "Relative file path where server cert is stored. It will be prefixed with $HOME."); +DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); diff --git a/fbpcs/emp_games/lift/common/CommonLiftOptions.h b/fbpcs/emp_games/lift/common/CommonLiftOptions.h new file mode 100644 index 000000000..bf947427c --- /dev/null +++ b/fbpcs/emp_games/lift/common/CommonLiftOptions.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +DECLARE_bool(compute_publisher_breakdowns); +DECLARE_bool(log_cost); +DECLARE_bool(use_tls); +DECLARE_bool(use_xor_encryption); +DECLARE_int32(concurrency); +DECLARE_int32(file_start_index); +DECLARE_int32(num_conversions_per_user); +DECLARE_int32(num_files); +DECLARE_int32(party); +DECLARE_int32(port); +DECLARE_int64(epoch); +DECLARE_string(ca_cert_path); +DECLARE_string(input_base_path); +DECLARE_string(log_cost_s3_bucket); +DECLARE_string(log_cost_s3_region); +DECLARE_string(pc_feature_flags); +DECLARE_string(private_key_path); +DECLARE_string(run_name); +DECLARE_string(server_cert_path); +DECLARE_string(server_ip); diff --git a/fbpcs/emp_games/lift/metadata_compaction/MainUtil.h b/fbpcs/emp_games/lift/metadata_compaction/MainUtil.h index 52b96632f..d94f30771 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/MainUtil.h +++ b/fbpcs/emp_games/lift/metadata_compaction/MainUtil.h @@ -8,13 +8,13 @@ #pragma once #include +#include #include "folly/Format.h" #include "fbpcf/engine/communication/SocketPartyCommunicationAgentFactory.h" #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorApp.h" #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorGame.h" #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorGameFactory.h" - namespace private_lift { struct LiftMetadataCompactionFilePaths { diff --git a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp index 51b25f9c9..f037377cb 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp +++ b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp @@ -6,93 +6,89 @@ */ #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h" +#include +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" -DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); -DEFINE_bool( - use_xor_encryption, - true, - "Reveal output with XOR secret shares instead of in the clear to both parties"); -DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); -DEFINE_int32( - port, - 10000, - "Network port for establishing connection to other player"); - -// Lift settings DEFINE_string(input_path, "", "Input file to run lift metadata compaction"); +DEFINE_string( + output_global_params_base_path, + "", + "Local or s3 base path where output global param files are written to"); DEFINE_string( output_global_params_path, "", "Output file to write global params from input data."); +DEFINE_string( + output_secret_shares_base_path, + "", + "Local or s3 base path where output secret share files are written to"); DEFINE_string( output_secret_shares_path, "", "Output file to write compacted metadata secret share results."); +// Common lift options +DEFINE_bool( + compute_publisher_breakdowns, + true, + "To enable or disable computing publisher breakdown for result validation"); +DEFINE_bool( + log_cost, + false, + "Log cost info into cloud which will be used for dashboard"); +DEFINE_bool( + use_tls, + false, + "Whether to use TLS when communicating with other parties."); +DEFINE_bool( + use_xor_encryption, + true, + "Reveal output with XOR secret shares instead of in the clear to both parties"); +DEFINE_int32(concurrency, 1, "max number of games that will run concurrently"); DEFINE_int32( file_start_index, 0, "First file that will be read with base path"); +DEFINE_int32( + num_conversions_per_user, + 4, + "Cap and pad to this many conversions per user"); DEFINE_int32(num_files, 0, "Number of files that should be read"); -DEFINE_string( - input_base_path, - "", - "Local or s3 base path for the sharded input files"); -DEFINE_string( - output_global_params_base_path, - "", - "Local or s3 base path where output global param files are written to"); -DEFINE_string( - output_secret_shares_base_path, - "", - "Local or s3 base path where output secret share files are written to"); -DEFINE_int32(concurrency, 1, "max number of games that will run concurrently"); +DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); DEFINE_int32( + port, + 10000, + "Network port for establishing connection to other player"); +DEFINE_int64( epoch, 1546300800, "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps"); -DEFINE_int32( - num_conversions_per_user, - 4, - "Cap and pad to this many conversions per user"); -DEFINE_bool( - compute_publisher_breakdowns, - true, - "To enable or disable computing publisher breakdown for result validation"); - -// TLS Settings -DEFINE_bool( - use_tls, - false, - "Whether to use TLS when communicating with other parties."); DEFINE_string( ca_cert_path, "", "Relative file path where root CA cert is stored. It will be prefixed with $HOME."); DEFINE_string( - server_cert_path, + input_base_path, "", - "Relative file path where server cert is stored. It will be prefixed with $HOME."); + "Local or s3 base path for the sharded input files"); +DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); +DEFINE_string( + log_cost_s3_region, + ".s3.us-west-2.amazonaws.com/", + "s3 region name"); +DEFINE_string( + pc_feature_flags, + "", + "A String of PC Feature Flags passing from PCS, separated by comma"); DEFINE_string( private_key_path, "", "Relative file path where private key is stored. It will be prefixed with $HOME."); - -// Logging flags DEFINE_string( run_name, "", "A user given run name that will be used in s3 filename"); -DEFINE_bool( - log_cost, - false, - "Log cost info into cloud which will be used for dashboard"); -DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); -DEFINE_string( - log_cost_s3_region, - ".s3.us-west-2.amazonaws.com/", - "s3 region name"); - DEFINE_string( - pc_feature_flags, + server_cert_path, "", - "A String of PC Feature Flags passing from PCS, separated by comma"); + "Relative file path where server cert is stored. It will be prefixed with $HOME."); +DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); diff --git a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h index c24c4684d..15a54cba5 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h +++ b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h @@ -9,36 +9,9 @@ #include -// MPC settings -DECLARE_int32(party); -DECLARE_bool(use_xor_encryption); -DECLARE_string(server_ip); -DECLARE_int32(port); - // Lift settings DECLARE_string(input_path); -DECLARE_string(output_global_params_path); -DECLARE_string(output_secret_shares_path); -DECLARE_int32(file_start_index); -DECLARE_int32(num_files); -DECLARE_string(input_base_path); DECLARE_string(output_global_params_base_path); +DECLARE_string(output_global_params_path); DECLARE_string(output_secret_shares_base_path); -DECLARE_int32(concurrency); -DECLARE_int32(epoch); -DECLARE_int32(num_conversions_per_user); -DECLARE_bool(compute_publisher_breakdowns); - -// TLS Settings -DECLARE_bool(use_tls); -DECLARE_string(ca_cert_path); -DECLARE_string(server_cert_path); -DECLARE_string(private_key_path); - -// Logging flags -DECLARE_string(run_name); -DECLARE_bool(log_cost); -DECLARE_string(log_cost_s3_bucket); -DECLARE_string(log_cost_s3_region); - -DECLARE_string(pc_feature_flags); +DECLARE_string(output_secret_shares_path); diff --git a/fbpcs/emp_games/lift/metadata_compaction/main.cpp b/fbpcs/emp_games/lift/metadata_compaction/main.cpp index 3a41c2774..b9119c84c 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/main.cpp +++ b/fbpcs/emp_games/lift/metadata_compaction/main.cpp @@ -13,8 +13,10 @@ #include "fbpcf/aws/AwsSdk.h" #include "fbpcs/performance_tools/CostEstimation.h" +#include #include "fbpcs/emp_games/common/Constants.h" #include "fbpcs/emp_games/common/Util.h" +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" #include "fbpcs/emp_games/lift/metadata_compaction/MainUtil.h" #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h" diff --git a/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp new file mode 100644 index 000000000..63cd9e8b7 --- /dev/null +++ b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h" +#include + +DEFINE_bool( + is_conversion_lift, + true, + "Use conversion_lift logic (as opposed to converter_lift logic)"); +DEFINE_string( + input_directory, + "", + "Data directory where input files are located"); +DEFINE_string( + input_expanded_key_path, + "out.csv_expanded_key_0", + "Input file name of the expanded key for UDP decryption. Used when decoupled UDP is enabled."); +DEFINE_string( + input_filenames, + "in.csv_0[,in.csv_1,in.csv_2,...]", + "List of input file names that should be parsed (should have a header)"); +DEFINE_string( + input_global_params_path, + "out.csv_global_params_0", + "Input file name of global parameter setup. Used when reading inputs in secret share format rather than plaintext."); +DEFINE_string( + output_base_path, + "", + "Local or s3 base path where output files are written to"); +DEFINE_string( + output_directory, + "", + "Local or s3 path where output files are written to"); +DEFINE_string( + output_filenames, + "out.csv_0[,out.csv_1,out.csv_2,...]", + "List of output file names that correspond to input filenames (positionally)"); +DEFINE_string( + run_id, + "", + "A run_id used to identify all the logs in a PL run."); diff --git a/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h new file mode 100644 index 000000000..c961dcc3d --- /dev/null +++ b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" + +DECLARE_bool(is_conversion_lift); +DECLARE_string(input_directory); +DECLARE_string(input_expanded_key_path); +DECLARE_string(input_filenames); +DECLARE_string(input_global_params_path); +DECLARE_string(output_base_path); +DECLARE_string(output_directory); +DECLARE_string(output_filenames); +DECLARE_string(run_id); diff --git a/fbpcs/emp_games/lift/pcf2_calculator/main.cpp b/fbpcs/emp_games/lift/pcf2_calculator/main.cpp index 0caba8637..5ec07e07e 100644 --- a/fbpcs/emp_games/lift/pcf2_calculator/main.cpp +++ b/fbpcs/emp_games/lift/pcf2_calculator/main.cpp @@ -19,114 +19,10 @@ #include "fbpcf/aws/AwsSdk.h" #include "fbpcs/emp_games/common/FeatureFlagUtil.h" +#include "fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h" #include "fbpcs/emp_games/lift/pcf2_calculator/MainUtil.h" #include "fbpcs/performance_tools/CostEstimation.h" -DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); -DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); -DEFINE_int32( - port, - 10000, - "Network port for establishing connection to other player"); -DEFINE_string( - input_directory, - "", - "Data directory where input files are located"); -DEFINE_string( - input_filenames, - "in.csv_0[,in.csv_1,in.csv_2,...]", - "List of input file names that should be parsed (should have a header)"); -DEFINE_string( - input_global_params_path, - "out.csv_global_params_0", - "Input file name of global parameter setup. Used when reading inputs in secret share format rather than plaintext."); -DEFINE_string( - input_expanded_key_path, - "out.csv_expanded_key_0", // TODO - "Input file name of the expanded key for UDP decryption. Used when decoupled UDP is enabled."); -DEFINE_string( - output_directory, - "", - "Local or s3 path where output files are written to"); -DEFINE_string( - output_filenames, - "out.csv_0[,out.csv_1,out.csv_2,...]", - "List of output file names that correspond to input filenames (positionally)"); -DEFINE_string( - input_base_path, - "", - "Local or s3 base path for the sharded input files"); -DEFINE_string( - output_base_path, - "", - "Local or s3 base path where output files are written to"); -DEFINE_int32( - file_start_index, - 0, - "First file that will be read with base path"); -DEFINE_int32(num_files, 0, "Number of files that should be read"); -DEFINE_int64( - epoch, - 1546300800, - "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps"); -DEFINE_bool( - is_conversion_lift, - true, - "Use conversion_lift logic (as opposed to converter_lift logic)"); -DEFINE_bool( - use_xor_encryption, - true, - "Reveal output with XOR secret shares instead of in the clear to both parties"); -DEFINE_int32( - num_conversions_per_user, - 4, - "Cap and pad to this many conversions per user"); -DEFINE_int32( - concurrency, - 1, - "max number of game(s) that will run concurrently?"); -DEFINE_string( - run_name, - "", - "A user given run name that will be used in s3 filename"); -DEFINE_string( - run_id, - "", - "A run_id used to identify all the logs in a PL run."); -DEFINE_bool( - log_cost, - false, - "Log cost info into cloud which will be used for dashboard"); -DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); -DEFINE_string( - log_cost_s3_region, - ".s3.us-west-2.amazonaws.com/", - "s3 regioni name"); -DEFINE_bool( - compute_publisher_breakdowns, - true, - "To enable or disable computing publisher breakdown for result validation"); -DEFINE_string( - pc_feature_flags, - "", - "A String of PC Feature Flags passing from PCS, separated by comma"); -DEFINE_bool( - use_tls, - false, - "Whether to use TLS when communicating with other parties."); -DEFINE_string( - ca_cert_path, - "", - "Relative file path where root CA cert is stored. It will be prefixed with $HOME."); -DEFINE_string( - server_cert_path, - "", - "Relative file path where server cert is stored. It will be prefixed with $HOME."); -DEFINE_string( - private_key_path, - "", - "Relative file path where private key is stored. It will be prefixed with $HOME."); - int main(int argc, char** argv) { folly::init(&argc, &argv); gflags::ParseCommandLineFlags(&argc, &argv, true);