From 284286948bfd562f63d988500a21f206693cce4c Mon Sep 17 00:00:00 2001 From: Caleb Jaeger Date: Thu, 16 Mar 2023 20:09:11 -0700 Subject: [PATCH] Move PCF2-Lift Options to a shared location (#2198) Summary: Pull Request resolved: https://github.com/facebookresearch/fbpcs/pull/2198 This is moving the Lift options out of main so that they can be used in other parts of the code. This is especially important for feature flagging within the cpp code. Differential Revision: D44062784 fbshipit-source-id: 98950f8f15f994a53b6a20e2ed66daecc4b7c2bc --- .../calculator/test/CalculatorAppTest.cpp | 2 - .../lift/common/CommonLiftOptions.cpp | 74 +++++++++++++ .../emp_games/lift/common/CommonLiftOptions.h | 31 ++++++ .../MetadataCompactionOptions.cpp | 84 ++------------- .../MetadataCompactionOptions.h | 32 +----- .../lift/metadata_compaction/main.cpp | 2 + .../lift/pcf2_calculator/LiftOptions.cpp | 42 ++++++++ .../lift/pcf2_calculator/LiftOptions.h | 20 ++++ fbpcs/emp_games/lift/pcf2_calculator/main.cpp | 102 +----------------- 9 files changed, 179 insertions(+), 210 deletions(-) create mode 100644 fbpcs/emp_games/lift/common/CommonLiftOptions.cpp create mode 100644 fbpcs/emp_games/lift/common/CommonLiftOptions.h create mode 100644 fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp create mode 100644 fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h diff --git a/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp b/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp index 7b25b7133..3cbd89c78 100644 --- a/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp +++ b/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp @@ -24,8 +24,6 @@ constexpr int32_t tsOffset = 10; DEFINE_bool(is_conversion_lift, true, "is conversion lift"); -DEFINE_int32(num_conversions_per_user, 4, "num of conversions per user"); -DEFINE_int64(epoch, 1546300800, "epoch"); namespace private_lift { class CalculatorAppTest : public ::testing::Test { diff --git a/fbpcs/emp_games/lift/common/CommonLiftOptions.cpp b/fbpcs/emp_games/lift/common/CommonLiftOptions.cpp new file mode 100644 index 000000000..4699acd30 --- /dev/null +++ b/fbpcs/emp_games/lift/common/CommonLiftOptions.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +DEFINE_bool( + compute_publisher_breakdowns, + true, + "To enable or disable computing publisher breakdown for result validation"); +DEFINE_bool( + log_cost, + false, + "Log cost info into cloud which will be used for dashboard"); +DEFINE_bool( + use_tls, + false, + "Whether to use TLS when communicating with other parties."); +DEFINE_bool( + use_xor_encryption, + true, + "Reveal output with XOR secret shares instead of in the clear to both parties"); +DEFINE_int32(concurrency, 1, "max number of games that will run concurrently"); +DEFINE_int32( + file_start_index, + 0, + "First file that will be read with base path"); +DEFINE_int32( + num_conversions_per_user, + 4, + "Cap and pad to this many conversions per user"); +DEFINE_int32(num_files, 0, "Number of files that should be read"); +DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); +DEFINE_int32( + port, + 10000, + "Network port for establishing connection to other player"); +DEFINE_int64( + epoch, + 1546300800, + "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps"); +DEFINE_string( + ca_cert_path, + "", + "Relative file path where root CA cert is stored. It will be prefixed with $HOME."); +DEFINE_string( + input_base_path, + "", + "Local or s3 base path for the sharded input files"); +DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); +DEFINE_string( + log_cost_s3_region, + ".s3.us-west-2.amazonaws.com/", + "s3 region name"); +DEFINE_string( + pc_feature_flags, + "", + "A String of PC Feature Flags passing from PCS, separated by comma"); +DEFINE_string( + private_key_path, + "", + "Relative file path where private key is stored. It will be prefixed with $HOME."); +DEFINE_string( + run_name, + "", + "A user given run name that will be used in s3 filename"); +DEFINE_string( + server_cert_path, + "", + "Relative file path where server cert is stored. It will be prefixed with $HOME."); +DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); diff --git a/fbpcs/emp_games/lift/common/CommonLiftOptions.h b/fbpcs/emp_games/lift/common/CommonLiftOptions.h new file mode 100644 index 000000000..bf947427c --- /dev/null +++ b/fbpcs/emp_games/lift/common/CommonLiftOptions.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +DECLARE_bool(compute_publisher_breakdowns); +DECLARE_bool(log_cost); +DECLARE_bool(use_tls); +DECLARE_bool(use_xor_encryption); +DECLARE_int32(concurrency); +DECLARE_int32(file_start_index); +DECLARE_int32(num_conversions_per_user); +DECLARE_int32(num_files); +DECLARE_int32(party); +DECLARE_int32(port); +DECLARE_int64(epoch); +DECLARE_string(ca_cert_path); +DECLARE_string(input_base_path); +DECLARE_string(log_cost_s3_bucket); +DECLARE_string(log_cost_s3_region); +DECLARE_string(pc_feature_flags); +DECLARE_string(private_key_path); +DECLARE_string(run_name); +DECLARE_string(server_cert_path); +DECLARE_string(server_ip); diff --git a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp index 51b25f9c9..c0d029b69 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp +++ b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp @@ -7,92 +7,20 @@ #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h" -DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); -DEFINE_bool( - use_xor_encryption, - true, - "Reveal output with XOR secret shares instead of in the clear to both parties"); -DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); -DEFINE_int32( - port, - 10000, - "Network port for establishing connection to other player"); - -// Lift settings DEFINE_string(input_path, "", "Input file to run lift metadata compaction"); -DEFINE_string( - output_global_params_path, - "", - "Output file to write global params from input data."); -DEFINE_string( - output_secret_shares_path, - "", - "Output file to write compacted metadata secret share results."); -DEFINE_int32( - file_start_index, - 0, - "First file that will be read with base path"); -DEFINE_int32(num_files, 0, "Number of files that should be read"); -DEFINE_string( - input_base_path, - "", - "Local or s3 base path for the sharded input files"); DEFINE_string( output_global_params_base_path, "", "Local or s3 base path where output global param files are written to"); DEFINE_string( - output_secret_shares_base_path, - "", - "Local or s3 base path where output secret share files are written to"); -DEFINE_int32(concurrency, 1, "max number of games that will run concurrently"); -DEFINE_int32( - epoch, - 1546300800, - "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps"); -DEFINE_int32( - num_conversions_per_user, - 4, - "Cap and pad to this many conversions per user"); -DEFINE_bool( - compute_publisher_breakdowns, - true, - "To enable or disable computing publisher breakdown for result validation"); - -// TLS Settings -DEFINE_bool( - use_tls, - false, - "Whether to use TLS when communicating with other parties."); -DEFINE_string( - ca_cert_path, - "", - "Relative file path where root CA cert is stored. It will be prefixed with $HOME."); -DEFINE_string( - server_cert_path, - "", - "Relative file path where server cert is stored. It will be prefixed with $HOME."); -DEFINE_string( - private_key_path, + output_global_params_path, "", - "Relative file path where private key is stored. It will be prefixed with $HOME."); - -// Logging flags + "Output file to write global params from input data."); DEFINE_string( - run_name, + output_secret_shares_base_path, "", - "A user given run name that will be used in s3 filename"); -DEFINE_bool( - log_cost, - false, - "Log cost info into cloud which will be used for dashboard"); -DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); -DEFINE_string( - log_cost_s3_region, - ".s3.us-west-2.amazonaws.com/", - "s3 region name"); - + "Local or s3 base path where output secret share files are written to"); DEFINE_string( - pc_feature_flags, + output_secret_shares_path, "", - "A String of PC Feature Flags passing from PCS, separated by comma"); + "Output file to write compacted metadata secret share results."); diff --git a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h index c24c4684d..3bc0d723f 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h +++ b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h @@ -8,37 +8,11 @@ #pragma once #include - -// MPC settings -DECLARE_int32(party); -DECLARE_bool(use_xor_encryption); -DECLARE_string(server_ip); -DECLARE_int32(port); +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" // Lift settings DECLARE_string(input_path); -DECLARE_string(output_global_params_path); -DECLARE_string(output_secret_shares_path); -DECLARE_int32(file_start_index); -DECLARE_int32(num_files); -DECLARE_string(input_base_path); DECLARE_string(output_global_params_base_path); +DECLARE_string(output_global_params_path); DECLARE_string(output_secret_shares_base_path); -DECLARE_int32(concurrency); -DECLARE_int32(epoch); -DECLARE_int32(num_conversions_per_user); -DECLARE_bool(compute_publisher_breakdowns); - -// TLS Settings -DECLARE_bool(use_tls); -DECLARE_string(ca_cert_path); -DECLARE_string(server_cert_path); -DECLARE_string(private_key_path); - -// Logging flags -DECLARE_string(run_name); -DECLARE_bool(log_cost); -DECLARE_string(log_cost_s3_bucket); -DECLARE_string(log_cost_s3_region); - -DECLARE_string(pc_feature_flags); +DECLARE_string(output_secret_shares_path); diff --git a/fbpcs/emp_games/lift/metadata_compaction/main.cpp b/fbpcs/emp_games/lift/metadata_compaction/main.cpp index 3a41c2774..5f4679086 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/main.cpp +++ b/fbpcs/emp_games/lift/metadata_compaction/main.cpp @@ -5,6 +5,7 @@ * LICENSE file in the root directory of this source tree. */ +#include #include #include #include "folly/init/Init.h" @@ -13,6 +14,7 @@ #include "fbpcf/aws/AwsSdk.h" #include "fbpcs/performance_tools/CostEstimation.h" +#include "fbpcs/emp_games/common/CommonLiftOptions.h" #include "fbpcs/emp_games/common/Constants.h" #include "fbpcs/emp_games/common/Util.h" #include "fbpcs/emp_games/lift/metadata_compaction/MainUtil.h" diff --git a/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp new file mode 100644 index 000000000..c6233bedf --- /dev/null +++ b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h" +#include + +DEFINE_bool( + is_conversion_lift, + true, + "Use conversion_lift logic (as opposed to converter_lift logic)"); +DEFINE_string( + input_directory, + "", + "Data directory where input files are located"); +DEFINE_string( + input_filenames, + "in.csv_0[,in.csv_1,in.csv_2,...]", + "List of input file names that should be parsed (should have a header)"); +DEFINE_string( + input_global_params_path, + "out.csv_global_params_0", + "Input file name of global parameter setup. Used when reading inputs in secret share format rather than plaintext."); +DEFINE_string( + output_base_path, + "", + "Local or s3 base path where output files are written to"); +DEFINE_string( + output_directory, + "", + "Local or s3 path where output files are written to"); +DEFINE_string( + output_filenames, + "out.csv_0[,out.csv_1,out.csv_2,...]", + "List of output file names that correspond to input filenames (positionally)"); +DEFINE_string( + run_id, + "", + "A run_id used to identify all the logs in a PL run."); diff --git a/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h new file mode 100644 index 000000000..1b711cfad --- /dev/null +++ b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" + +DECLARE_bool(is_conversion_lift); +DECLARE_string(input_directory); +DECLARE_string(input_filenames); +DECLARE_string(input_global_params_path); +DECLARE_string(output_base_path); +DECLARE_string(output_directory); +DECLARE_string(output_filenames); +DECLARE_string(run_id); diff --git a/fbpcs/emp_games/lift/pcf2_calculator/main.cpp b/fbpcs/emp_games/lift/pcf2_calculator/main.cpp index a95347a3b..6064a4b4f 100644 --- a/fbpcs/emp_games/lift/pcf2_calculator/main.cpp +++ b/fbpcs/emp_games/lift/pcf2_calculator/main.cpp @@ -19,110 +19,10 @@ #include "fbpcf/aws/AwsSdk.h" #include "fbpcs/emp_games/common/FeatureFlagUtil.h" +#include "fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h" #include "fbpcs/emp_games/lift/pcf2_calculator/MainUtil.h" #include "fbpcs/performance_tools/CostEstimation.h" -DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); -DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); -DEFINE_int32( - port, - 10000, - "Network port for establishing connection to other player"); -DEFINE_string( - input_directory, - "", - "Data directory where input files are located"); -DEFINE_string( - input_filenames, - "in.csv_0[,in.csv_1,in.csv_2,...]", - "List of input file names that should be parsed (should have a header)"); -DEFINE_string( - input_global_params_path, - "out.csv_global_params_0", - "Input file name of global parameter setup. Used when reading inputs in secret share format rather than plaintext."); -DEFINE_string( - output_directory, - "", - "Local or s3 path where output files are written to"); -DEFINE_string( - output_filenames, - "out.csv_0[,out.csv_1,out.csv_2,...]", - "List of output file names that correspond to input filenames (positionally)"); -DEFINE_string( - input_base_path, - "", - "Local or s3 base path for the sharded input files"); -DEFINE_string( - output_base_path, - "", - "Local or s3 base path where output files are written to"); -DEFINE_int32( - file_start_index, - 0, - "First file that will be read with base path"); -DEFINE_int32(num_files, 0, "Number of files that should be read"); -DEFINE_int64( - epoch, - 1546300800, - "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps"); -DEFINE_bool( - is_conversion_lift, - true, - "Use conversion_lift logic (as opposed to converter_lift logic)"); -DEFINE_bool( - use_xor_encryption, - true, - "Reveal output with XOR secret shares instead of in the clear to both parties"); -DEFINE_int32( - num_conversions_per_user, - 4, - "Cap and pad to this many conversions per user"); -DEFINE_int32( - concurrency, - 1, - "max number of game(s) that will run concurrently?"); -DEFINE_string( - run_name, - "", - "A user given run name that will be used in s3 filename"); -DEFINE_string( - run_id, - "", - "A run_id used to identify all the logs in a PL run."); -DEFINE_bool( - log_cost, - false, - "Log cost info into cloud which will be used for dashboard"); -DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); -DEFINE_string( - log_cost_s3_region, - ".s3.us-west-2.amazonaws.com/", - "s3 regioni name"); -DEFINE_bool( - compute_publisher_breakdowns, - true, - "To enable or disable computing publisher breakdown for result validation"); -DEFINE_string( - pc_feature_flags, - "", - "A String of PC Feature Flags passing from PCS, separated by comma"); -DEFINE_bool( - use_tls, - false, - "Whether to use TLS when communicating with other parties."); -DEFINE_string( - ca_cert_path, - "", - "Relative file path where root CA cert is stored. It will be prefixed with $HOME."); -DEFINE_string( - server_cert_path, - "", - "Relative file path where server cert is stored. It will be prefixed with $HOME."); -DEFINE_string( - private_key_path, - "", - "Relative file path where private key is stored. It will be prefixed with $HOME."); - int main(int argc, char** argv) { folly::init(&argc, &argv); gflags::ParseCommandLineFlags(&argc, &argv, true);