diff --git a/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp b/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp index 7b25b7133..3cbd89c78 100644 --- a/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp +++ b/fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp @@ -24,8 +24,6 @@ constexpr int32_t tsOffset = 10; DEFINE_bool(is_conversion_lift, true, "is conversion lift"); -DEFINE_int32(num_conversions_per_user, 4, "num of conversions per user"); -DEFINE_int64(epoch, 1546300800, "epoch"); namespace private_lift { class CalculatorAppTest : public ::testing::Test { diff --git a/fbpcs/emp_games/lift/common/CommonLiftOptions.cpp b/fbpcs/emp_games/lift/common/CommonLiftOptions.cpp new file mode 100644 index 000000000..d83913173 --- /dev/null +++ b/fbpcs/emp_games/lift/common/CommonLiftOptions.cpp @@ -0,0 +1,75 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" +#include + +DEFINE_bool( + compute_publisher_breakdowns, + true, + "To enable or disable computing publisher breakdown for result validation"); +DEFINE_bool( + log_cost, + false, + "Log cost info into cloud which will be used for dashboard"); +DEFINE_bool( + use_tls, + false, + "Whether to use TLS when communicating with other parties."); +DEFINE_bool( + use_xor_encryption, + true, + "Reveal output with XOR secret shares instead of in the clear to both parties"); +DEFINE_int32(concurrency, 1, "max number of games that will run concurrently"); +DEFINE_int32( + file_start_index, + 0, + "First file that will be read with base path"); +DEFINE_int32( + num_conversions_per_user, + 4, + "Cap and pad to this many conversions per user"); +DEFINE_int32(num_files, 0, "Number of files that should be read"); +DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); +DEFINE_int32( + port, + 10000, + "Network port for establishing connection to other player"); +DEFINE_int64( + epoch, + 1546300800, + "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps"); +DEFINE_string( + ca_cert_path, + "", + "Relative file path where root CA cert is stored. It will be prefixed with $HOME."); +DEFINE_string( + input_base_path, + "", + "Local or s3 base path for the sharded input files"); +DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); +DEFINE_string( + log_cost_s3_region, + ".s3.us-west-2.amazonaws.com/", + "s3 region name"); +DEFINE_string( + pc_feature_flags, + "", + "A String of PC Feature Flags passing from PCS, separated by comma"); +DEFINE_string( + private_key_path, + "", + "Relative file path where private key is stored. It will be prefixed with $HOME."); +DEFINE_string( + run_name, + "", + "A user given run name that will be used in s3 filename"); +DEFINE_string( + server_cert_path, + "", + "Relative file path where server cert is stored. It will be prefixed with $HOME."); +DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); diff --git a/fbpcs/emp_games/lift/common/CommonLiftOptions.h b/fbpcs/emp_games/lift/common/CommonLiftOptions.h new file mode 100644 index 000000000..bf947427c --- /dev/null +++ b/fbpcs/emp_games/lift/common/CommonLiftOptions.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +DECLARE_bool(compute_publisher_breakdowns); +DECLARE_bool(log_cost); +DECLARE_bool(use_tls); +DECLARE_bool(use_xor_encryption); +DECLARE_int32(concurrency); +DECLARE_int32(file_start_index); +DECLARE_int32(num_conversions_per_user); +DECLARE_int32(num_files); +DECLARE_int32(party); +DECLARE_int32(port); +DECLARE_int64(epoch); +DECLARE_string(ca_cert_path); +DECLARE_string(input_base_path); +DECLARE_string(log_cost_s3_bucket); +DECLARE_string(log_cost_s3_region); +DECLARE_string(pc_feature_flags); +DECLARE_string(private_key_path); +DECLARE_string(run_name); +DECLARE_string(server_cert_path); +DECLARE_string(server_ip); diff --git a/fbpcs/emp_games/lift/metadata_compaction/MainUtil.h b/fbpcs/emp_games/lift/metadata_compaction/MainUtil.h index 52b96632f..d94f30771 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/MainUtil.h +++ b/fbpcs/emp_games/lift/metadata_compaction/MainUtil.h @@ -8,13 +8,13 @@ #pragma once #include +#include #include "folly/Format.h" #include "fbpcf/engine/communication/SocketPartyCommunicationAgentFactory.h" #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorApp.h" #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorGame.h" #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorGameFactory.h" - namespace private_lift { struct LiftMetadataCompactionFilePaths { diff --git a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp index 51b25f9c9..f037377cb 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp +++ b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp @@ -6,93 +6,89 @@ */ #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h" +#include +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" -DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); -DEFINE_bool( - use_xor_encryption, - true, - "Reveal output with XOR secret shares instead of in the clear to both parties"); -DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); -DEFINE_int32( - port, - 10000, - "Network port for establishing connection to other player"); - -// Lift settings DEFINE_string(input_path, "", "Input file to run lift metadata compaction"); +DEFINE_string( + output_global_params_base_path, + "", + "Local or s3 base path where output global param files are written to"); DEFINE_string( output_global_params_path, "", "Output file to write global params from input data."); +DEFINE_string( + output_secret_shares_base_path, + "", + "Local or s3 base path where output secret share files are written to"); DEFINE_string( output_secret_shares_path, "", "Output file to write compacted metadata secret share results."); +// Common lift options +DEFINE_bool( + compute_publisher_breakdowns, + true, + "To enable or disable computing publisher breakdown for result validation"); +DEFINE_bool( + log_cost, + false, + "Log cost info into cloud which will be used for dashboard"); +DEFINE_bool( + use_tls, + false, + "Whether to use TLS when communicating with other parties."); +DEFINE_bool( + use_xor_encryption, + true, + "Reveal output with XOR secret shares instead of in the clear to both parties"); +DEFINE_int32(concurrency, 1, "max number of games that will run concurrently"); DEFINE_int32( file_start_index, 0, "First file that will be read with base path"); +DEFINE_int32( + num_conversions_per_user, + 4, + "Cap and pad to this many conversions per user"); DEFINE_int32(num_files, 0, "Number of files that should be read"); -DEFINE_string( - input_base_path, - "", - "Local or s3 base path for the sharded input files"); -DEFINE_string( - output_global_params_base_path, - "", - "Local or s3 base path where output global param files are written to"); -DEFINE_string( - output_secret_shares_base_path, - "", - "Local or s3 base path where output secret share files are written to"); -DEFINE_int32(concurrency, 1, "max number of games that will run concurrently"); +DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); DEFINE_int32( + port, + 10000, + "Network port for establishing connection to other player"); +DEFINE_int64( epoch, 1546300800, "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps"); -DEFINE_int32( - num_conversions_per_user, - 4, - "Cap and pad to this many conversions per user"); -DEFINE_bool( - compute_publisher_breakdowns, - true, - "To enable or disable computing publisher breakdown for result validation"); - -// TLS Settings -DEFINE_bool( - use_tls, - false, - "Whether to use TLS when communicating with other parties."); DEFINE_string( ca_cert_path, "", "Relative file path where root CA cert is stored. It will be prefixed with $HOME."); DEFINE_string( - server_cert_path, + input_base_path, "", - "Relative file path where server cert is stored. It will be prefixed with $HOME."); + "Local or s3 base path for the sharded input files"); +DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); +DEFINE_string( + log_cost_s3_region, + ".s3.us-west-2.amazonaws.com/", + "s3 region name"); +DEFINE_string( + pc_feature_flags, + "", + "A String of PC Feature Flags passing from PCS, separated by comma"); DEFINE_string( private_key_path, "", "Relative file path where private key is stored. It will be prefixed with $HOME."); - -// Logging flags DEFINE_string( run_name, "", "A user given run name that will be used in s3 filename"); -DEFINE_bool( - log_cost, - false, - "Log cost info into cloud which will be used for dashboard"); -DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); -DEFINE_string( - log_cost_s3_region, - ".s3.us-west-2.amazonaws.com/", - "s3 region name"); - DEFINE_string( - pc_feature_flags, + server_cert_path, "", - "A String of PC Feature Flags passing from PCS, separated by comma"); + "Relative file path where server cert is stored. It will be prefixed with $HOME."); +DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); diff --git a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h index c24c4684d..15a54cba5 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h +++ b/fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h @@ -9,36 +9,9 @@ #include -// MPC settings -DECLARE_int32(party); -DECLARE_bool(use_xor_encryption); -DECLARE_string(server_ip); -DECLARE_int32(port); - // Lift settings DECLARE_string(input_path); -DECLARE_string(output_global_params_path); -DECLARE_string(output_secret_shares_path); -DECLARE_int32(file_start_index); -DECLARE_int32(num_files); -DECLARE_string(input_base_path); DECLARE_string(output_global_params_base_path); +DECLARE_string(output_global_params_path); DECLARE_string(output_secret_shares_base_path); -DECLARE_int32(concurrency); -DECLARE_int32(epoch); -DECLARE_int32(num_conversions_per_user); -DECLARE_bool(compute_publisher_breakdowns); - -// TLS Settings -DECLARE_bool(use_tls); -DECLARE_string(ca_cert_path); -DECLARE_string(server_cert_path); -DECLARE_string(private_key_path); - -// Logging flags -DECLARE_string(run_name); -DECLARE_bool(log_cost); -DECLARE_string(log_cost_s3_bucket); -DECLARE_string(log_cost_s3_region); - -DECLARE_string(pc_feature_flags); +DECLARE_string(output_secret_shares_path); diff --git a/fbpcs/emp_games/lift/metadata_compaction/main.cpp b/fbpcs/emp_games/lift/metadata_compaction/main.cpp index 3a41c2774..b9119c84c 100644 --- a/fbpcs/emp_games/lift/metadata_compaction/main.cpp +++ b/fbpcs/emp_games/lift/metadata_compaction/main.cpp @@ -13,8 +13,10 @@ #include "fbpcf/aws/AwsSdk.h" #include "fbpcs/performance_tools/CostEstimation.h" +#include #include "fbpcs/emp_games/common/Constants.h" #include "fbpcs/emp_games/common/Util.h" +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" #include "fbpcs/emp_games/lift/metadata_compaction/MainUtil.h" #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h" diff --git a/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp new file mode 100644 index 000000000..63cd9e8b7 --- /dev/null +++ b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h" +#include + +DEFINE_bool( + is_conversion_lift, + true, + "Use conversion_lift logic (as opposed to converter_lift logic)"); +DEFINE_string( + input_directory, + "", + "Data directory where input files are located"); +DEFINE_string( + input_expanded_key_path, + "out.csv_expanded_key_0", + "Input file name of the expanded key for UDP decryption. Used when decoupled UDP is enabled."); +DEFINE_string( + input_filenames, + "in.csv_0[,in.csv_1,in.csv_2,...]", + "List of input file names that should be parsed (should have a header)"); +DEFINE_string( + input_global_params_path, + "out.csv_global_params_0", + "Input file name of global parameter setup. Used when reading inputs in secret share format rather than plaintext."); +DEFINE_string( + output_base_path, + "", + "Local or s3 base path where output files are written to"); +DEFINE_string( + output_directory, + "", + "Local or s3 path where output files are written to"); +DEFINE_string( + output_filenames, + "out.csv_0[,out.csv_1,out.csv_2,...]", + "List of output file names that correspond to input filenames (positionally)"); +DEFINE_string( + run_id, + "", + "A run_id used to identify all the logs in a PL run."); diff --git a/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h new file mode 100644 index 000000000..c961dcc3d --- /dev/null +++ b/fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h" + +DECLARE_bool(is_conversion_lift); +DECLARE_string(input_directory); +DECLARE_string(input_expanded_key_path); +DECLARE_string(input_filenames); +DECLARE_string(input_global_params_path); +DECLARE_string(output_base_path); +DECLARE_string(output_directory); +DECLARE_string(output_filenames); +DECLARE_string(run_id); diff --git a/fbpcs/emp_games/lift/pcf2_calculator/main.cpp b/fbpcs/emp_games/lift/pcf2_calculator/main.cpp index 0caba8637..5ec07e07e 100644 --- a/fbpcs/emp_games/lift/pcf2_calculator/main.cpp +++ b/fbpcs/emp_games/lift/pcf2_calculator/main.cpp @@ -19,114 +19,10 @@ #include "fbpcf/aws/AwsSdk.h" #include "fbpcs/emp_games/common/FeatureFlagUtil.h" +#include "fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h" #include "fbpcs/emp_games/lift/pcf2_calculator/MainUtil.h" #include "fbpcs/performance_tools/CostEstimation.h" -DEFINE_int32(party, 1, "1 = publisher, 2 = partner"); -DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address"); -DEFINE_int32( - port, - 10000, - "Network port for establishing connection to other player"); -DEFINE_string( - input_directory, - "", - "Data directory where input files are located"); -DEFINE_string( - input_filenames, - "in.csv_0[,in.csv_1,in.csv_2,...]", - "List of input file names that should be parsed (should have a header)"); -DEFINE_string( - input_global_params_path, - "out.csv_global_params_0", - "Input file name of global parameter setup. Used when reading inputs in secret share format rather than plaintext."); -DEFINE_string( - input_expanded_key_path, - "out.csv_expanded_key_0", // TODO - "Input file name of the expanded key for UDP decryption. Used when decoupled UDP is enabled."); -DEFINE_string( - output_directory, - "", - "Local or s3 path where output files are written to"); -DEFINE_string( - output_filenames, - "out.csv_0[,out.csv_1,out.csv_2,...]", - "List of output file names that correspond to input filenames (positionally)"); -DEFINE_string( - input_base_path, - "", - "Local or s3 base path for the sharded input files"); -DEFINE_string( - output_base_path, - "", - "Local or s3 base path where output files are written to"); -DEFINE_int32( - file_start_index, - 0, - "First file that will be read with base path"); -DEFINE_int32(num_files, 0, "Number of files that should be read"); -DEFINE_int64( - epoch, - 1546300800, - "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps"); -DEFINE_bool( - is_conversion_lift, - true, - "Use conversion_lift logic (as opposed to converter_lift logic)"); -DEFINE_bool( - use_xor_encryption, - true, - "Reveal output with XOR secret shares instead of in the clear to both parties"); -DEFINE_int32( - num_conversions_per_user, - 4, - "Cap and pad to this many conversions per user"); -DEFINE_int32( - concurrency, - 1, - "max number of game(s) that will run concurrently?"); -DEFINE_string( - run_name, - "", - "A user given run name that will be used in s3 filename"); -DEFINE_string( - run_id, - "", - "A run_id used to identify all the logs in a PL run."); -DEFINE_bool( - log_cost, - false, - "Log cost info into cloud which will be used for dashboard"); -DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name"); -DEFINE_string( - log_cost_s3_region, - ".s3.us-west-2.amazonaws.com/", - "s3 regioni name"); -DEFINE_bool( - compute_publisher_breakdowns, - true, - "To enable or disable computing publisher breakdown for result validation"); -DEFINE_string( - pc_feature_flags, - "", - "A String of PC Feature Flags passing from PCS, separated by comma"); -DEFINE_bool( - use_tls, - false, - "Whether to use TLS when communicating with other parties."); -DEFINE_string( - ca_cert_path, - "", - "Relative file path where root CA cert is stored. It will be prefixed with $HOME."); -DEFINE_string( - server_cert_path, - "", - "Relative file path where server cert is stored. It will be prefixed with $HOME."); -DEFINE_string( - private_key_path, - "", - "Relative file path where private key is stored. It will be prefixed with $HOME."); - int main(int argc, char** argv) { folly::init(&argc, &argv); gflags::ParseCommandLineFlags(&argc, &argv, true);