Implementation of Integration Testing Framework For Model Compiler

WORKSPACE

@@ -55,7 +55,7 @@ common_cpp_deps()
 # Virtual-people-common
 http_archive(
     name = "virtual_people_common",
-    sha256 = "23591a296092da0cca03732072a84625ead0eec36ef8f38a87940e6af2d706a5",
-    strip_prefix = "virtual-people-common-ed17b00b7fe7a2c5646479e42dc3bb32b2f5c80f",
-    url = "https://github.com/world-federation-of-advertisers/virtual-people-common/archive/ed17b00b7fe7a2c5646479e42dc3bb32b2f5c80f.tar.gz",
+    sha256 = "dbfd7ac163f44118104a50fea8c821f14d6444f130f6f2998b26f1142843c703",
+    strip_prefix = "virtual-people-common-1ee774a28062186f39d82c7d1432111df5ab93f2",
+    url = "https://github.com/world-federation-of-advertisers/virtual-people-common/archive/1ee774a28062186f39d82c7d1432111df5ab93f2.tar.gz",
 )

src/main/cc/wfa/virtual_people/training/model_compiler/BUILD.bazel

@@ -3,6 +3,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
 package(default_visibility = [
     "//src/main/cc/wfa/virtual_people/training/model_compiler:__subpackages__",
     "//src/test/cc/wfa/virtual_people/training/model_compiler:__subpackages__",
+    "//src/test/cc/wfa/virtual_people/training/util:__subpackages__",
 ])
 
 _INCLUDE_PREFIX = "/src/main/cc"

src/test/cc/wfa/virtual_people/training/model_compiler/BUILD.bazel

@@ -81,3 +81,4 @@ cc_test(
         "@wfa_common_cpp//src/main/cc/common_cpp/testing:status",
     ],
 )
+

src/test/cc/wfa/virtual_people/training/model_compiler/test_data/BUILD.bazel

@@ -1 +1,8 @@
+package(default_visibility = ["//visibility:public"])
+
 exports_files(glob(["*.textproto"]))
+
+filegroup(
+    name = "test_data",
+    srcs = glob(["*.textproto"]),
+)

src/test/cc/wfa/virtual_people/training/util/BUILD.bazel

@@ -0,0 +1,41 @@
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_test")
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "golden_generator",
+    srcs = ["golden_generator.cc"],
+    data = [
+        "//src/test/cc/wfa/virtual_people/training/model_compiler/test_data",
+        "//src/test/cc/wfa/virtual_people/training/util/test_data",
+    ],
+    deps = [
+        "@com_github_google_glog//:glog",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/flags:flag",
+        "@com_google_absl//absl/flags:parse",
+        "@virtual_people_common//src/main/cc/wfa/virtual_people/common/integration_testing_framework:golden_generator",
+        "@virtual_people_common//src/main/proto/wfa/virtual_people/common:config_cc_proto",
+        "@wfa_common_cpp//src/main/cc/common_cpp/protobuf_util:textproto_io",
+    ],
+)
+
+cc_test(
+    name = "integration_test",
+    srcs = ["integration_test.cc"],
+    data = [
+        "//src/main/cc/wfa/virtual_people/training/model_compiler:compiler_main",
+        "//src/test/cc/wfa/virtual_people/training/model_compiler/test_data",
+        "//src/test/cc/wfa/virtual_people/training/util/test_data",
+        "@virtual_people_common//src/main/proto/wfa/virtual_people/common:model_proto",
+    ],
+    deps = [
+        "@bazel_tools//tools/cpp/runfiles",
+        "@com_github_google_glog//:glog",
+        "@com_google_absl//absl/status",
+        "@com_google_googletest//:gtest_main",
+        "@com_google_protobuf//:protobuf",
+        "@virtual_people_common//src/main/proto/wfa/virtual_people/common:config_cc_proto",
+        "@wfa_common_cpp//src/main/cc/common_cpp/protobuf_util:textproto_io",
+    ],
+)

src/test/cc/wfa/virtual_people/training/util/golden_generator.cc

@@ -0,0 +1,62 @@
+// Copyright 2022 The Cross-Media Measurement Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a tool to generate golden files for binaries based on input config.
+// Example usage:
+// bazel build -c opt \
+// //src/test/cc/wfa/virtual_people/training/util:golden_generator.cc
+// bazel-bin/src/test/cc/wfa/virtual_people/training/util/golden_generator
+// or
+// bazel build -c opt \
+// //src/test/cc/wfa/virtual_people/training/util:golden_generator.cc
+// bazel-bin/src/test/cc/wfa/virtual_people/training/util/golden_generator
+// --input_path=path/to/config.textproto
+
+#include <string>
+#include <vector>
+
+#include "absl/flags/flag.h"
+#include "absl/flags/parse.h"
+#include "absl/status/status.h"
+#include "common_cpp/protobuf_util/textproto_io.h"
+#include "glog/logging.h"
+#include "wfa/virtual_people/common/config.pb.h"
+#include "wfa/virtual_people/common/integration_testing_framework/golden_generator.h"
+
+ABSL_FLAG(
+    std::string, input_path,
+    "src/test/cc/wfa/virtual_people/training/util/test_data/config.textproto",
+    "Path to the input config textproto");
+
+int main(int argc, char** argv) {
+  absl::ParseCommandLine(argc, argv);
+
+  std::string input_path = absl::GetFlag(FLAGS_input_path);
+  CHECK(!input_path.empty()) << "input_path is not set.";
+
+  wfa_virtual_people::IntegrationTestList config;
+  absl::Status readConfigStatus = wfa::ReadTextProtoFile(input_path, config);
+  CHECK(readConfigStatus == absl::OkStatus())
+      << "Read Config Status: " << readConfigStatus;
+
+  // TODO: Runs into segmentation fault after calling GoldenGenerator
+  std::vector<std::string> executeVector(GoldenGenerator(config));
+
+  for (std::string execute : executeVector) {
+    CHECK(std::system(execute.c_str()))
+        << "Execution of " << execute << " failed.";
+  }
+
+  return 0;
+}

src/test/cc/wfa/virtual_people/training/util/integration_test.cc

@@ -0,0 +1,175 @@
+// Copyright 2022 The Cross-Media Measurement Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fcntl.h>
+
+#include <string>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "common_cpp/protobuf_util/textproto_io.h"
+#include "glog/logging.h"
+#include "google/protobuf/compiler/parser.h"
+#include "google/protobuf/dynamic_message.h"
+#include "google/protobuf/io/tokenizer.h"
+#include "google/protobuf/io/zero_copy_stream_impl.h"
+#include "google/protobuf/util/message_differencer.h"
+#include "gtest/gtest.h"
+#include "tools/cpp/runfiles/runfiles.h"
+#include "wfa/virtual_people/common/config.pb.h"
+
+using bazel::tools::cpp::runfiles::Runfiles;
+using ::wfa::ReadTextProtoFile;
+
+namespace wfa_virtual_people {
+namespace {
+
+struct Targets {
+  std::vector<std::string> protoDependecies;
+  std::string name;
+  std::string output;
+  std::string golden;
+  std::string proto;
+  std::string protoType;
+};
+
+// Returns a list of Targets parsed from the input config.
+// Additionally, locates the runfiles path for the given binary and executes the
+// binary with its given binary_parameters from the input config.
+std::vector<Targets> ParseConfig(std::string path) {
+  IntegrationTestList config;
+  std::vector<Targets> targets;
+  std::vector<std::string> protoDependencies;
+  std::unique_ptr<Runfiles> runfiles(Runfiles::CreateForTest());
+  std::string name, output, golden, proto, protoType, rPath, execute;
+
+  absl::Status readConfigStatus = ReadTextProtoFile(path, config);
+  CHECK(readConfigStatus == absl::OkStatus())
+      << "Read Config Status: " << readConfigStatus;
+
+  for (int testIndex = 0; testIndex < config.tests_size(); testIndex++) {
+    IntegrationTest it = config.tests().at(testIndex);
+    rPath = runfiles->Rlocation(it.binary());
+    for (int testCaseIndex = 0; testCaseIndex < it.test_cases_size();
+         testCaseIndex++) {
+      TestCase tc = it.test_cases().at(testCaseIndex);
+      name = it.name() + "_" + tc.name();
+      execute = rPath;
+      for (int binaryParameterIndex = 0;
+           binaryParameterIndex < tc.binary_parameters_size();
+           binaryParameterIndex++) {
+        BinaryParameter bp = tc.binary_parameters().at(binaryParameterIndex);
+        execute += " --" + bp.name() + "=" + bp.value();
+        if (!bp.golden().golden_path().empty() && open(bp.golden().golden_path().data(), O_RDONLY) != -1) {
+          protoDependencies = std::vector<std::string>(
+              bp.golden().proto_dependencies().begin(), bp.golden().proto_dependencies().end());
+          protoDependencies.push_back(bp.golden().proto_path());
+          output = bp.value();
+          golden = bp.golden().golden_path();
+          proto = bp.golden().proto_path();
+          protoType = bp.golden().proto_type();
+          targets.push_back(
+              {protoDependencies, name, output, golden, proto, protoType});
+        }
+      }
+      std::system(execute.c_str());
+    }
+  }
+
+  return targets;
+}
+
+// Declares classes and variables used in the parameterized TEST_P.
+class IntegrationTestParamaterizedFixture
+    : public ::testing::TestWithParam<Targets> {
+ protected:
+  google::protobuf::DescriptorPool descriptorPoolUnderlay;
+  google::protobuf::FileDescriptorProto fileDescriptorProto;
+  google::protobuf::compiler::Parser parser;
+  google::protobuf::util::MessageDifferencer messageDifferencer;
+};
+
+// Parses proto type to compare from input config and compares output textproto
+// against golden textproto.
+TEST_P(IntegrationTestParamaterizedFixture, Test) {
+  Targets targets(GetParam());
+
+  google::protobuf::DescriptorPool descriptorPool(&descriptorPoolUnderlay);
+
+  descriptorPool.AllowUnknownDependencies();
+  descriptorPoolUnderlay.AllowUnknownDependencies();
+
+  std::unique_ptr<Runfiles> runfiles(Runfiles::CreateForTest());
+
+  for (std::string protoDependency : targets.protoDependecies) {
+    std::string rPath = runfiles->Rlocation(protoDependency);
+
+    int fd = open(rPath.data(), O_RDONLY);
+    google::protobuf::io::FileInputStream fileInputStream(fd);
+    CHECK(fileInputStream.GetErrno() == 0)
+        << "Errno: " << fileInputStream.GetErrno();
+
+    google::protobuf::io::Tokenizer tokenizer(&fileInputStream, NULL);
+    fileInputStream.SetCloseOnDelete(true);
+    fileDescriptorProto.set_name(protoDependency);
+    parser.Parse(&tokenizer, &fileDescriptorProto);
+    fileDescriptorProto.clear_options();
+    if (protoDependency == targets.proto) {
+      descriptorPool.BuildFile(fileDescriptorProto);
+    } else {
+      descriptorPoolUnderlay.BuildFile(fileDescriptorProto);
+    }
+  }
+
+  google::protobuf::DynamicMessageFactory dynamicMessageFactory;
+
+  const google::protobuf::FileDescriptor* fileDescriptor =
+      descriptorPool.FindFileByName(targets.proto);
+  const google::protobuf::Descriptor* descriptor =
+      fileDescriptor->FindMessageTypeByName(targets.protoType);
+  const google::protobuf::Message* immutableMessage =
+      dynamicMessageFactory.GetPrototype(descriptor);
+
+  std::unique_ptr<google::protobuf::Message> output =
+      std::unique_ptr<google::protobuf::Message>(immutableMessage->New());
+  absl::Status outputStatus = ReadTextProtoFile(targets.output, *output);
+  CHECK(outputStatus == absl::OkStatus()) << "Output Status: " << outputStatus;
+
+  std::unique_ptr<google::protobuf::Message> golden =
+      std::unique_ptr<google::protobuf::Message>(immutableMessage->New());
+  absl::Status goldenStatus = ReadTextProtoFile(targets.golden, *golden);
+  CHECK(goldenStatus == absl::OkStatus()) << "Golden Status: " << goldenStatus;
+
+  if (messageDifferencer.Equals(*output, *golden)) {
+    SUCCEED();
+  } else {
+    FAIL() << std::system(("diff " + targets.output + " " + targets.golden).c_str());
+  }
+}
+
+// Instantiates all test cases from the input config and assigns each one a
+// name.
+INSTANTIATE_TEST_SUITE_P(
+    IntegrationTest, IntegrationTestParamaterizedFixture,
+    ::testing::ValuesIn(
+        ParseConfig("src/test/cc/wfa/virtual_people/training/util/"
+                    "test_data/config.textproto")),
+    [](const ::testing::TestParamInfo<
+        IntegrationTestParamaterizedFixture::ParamType>& info) {
+      std::string name = info.param.name;
+      return name;
+    });
+
+}  // namespace
+}  // namespace wfa_virtual_people

src/test/cc/wfa/virtual_people/training/util/test_data/BUILD.bazel

@@ -0,0 +1,8 @@
+package(default_visibility = ["//visibility:public"])
+
+exports_files(glob(["*.textproto"]))
+
+filegroup(
+    name = "test_data",
+    srcs = glob(["*.textproto"]),
+)

src/test/cc/wfa/virtual_people/training/util/test_data/config.textproto

@@ -0,0 +1,72 @@
+tests {
+    name: "model_compiler"
+    binary: "virtual_people_training/src/main/cc/wfa/virtual_people/training/model_compiler/compiler_main"
+    test_cases {
+      name: "population_node"
+      binary_parameters {
+        name: "input_path"
+        value: "src/test/cc/wfa/virtual_people/training/model_compiler/test_data/model_node_config_population_node.textproto"
+      }
+      binary_parameters {
+        name: "output_path"
+        value: "src/test/cc/wfa/virtual_people/training/model_compiler/compiled_node_for_population_node.textproto"
+        golden {
+          golden_path: "src/test/cc/wfa/virtual_people/training/model_compiler/test_data/compiled_node_for_population_node.textproto"
+          proto_path: "virtual_people_common/src/main/proto/wfa/virtual_people/common/_virtual_imports/model_proto/wfa/virtual_people/common/model.proto"
+          proto_dependencies: "virtual_people_common/src/main/proto/wfa/virtual_people/common/_virtual_imports/field_filter_proto/wfa/virtual_people/common/field_filter.proto"
+          proto_type: "CompiledNode"
+        }
+      }
+    }
+    test_cases {
+      name: "population_node_redistribute_probabilities_for_empty_pools"
+      binary_parameters {
+        name: "input_path"
+        value: "src/test/cc/wfa/virtual_people/training/model_compiler/test_data/model_node_config_population_node_redistribute_probabilities_for_empty_pools.textproto"
+      }
+      binary_parameters {
+        name: "output_path"
+        value: "src/test/cc/wfa/virtual_people/training/model_compiler/compiled_node_for_population_node_redistribute_probabilities_for_empty_pools.textproto"
+        golden {
+          golden_path: "src/test/cc/wfa/virtual_people/training/model_compiler/test_data/compiled_node_for_population_node_redistribute_probabilities_for_empty_pools.textproto"
+          proto_path: "virtual_people_common/src/main/proto/wfa/virtual_people/common/_virtual_imports/model_proto/wfa/virtual_people/common/model.proto"
+          proto_dependencies: "virtual_people_common/src/main/proto/wfa/virtual_people/common/_virtual_imports/field_filter_proto/wfa/virtual_people/common/field_filter.proto"
+          proto_type: "CompiledNode"
+        }
+      }
+    }
+    test_cases {
+      name: "population_node_kappa_less_than_one"
+      binary_parameters {
+        name: "input_path"
+        value: "src/test/cc/wfa/virtual_people/training/model_compiler/test_data/model_node_config_population_node_kappa_less_than_one.textproto"
+      }
+      binary_parameters {
+        name: "output_path"
+        value: "src/test/cc/wfa/virtual_people/training/model_compiler/compiled_node_for_population_node_kappa_less_than_one.textproto"
+        golden {
+          golden_path: "src/test/cc/wfa/virtual_people/training/model_compiler/test_data/compiled_node_for_population_node_kappa_less_than_one.textproto"
+          proto_path: "virtual_people_common/src/main/proto/wfa/virtual_people/common/_virtual_imports/model_proto/wfa/virtual_people/common/model.proto"
+          proto_dependencies: "virtual_people_common/src/main/proto/wfa/virtual_people/common/_virtual_imports/field_filter_proto/wfa/virtual_people/common/field_filter.proto"
+          proto_type: "CompiledNode"
+        }
+      }
+    }
+    test_cases {
+      name: "population_node_discretization"
+      binary_parameters {
+        name: "input_path"
+        value: "src/test/cc/wfa/virtual_people/training/model_compiler/test_data/model_node_config_population_node_discretization.textproto"
+      }
+      binary_parameters {
+        name: "output_path"
+        value: "src/test/cc/wfa/virtual_people/training/model_compiler/compiled_node_for_population_node_discretization.textproto"
+        golden {
+          golden_path: "src/test/cc/wfa/virtual_people/training/model_compiler/test_data/compiled_node_for_population_node_discretization.textproto"
+          proto_path: "virtual_people_common/src/main/proto/wfa/virtual_people/common/_virtual_imports/model_proto/wfa/virtual_people/common/model.proto"
+          proto_dependencies: "virtual_people_common/src/main/proto/wfa/virtual_people/common/_virtual_imports/field_filter_proto/wfa/virtual_people/common/field_filter.proto"
+          proto_type: "CompiledNode"
+        }
+      }
+    }
+  }