Changes to make v1.22.10 build on SmartOS.

.bazelrc

@@ -16,8 +16,9 @@ build --color=yes
 build --workspace_status_command="bash bazel/get_workspace_status"
 build --incompatible_strict_action_env
 build --host_force_python=PY3
-build --java_runtime_version=remotejdk_11
-build --tool_java_runtime_version=remotejdk_11
+# On Illumos we need to use the OpenJDK version installed locally.
+#build --java_runtime_version=remotejdk_11
+#build --tool_java_runtime_version=remotejdk_11
 build --platform_mappings=bazel/platform_mappings
 build --experimental_repository_downloader_retries=2
 
@@ -407,3 +408,20 @@ build:windows --dynamic_mode=off
 try-import %workspace%/clang.bazelrc
 try-import %workspace%/user.bazelrc
 try-import %workspace%/local_tsan.bazelrc
+
+# Illumos
+build:illumos --cxxopt=-std=c++17
+build:illumos --define hot_restart=disabled
+build:illumos --define wasm=disabled
+# Disables gperftools. I (@siepkes) couldn't get 'libtcmalloc_and_profiler.a' to be built.
+# Not because it doesn't build on Illumos but because I couldn't get Bazel -> foreign_cc
+# -> autoconf to build the thing. And then Bazel complains about not finding it.
+# When building the "vanilla" gperftools repo the 'libtcmalloc_and_profiler.a' artifact
+# is build just fine.
+build:illumos --define tcmalloc=disabled
+
+build:illumos --java_runtime_version=local_jdk
+build:illumos --java_language_version=11
+build:illumos --tool_java_runtime_version=local_jdk
+build:illumos --tool_java_language_version=11
+build:illumos --extra_toolchains=//:repository_default_toolchain_definition

BUILD

@@ -53,3 +53,22 @@ package_group(
         "//examples/...",
     ],
 )
+
+load(
+  "@bazel_tools//tools/jdk:default_java_toolchain.bzl",
+  "default_java_toolchain", "DEFAULT_TOOLCHAIN_CONFIGURATION", "BASE_JDK9_JVM_OPTS", "DEFAULT_JAVACOPTS"
+)
+
+# On Illumos this config gets activated. We use it to force Bazel to use the local (pkgsrc) JDK. If we don't
+# it will try to use '@bazel_tools//tools/jdk:remote_jdk11'. Which won't work because there is no remote Illumos
+# JDK which can be downloaded configured in the 'rules_java' project.
+default_java_toolchain(
+  name = "repository_default_toolchain",
+  configuration = DEFAULT_TOOLCHAIN_CONFIGURATION,        # One of predefined configurations
+                                                          # Other parameters are from java_toolchain rule:
+  java_runtime = "@local_jdk//:jdk",                      # JDK to use for compilation and toolchain's tools execution
+  jvm_opts = BASE_JDK9_JVM_OPTS,   # Additional JDK options
+  javacopts = DEFAULT_JAVACOPTS,   # Additional javac options
+  source_version = "11",
+  target_version = "11",
+)

README.md

@@ -1,3 +1,215 @@
+# Envoy SmartOS / Illumos / Solaris port
+
+**(Scroll down for original Envoy readme.md)**
+
+This repo contains a SmartOS port for Envoy. It will probably also work on Solaris though it will probably require modifications since we assume the use of pkgsrc.
+
+To build this Envoy port you need Bazel. This requires a SmartOS / Illumos / Solaris port of Bazel since Bazel does not natively support these platforms. See the [bazel-smartos](https://github.com/siepkes/bazel-smartos) repo for a SmartOS port. My intention being to properly upstream the thing but that takes some work... Feel free to reach out to me if you encounter any issues.
+
+## Running Envoy on SmartOS
+
+**WARNING: When running set the environmental variable `EVENT_NOEVPORT=yes`.**
+
+Envoy uses libevent which uses event ports on Illumos (the native non-blocking IO implementation on Illumos). For some reason when using event ports libevent starts making a massive number of syscalls (as many as the CPU limits allow). Therefor we disable the event ports implementation in libevent for now.
+
+```
+$ export EVENT_NOEVPORT=yes
+$ ./envoy-static --disable-hot-restart -c ./config.yaml
+```
+
+## Building Envoy on SmartOS
+
+Create a SmartOS container (`joyent` brand if your on Joyent's public cloud / Triton). Steps below are performed on a container running the `base-64-lts` image version `20.4.0` (`1d05e788-5409-11eb-b12f-037bd7fee4ee`).
+
+The following things are good to know:
+
+* As stated in the [bazel-smartos](https://github.com/siepkes/bazel-smartos) repo the Bazel binary depends on a specific GCC version (due to hardcoded version in some paths).
+* When using `export NUM_CPUS=2` you probably require at least 16GB of RAM and 32 GB of swap in your VM to build Envoy. Trying to build with an 8GB VM led to an `ld: fatal: mmap anon failed: Resource temporarily unavailable` error while linking in my case. You can experiment with lowering `NUM_CPUS` if you run in to memory problems.
+
+Install required build packages. Envoy makes use of modern C++ so will only work with a modern C++ compiler.
+
+```
+# pkgin -y install go117 ninja-build gcc9 git-base zip unzip openjdk11 libtool cmake automake ninja-build autoconf gmake python39 py39-expat
+```
+
+Recent version of Bazel's `rules_pkg` run some tests which expect the ability to use UTF-8 characters in directory and file names. Modify `/etc/defaults/init` and add the following to enable UTF-8 support. Reboot the container after making these modifications:
+
+```
+CMASK=022
+# Specifies the locale to use for all purposes except as overridden by the variables below.
+LANG=en_US.UTF-8
+# Determines the effect of character handling functions.
+LC_CTYPE=en_US.UTF-8
+# Determines the decimal point character and thousands separator character.
+LC_NUMERIC=en_US.UTF-8
+# Determines the date/time format.
+LC_TIME=en_US.UTF-8
+# Determines the order in which the output is sorted.
+LC_COLLATE=en_US.UTF-8
+# Determines the format for monetary symbols.
+LC_MONETARY=en_US.UTF-8
+# Determines the format of standard operating system messages.
+LC_MESSAGES=en_US.UTF-8
+# Used to set a single locale for all purposes, overrides all variables listed above.
+# Not always supported by all applications.
+LC_ALL=en_US.UTF-8
+```
+
+Bazel will try to build the extensions that use Python (for example Kafka filter) for every Python version that is installed. Meaning you need to have the Python modules such as `py39-expat` installed for every installed Python version (for example `py37-expat` if Python 3.7 is also installed). When bumping Triton image version verify the package Python version. So beware that having other versions of Python installed in your build VM might complicate the build process. The same goes for Go; Having a more recent version or multiple versions of Go installed can lead to build issues.
+
+Configure the build environment:
+```
+$ git clone https://github.com/siepkes/envoy-smartos.git
+```
+
+Build Envoy with clang (building with GCC might not work due to [issues](https://github.com/envoyproxy/envoy/issues/14788)):
+```
+$ cd envoy-smartos
+$ git checkout smartos-v1.22.10
+$ export NUM_CPUS=2  # Needed to prevent a CPU detection algorithm from going awry.
+$ export JAVA_HOME="/opt/local/java/openjdk11"
+$ bazel build -c opt --jobs=3 \
+   --config=illumos \
+   --define hot_restart=disabled \
+   --verbose_failures \
+   --subcommands \
+   --sandbox_debug \
+   --toolchain_resolution_debug \
+   --//source/extensions/bootstrap/wasm:enabled=false \
+   --//source/extensions/wasm_runtime/v8:enabled=false \
+   --package_path %workspace%:/root/envoy-smartos/ \
+   //source/exe:envoy-static
+```
+
+To troubleshoot build issues Bazel can be made more talkative by adding the following flags:
+
+```
+--sandbox_debug --verbose_failures --toolchain_resolution_debug
+```
+
+This will result in a statically linked binary of Envoy in `./bazel-bin/source/exe/envoy-static`.
+
+The binary will include debug symbols which you can strip to bring down the size of the binary substantially. Beware that this will make the backtrace library unusable (ie. stacktraces become hard to read):
+
+```
+$ strip --strip-debug ./bazel-bin/source/exe/envoy-static
+```
+
+## Known issues / TODO's / Remarks
+
+Below is a list of known issues of this port. These are mostly open issues because they represent functionality I didn't need right away and stood in the way of doing a sucessful build. I'm obviously open to any PR / help anyone can offer though!
+
+### Make webassembly runtime work
+
+We currently disable WASM in `.bazerc` when building. Reason for this is that the V8 WASM runtime currently doesn't build on Illumos. Envoy can be configured to use a different WASM runtime but for now WASM is just disabled.
+
+Additionally building the WASM extensions which GCC does not work. Leading to errors such as the one below. Apparantly these issues are not present when using clang instead of GCC (See [Envoy issue 14788](https://github.com/envoyproxy/envoy/issues/14788)).
+
+```
+external/com_google_absl/absl/time/internal/cctz/include/cctz/civil_time_detail.h: In function 'constexpr int absl::time_internal::cctz::detail::impl::days_per_month(absl::time_internal::cctz::year_t, absl::time_internal::cctz::detail::month_t)':
+external/com_google_absl/absl/time/internal/cctz/include/cctz/civil_time_detail.h:104:28: warning: array subscript has type 'char' [-Wchar-subscripts]
+   return k_days_per_month[m] + (m == 2 && is_leap_year(y));
+                            ^
+external/com_google_cel_cpp/eval/eval/ternary_step.cc: In function 'absl::StatusOr<std::unique_ptr<google::api::expr::runtime::ExpressionStep> > google::api::expr::runtime::CreateTernaryStep(int64_t)':
+external/com_google_cel_cpp/eval/eval/ternary_step.cc:75:10: error: could not convert 'step' from 'std::unique_ptr<google::api::expr::runtime::ExpressionStep>' to 'absl::StatusOr<std::unique_ptr<google::api::expr::runtime::ExpressionStep> >'
+   return step;
+          ^~~~
+
+```
+
+### Make event ports work
+
+Currently we disable event ports by using the environmental variable `EVENT_NOEVPORT=yes`. When using event ports Envoy (or more likely libevent) starts making a massive number of syscalls. I'm guessing this is because some (event) loop in libevent is going haywire. Probably need to take a look at `libevent_scheduler.cc` how libevent is configured.
+
+### Final binary requires GCC package
+
+Due to the way the linking is currently configured the final Envoy binary requires the GCC package to be installed in the container:
+
+```
+$ ldd bazel-bin/source/exe/envoy-static
+        librt.so.1 =>    /lib/64/librt.so.1
+        libdl.so.1 =>    /lib/64/libdl.so.1
+        libpthread.so.1 =>       /lib/64/libpthread.so.1
+        libm.so.2 =>     /lib/64/libm.so.2
+        libstdc++.so.6 =>        /opt/local/gcc7//lib/amd64/libstdc++.so.6
+        libxnet.so.1 =>  /lib/64/libxnet.so.1
+        libsocket.so.1 =>        /lib/64/libsocket.so.1
+        libnsl.so.1 =>   /lib/64/libnsl.so.1
+        libgcc_s.so.1 =>         /opt/local/gcc7//lib/amd64/libgcc_s.so.1
+        libc.so.1 =>     /lib/64/libc.so.1
+        libmp.so.2 =>    /lib/64/libmp.so.2
+        libmd.so.1 =>    /lib/64/libmd.so.1
+```
+
+### Get entire test suite to run
+
+Headline covers it.
+
+```
+$ export JAVA_HOME="/opt/local/java/openjdk11"
+$ bazel test --host_javabase=@local_jdk//:jdk //test/...
+```
+
+### Hot restart disabled
+
+Currently we pass `--define hot_restart=disabled` via `.bazelrc` when building to disable Hot restart (ie. restart Envoy without client connections being closed). Hot restart is disabled because it didn't work without modifications and I didn't have a need for it.
+
+#### py37-expat package requirement.
+
+The `py37-expat` package must be installed otherwise the build dies with the output below. I (@siepkes) think this might actually be a bug in upstream since requiring manual install of the package is not really what Bazel is about?
+
+```
+[INFO 08:56:26.568 src/main/cpp/rc_file.cc:131] Skipped optional import of /root/envoy-smartos/local_tsan.bazelrc, the specified rc file either does not exist or is not readable.
+[INFO 08:56:26.568 src/main/cpp/rc_file.cc:56] Parsing the RcFile /dev/null
+[INFO 08:56:26.569 src/main/cpp/blaze.cc:1623] Debug logging requested, sending all client log statements to stderr
+[INFO 08:56:26.570 src/main/cpp/blaze.cc:1506] Acquired the client lock, waited 0 milliseconds
+[INFO 08:56:26.577 src/main/cpp/blaze.cc:1694] Trying to connect to server (timeout: 30 secs)...
+[INFO 08:56:26.590 src/main/cpp/blaze_util_illumos.cc:126] PID: 658256 (/root/envoy-smartos).
+[INFO 08:56:26.590 src/main/cpp/blaze.cc:1261] Connected (server pid=658256).
+[INFO 08:56:26.590 src/main/cpp/blaze.cc:1971] Releasing client lock, let the server manage concurrent requests.
+INFO: Repository config_validation_pip3 instantiated at:
+  no stack (--record_rule_instantiation_callstack not enabled)
+Repository rule pip_import defined at:
+  /root/.cache/bazel/_bazel_root/7558a64af10a6eb79f74e70211660103/external/rules_python/python/pip.bzl:51:29: in <toplevel>
+ERROR: An error occurred during the fetch of repository 'config_validation_pip3':
+   pip_import failed:  (Traceback (most recent call last):
+  File "/opt/local/lib/python3.7/runpy.py", line 193, in _run_module_as_main
+    "__main__", mod_spec)
+  File "/opt/local/lib/python3.7/runpy.py", line 85, in _run_code
+    exec(code, run_globals)
+  File "/root/.cache/bazel/_bazel_root/7558a64af10a6eb79f74e70211660103/external/rules_python/tools/piptool.par/__main__.py", line 26, in <module>
+  File "/root/.cache/bazel/_bazel_root/7558a64af10a6eb79f74e70211660103/external/rules_python/tools/piptool.par/piptool_deps_pypi__setuptools_44_0_0/pkg_resources/__init__.py", line 35, in <module>
+  File "/opt/local/lib/python3.7/plistlib.py", line 65, in <module>
+    from xml.parsers.expat import ParserCreate
+  File "/opt/local/lib/python3.7/xml/parsers/expat.py", line 4, in <module>
+    from pyexpat import *
+ModuleNotFoundError: No module named 'pyexpat'
+)
+ERROR: no such package '@config_validation_pip3//': pip_import failed:  (Traceback (most recent call last):
+  File "/opt/local/lib/python3.7/runpy.py", line 193, in _run_module_as_main
+    "__main__", mod_spec)
+  File "/opt/local/lib/python3.7/runpy.py", line 85, in _run_code
+    exec(code, run_globals)
+  File "/root/.cache/bazel/_bazel_root/7558a64af10a6eb79f74e70211660103/external/rules_python/tools/piptool.par/__main__.py", line 26, in <module>
+  File "/root/.cache/bazel/_bazel_root/7558a64af10a6eb79f74e70211660103/external/rules_python/tools/piptool.par/piptool_deps_pypi__setuptools_44_0_0/pkg_resources/__init__.py", line 35, in <module>
+  File "/opt/local/lib/python3.7/plistlib.py", line 65, in <module>
+    from xml.parsers.expat import ParserCreate
+  File "/opt/local/lib/python3.7/xml/parsers/expat.py", line 4, in <module>
+    from pyexpat import *
+ModuleNotFoundError: No module named 'pyexpat'
+)
+INFO: Elapsed time: 10.114s
+INFO: 0 processes.
+FAILED: Build did NOT complete successfully (0 packages loaded)
+    Fetching @headersplit_pip3; fetching 9s
+    Fetching @configs_pip3; fetching 9s
+    Fetching @kafka_pip3; fetching 9s
+    Fetching @thrift_pip3; fetching 9s
+    Fetching @protodoc_pip3; fetching 9s
+```
+
+# Original Envoy Readme
+
 ![Envoy Logo](https://github.com/envoyproxy/artwork/blob/main/PNG/Envoy_Logo_Final_PANTONE.png)
 
 [Cloud-native high-performance edge/middle/service proxy](https://www.envoyproxy.io/)

bazel/BUILD

@@ -471,6 +471,11 @@ config_setting(
     values = {"cpu": "x64_windows"},
 )
 
+config_setting(
+    name = "illumos_x86_64",
+    values = {"cpu": "x86_64"},
+)
+
 # Configuration settings to make doing selects for Apple vs non-Apple platforms
 # easier. More details: https://docs.bazel.build/versions/master/configurable-attributes.html#config_settingaliasing
 config_setting(
@@ -588,6 +593,7 @@ selects.config_setting_group(
         ":ios_x86_64",
         ":linux_x86_64",
         ":windows_x86_64",
+        "illumos_x86_64",
     ],
 )
 

bazel/dependency_imports.bzl

@@ -12,13 +12,18 @@ load("@base_pip3//:requirements.bzl", pip_dependencies = "install_deps")
 load("@emsdk//:emscripten_deps.bzl", "emscripten_deps")
 
 # go version for rules_go
-GO_VERSION = "1.17.5"
+# Align the Go version with the version that is installed in Illumos since we added the
+# 'go_register_toolchains(go_version = "host")' directive.
+GO_VERSION = "1.17.6"
 
 def envoy_dependency_imports(go_version = GO_VERSION):
     # TODO: allow building of tools for easier onboarding
     rules_foreign_cc_dependencies(register_default_tools = False, register_built_tools = False)
     go_rules_dependencies()
-    go_register_toolchains(go_version)
+    # Using 'host' makes Bazel use the go installation on our host. This
+    # is needed because the 'io_bazel_rules_go' tries to download a GO 
+    # installation. However it can't download one for Illumos / Solaris.
+    go_register_toolchains(go_version = "host")
     gazelle_dependencies()
     apple_rules_dependencies()
     pip_dependencies()

bazel/envoy_binary.bzl

@@ -40,7 +40,8 @@ def envoy_cc_binary(
         linkstatic = 1,
         visibility = visibility,
         malloc = tcmalloc_external_dep(repository),
-        stamp = 1,
+        # FIXME: Solaris ld doesn't support build-ld. Fix more elegantly.
+        stamp = 0,
         deps = deps,
         tags = tags,
         features = features,
@@ -74,14 +75,18 @@ def _envoy_linkopts():
             "-pthread",
             "-lrt",
             "-ldl",
-            "-Wl,-z,relro,-z,now",
-            "-Wl,--hash-style=gnu",
+    	    # The 'relro'flag is not supported by the Illumos linker.
+            "-Wl,-z,now",
+            # FIXME: GNU LD supports this option but the Illumos linker doesn't.
+            #'-Wl,--hash-style=gnu',
         ],
     }) + select({
         "@envoy//bazel:apple": [],
         "@envoy//bazel:boringssl_fips": [],
         "@envoy//bazel:windows_x86_64": [],
-        "//conditions:default": ["-pie"],
+        # FIXME: GCC on Illumos doesn't support position independent executables?
+        # "//conditions:default": ["-pie"],
+        "//conditions:default": [],
     }) + _envoy_select_exported_symbols(["-Wl,-E"])
 
 def _envoy_stamped_deps():
@@ -112,6 +117,7 @@ def _envoy_stamped_linkopts():
 
         # Note: assumes GNU GCC (or compatible) handling of `--build-id` flag.
         "//conditions:default": [
-            "-Wl,@$(location @envoy//bazel:gnu_build_id.ldscript)",
+            # TODO: Ilumos ld doesn't support build-ld. Fix more elegantly.
+            # "-Wl,@$(location @envoy//bazel:gnu_build_id.ldscript)",
         ],
     })

bazel/envoy_internal.bzl

@@ -7,7 +7,8 @@ def envoy_copts(repository, test = False):
     posix_options = [
         "-Wall",
         "-Wextra",
-        "-Werror",
+        # TODO: Doesn't work on Illumos.
+        #"-Werror",
         "-Wnon-virtual-dtor",
         "-Woverloaded-virtual",
         "-Wold-style-cast",

bazel/foreign_cc/BUILD

@@ -19,10 +19,15 @@ configure_make(
 # TODO: Consider our own gperftools.BUILD file as we do with many other packages
 configure_make(
     name = "gperftools_build",
+    # Additional flags needed so 'libtcmalloc_and_profiler.a' gets created.
     configure_options = [
         "--enable-shared=no",
         "--enable-frame-pointers",
         "--disable-libunwind",
+        "--enable-cpu-profiler",
+        "--enable-heap-profiler",
+        "--enable-heap-checker",
+        "--enable-debugalloc",
     ] + select({
         "//bazel:apple": ["AR=/usr/bin/ar"],
         "//conditions:default": [],