access_log: add ACCESS_LOG_TYPE substitution string (envoyproxy#26755)

Signed-off-by: ohadvano <[email protected]>

api/envoy/data/accesslog/v3/BUILD

@@ -6,6 +6,7 @@ licenses(["notice"])  # Apache 2
 
 api_proto_package(
     deps = [
+        "//envoy/annotations:pkg",
         "//envoy/config/core/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
     ],

api/envoy/data/accesslog/v3/accesslog.proto

@@ -10,6 +10,7 @@ import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
 import "google/protobuf/wrappers.proto";
 
+import "envoy/annotations/deprecation.proto";
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
 import "validate/validate.proto";
@@ -31,6 +32,19 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // Fields describing *upstream* interaction will explicitly include ``upstream``
 // in their name.
 
+enum AccessLogType {
+  NotSet = 0;
+  TcpUpstreamConnected = 1;
+  TcpPeriodic = 2;
+  TcpConnectionEnd = 3;
+  DownstreamStart = 4;
+  DownstreamPeriodic = 5;
+  DownstreamEnd = 6;
+  UpstreamPoolReady = 7;
+  UpstreamPeriodic = 8;
+  UpstreamEnd = 9;
+}
+
 message TCPAccessLogEntry {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.data.accesslog.v2.TCPAccessLogEntry";
@@ -80,7 +94,7 @@ message ConnectionProperties {
 }
 
 // Defines fields that are shared by all Envoy access logs.
-// [#next-free-field: 33]
+// [#next-free-field: 34]
 message AccessLogCommon {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.data.accesslog.v2.AccessLogCommon";
@@ -214,7 +228,13 @@ message AccessLogCommon {
   // And if it is necessary, unique ID or identifier can be added to the log entry
   // :ref:`stream_id <envoy_v3_api_field_data.accesslog.v3.AccessLogCommon.stream_id>` to
   // correlate all these intermediate log entries and final log entry.
-  bool intermediate_log_entry = 27;
+  //
+  // .. attention::
+  //
+  //   This field is deprecated in favor of ``access_log_type`` for better indication of the
+  //   type of the access log record.
+  bool intermediate_log_entry = 27
+      [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"];
 
   // If downstream connection in listener failed due to transport socket (e.g. TLS handshake), provides the
   // failure reason from the transport socket. The format of this field depends on the configured downstream
@@ -236,6 +256,14 @@ message AccessLogCommon {
   // For HTTP: Total number of bytes received from the upstream by the http stream.
   // For TCP: Total number of bytes sent to the upstream by the tcp proxy.
   uint64 upstream_wire_bytes_received = 32;
+
+  // The type of the access log, which indicates when the log was recorded.
+  // See :ref:`ACCESS_LOG_TYPE <config_access_log_format_access_log_type>` for the available values.
+  // In case the access log was recorded by a flow which does not correspond to one of the supported
+  // values, then the default value will be ``NotSet``.
+  // For more information about how access log behaves and when it is being recorded,
+  // please refer to :ref:`access logging <arch_overview_access_logs>`.
+  AccessLogType access_log_type = 33;
 }
 
 // Flags indicating occurrences during request/response processing.

changelogs/current.yaml

@@ -49,6 +49,10 @@ removed_config_or_runtime:
     removed runtime key ``envoy.reloadable_features.http_response_half_close`` and legacy code paths.
 
 new_features:
+- area: access_log
+  change: |
+    added %ACCESS_LOG_TYPE% substitution string, to help distinguishing between access log records and when they are being
+    recorded. Please refer to the access log configuration documentation for more information.
 - area: http
   change: |
     added Runtime feature ``envoy.reloadable_features.max_request_headers_size_kb`` to override the default value of
@@ -69,3 +73,7 @@ new_features:
     <envoy_v3_api_field_extensions.filters.http.ratelimit.v3.RateLimitPerRoute.domain>` to allow for setting rate limit domains on a
     per-route basis.
 deprecated:
+- area: access_log
+  change: |
+    deprecated (1.25.0) :ref:`intermediate_log_entry <envoy_v3_api_field_data.accesslog.v3.AccessLogCommon.intermediate_log_entry>`
+    in favour of :ref:`access_log_type <envoy_v3_api_field_data.accesslog.v3.AccessLogCommon.access_log_type>`.

contrib/golang/filters/http/source/golang_filter.cc

@@ -201,7 +201,8 @@ void Filter::onDestroy() {
 
 // access_log is executed before the log of the stream filter
 void Filter::log(const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*,
-                 const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&) {
+                 const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&,
+                 Envoy::AccessLog::AccessLogType) {
   // Todo log phase of stream filter
 }
 

contrib/golang/filters/http/source/golang_filter.h

@@ -153,7 +153,9 @@ class Filter : public Http::StreamFilter,
   void log(const Http::RequestHeaderMap* request_headers,
            const Http::ResponseHeaderMap* response_headers,
            const Http::ResponseTrailerMap* response_trailers,
-           const StreamInfo::StreamInfo& stream_info) override;
+           const StreamInfo::StreamInfo& stream_info,
+           Envoy::AccessLog::AccessLogType access_log_type =
+               Envoy::AccessLog::AccessLogType::NotSet) override;
 
   void onStreamComplete() override {}
 

docs/root/configuration/observability/access_log/usage.rst

@@ -1051,6 +1051,22 @@ The following command operators are supported:
 %FILTER_CHAIN_NAME%
   The :ref:`network filter chain name <envoy_v3_api_field_config.listener.v3.FilterChain.name>` of the downstream connection.
 
+.. _config_access_log_format_access_log_type:
+
+%ACCESS_LOG_TYPE%
+  The type of the access log, which indicates when the access log was recorded. If a non-supported log (from the list below),
+  uses this substitution string, then the value will be an empty string.
+
+  * TcpUpstreamConnected - When TCP Proxy filter has successfully established an upstream connection.
+  * TcpPeriodic - On any TCP Proxy filter periodic log record.
+  * TcpConnectionEnd - When a TCP connection is ended on TCP Proxy filter.
+  * DownstreamStart - When HTTP Connection Manager filter receives a new HTTP request.
+  * DownstreamPeriodic - On any HTTP Connection Manager periodic log record.
+  * DownstreamEnd - When an HTTP stream is ended on HTTP Connection Manager filter.
+  * UpstreamPoolReady - When a new HTTP request is received by the HTTP Router filter.
+  * UpstreamPeriodic - On any HTTP Router filter periodic log record.
+  * UpstreamEnd - When an HTTP request is finished on the HTTP Router filter.
+
 %ENVIRONMENT(X):Z%
   Environment value of environment variable X. If no valid environment variable X, '-' symbol will be used.
   Z is an optional parameter denoting string truncation up to Z characters long.

envoy/access_log/BUILD

@@ -17,5 +17,6 @@ envoy_cc_library(
         "//envoy/http:header_map_interface",
         "//envoy/stream_info:stream_info_interface",
         "//source/common/protobuf",
+        "@envoy_api//envoy/data/accesslog/v3:pkg_cc_proto",
     ],
 )

envoy/access_log/access_log.h

@@ -4,6 +4,7 @@
 #include <string>
 
 #include "envoy/common/pure.h"
+#include "envoy/data/accesslog/v3/accesslog.pb.h"
 #include "envoy/filesystem/filesystem.h"
 #include "envoy/http/header_map.h"
 #include "envoy/stream_info/stream_info.h"
@@ -54,6 +55,7 @@ class AccessLogManager {
 };
 
 using AccessLogManagerPtr = std::unique_ptr<AccessLogManager>;
+using AccessLogType = envoy::data::accesslog::v3::AccessLogType;
 
 /**
  * Interface for access log filters.
@@ -88,11 +90,14 @@ class Instance {
    * @param response_trailers supplies response trailers.
    * @param stream_info supplies additional information about the request not
    * contained in the request headers.
+   * @param access_log_type supplies additional information about the type of the
+   * log record, i.e the location in the code which recorded the log.
    */
   virtual void log(const Http::RequestHeaderMap* request_headers,
                    const Http::ResponseHeaderMap* response_headers,
                    const Http::ResponseTrailerMap* response_trailers,
-                   const StreamInfo::StreamInfo& stream_info) PURE;
+                   const StreamInfo::StreamInfo& stream_info,
+                   AccessLogType access_log_type = AccessLogType::NotSet) PURE;
 };
 
 using InstanceSharedPtr = std::shared_ptr<Instance>;

envoy/formatter/BUILD

@@ -12,6 +12,7 @@ envoy_cc_library(
     name = "substitution_formatter_interface",
     hdrs = ["substitution_formatter.h"],
     deps = [
+        "//envoy/access_log:access_log_interface",
         "//envoy/config:typed_config_interface",
         "//envoy/http:header_map_interface",
         "//envoy/stream_info:stream_info_interface",

envoy/formatter/substitution_formatter.h

@@ -3,6 +3,7 @@
 #include <memory>
 #include <string>
 
+#include "envoy/access_log/access_log.h"
 #include "envoy/common/pure.h"
 #include "envoy/config/typed_config.h"
 #include "envoy/http/header_map.h"
@@ -28,11 +29,12 @@ class Formatter {
    * @param local_reply_body supplies the local reply body.
    * @return std::string string containing the complete formatted substitution line.
    */
-  virtual std::string format(const Http::RequestHeaderMap& request_headers,
-                             const Http::ResponseHeaderMap& response_headers,
-                             const Http::ResponseTrailerMap& response_trailers,
-                             const StreamInfo::StreamInfo& stream_info,
-                             absl::string_view local_reply_body) const PURE;
+  virtual std::string
+  format(const Http::RequestHeaderMap& request_headers,
+         const Http::ResponseHeaderMap& response_headers,
+         const Http::ResponseTrailerMap& response_trailers,
+         const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body,
+         AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const PURE;
 };
 
 using FormatterPtr = std::unique_ptr<Formatter>;
@@ -56,11 +58,12 @@ class FormatterProvider {
    * @return absl::optional<std::string> optional string containing a single value extracted from
    * the given headers/trailers/stream.
    */
-  virtual absl::optional<std::string> format(const Http::RequestHeaderMap& request_headers,
-                                             const Http::ResponseHeaderMap& response_headers,
-                                             const Http::ResponseTrailerMap& response_trailers,
-                                             const StreamInfo::StreamInfo& stream_info,
-                                             absl::string_view local_reply_body) const PURE;
+  virtual absl::optional<std::string>
+  format(const Http::RequestHeaderMap& request_headers,
+         const Http::ResponseHeaderMap& response_headers,
+         const Http::ResponseTrailerMap& response_trailers,
+         const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body,
+         AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const PURE;
   /**
    * Extract a value from the provided headers/trailers/stream, preserving the value's type.
    * @param request_headers supplies the request headers.
@@ -71,11 +74,12 @@ class FormatterProvider {
    * @return ProtobufWkt::Value containing a single value extracted from the given
    *         headers/trailers/stream.
    */
-  virtual ProtobufWkt::Value formatValue(const Http::RequestHeaderMap& request_headers,
-                                         const Http::ResponseHeaderMap& response_headers,
-                                         const Http::ResponseTrailerMap& response_trailers,
-                                         const StreamInfo::StreamInfo& stream_info,
-                                         absl::string_view local_reply_body) const PURE;
+  virtual ProtobufWkt::Value formatValue(
+      const Http::RequestHeaderMap& request_headers,
+      const Http::ResponseHeaderMap& response_headers,
+      const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info,
+      absl::string_view local_reply_body,
+      AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const PURE;
 };
 
 using FormatterProviderPtr = std::unique_ptr<FormatterProvider>;