[APPSEC-9143] Extract expires information from target map payload.

Steepfile

@@ -637,7 +637,7 @@ target :ddtrace do
   ignore 'lib/ddtrace/transport/traces.rb'
   ignore 'lib/ddtrace/version.rb'
 
-  library 'pathname', 'set'
+  library 'pathname', 'set', 'date'
   library 'cgi'
   library 'logger', 'monitor'
   library 'tsort'

lib/datadog/core/remote/client.rb

@@ -46,7 +46,7 @@ def sync
 
             targets = Configuration::TargetMap.parse(response.targets)
 
-            contents = Configuration::ContentList.parse(response.target_files)
+            contents = Configuration::ContentList.parse(response.target_files, targets.expires)
 
             # TODO: sometimes it can strangely be so that paths.empty?
             # TODO: sometimes it can strangely be so that targets.empty?

lib/datadog/core/remote/configuration/content.rb

@@ -10,20 +10,21 @@ class Configuration
         # Content stores the information associated with a specific Configuration::Path
         class Content
           class << self
-            def parse(hash)
+            def parse(hash, expires)
               path = Path.parse(hash[:path])
               data = hash[:content]
 
-              new(path: path, data: data)
+              new(path: path, data: data, expires: expires)
             end
           end
 
-          attr_reader :path, :data, :hashes
+          attr_reader :path, :data, :hashes, :expires
           attr_accessor :version
 
-          def initialize(path:, data:)
+          def initialize(path:, data:, expires:)
             @path = path
             @data = data
+            @expires = expires
             @hashes = {}
             @version = 0
           end
@@ -49,8 +50,8 @@ def compute_and_store_hash(type)
         # It provides convinient methods for finding content base on Configuration::Path and Configuration::Target
         class ContentList < Array
           class << self
-            def parse(array)
-              new.concat(array.map { |c| Content.parse(c) })
+            def parse(array, expires)
+              new.concat(array.map { |c| Content.parse(c, expires) })
             end
           end
 

lib/datadog/core/remote/configuration/repository.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'date'
 require_relative 'content'
 
 module Datadog
@@ -90,8 +91,12 @@ def initialize(repository)
             def contents_to_config_states(contents)
               return [] if contents.empty?
 
-              contents.map do |content|
-                {
+              contents.each_with_object([]) do |content, acc|
+                # content expires datetime has zero offset
+                # new_offset return the current time with zero offset
+                next if content.expires && (content.expires < DateTime.now.new_offset)
+
+                acc << {
                   id: content.path.config_id,
                   version: content.version,
                   product: content.path.product

lib/datadog/core/remote/configuration/target.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'date'
 require_relative 'path'
 require_relative 'digest'
 
@@ -9,34 +10,43 @@ module Remote
       class Configuration
         # TargetMap stores information regarding Configuration::Path and Configuration::Target
         class TargetMap < Hash
+          class ParseError < StandardError; end
+
           class << self
             def parse(hash)
-              opaque_backend_state = hash['signed']['custom']['opaque_backend_state']
-              version = hash['signed']['version']
+              signed = hash['signed']
+
+              opaque_backend_state = signed['custom']['opaque_backend_state']
+              version = signed['version']
+              expires = signed['expires']
 
               map = new
 
               map.instance_eval do
                 @opaque_backend_state = opaque_backend_state
                 @version = version
+                @expires = DateTime.iso8601(expires) if expires
               end
 
-              hash['signed']['targets'].each_with_object(map) do |(p, t), m|
+              signed['targets'].each_with_object(map) do |(p, t), m|
                 path = Configuration::Path.parse(p)
                 target = Configuration::Target.parse(t)
 
                 m[path] = target
               end
+            rescue StandardError => e
+              raise ParseError, "fail to parse target map. #{e.class}, #{e.message}"
             end
           end
 
-          attr_reader :opaque_backend_state, :version
+          attr_reader :opaque_backend_state, :version, :expires
 
           def initialize
             super
 
             @opaque_backend_state = nil
             @version = nil
+            @expires = nil
           end
 
           private_class_method :new

sig/datadog/core/remote/configuration/content.rbs

@@ -3,19 +3,21 @@ module Datadog
     module Remote
       class Configuration
         class Content
-          def self.parse: (::Hash[Symbol, untyped] hash) -> Content
+          def self.parse: (::Hash[Symbol, untyped] hash, DateTime? expires) -> Content
 
           attr_reader path: Configuration::Path
 
           attr_reader data: StringIO
 
           attr_reader hashes: Hash[Symbol, String]
 
+          attr_reader expires: DateTime?
+
           attr_accessor version: Integer
 
           @length: Integer
 
-          def initialize: (path: Configuration::Path, data: StringIO) -> void
+          def initialize: (path: Configuration::Path, data: StringIO, expires: DateTime?) -> void
 
           def hexdigest: (Symbol type) -> String
 
@@ -27,7 +29,7 @@ module Datadog
         end
 
         class ContentList < Array[Content]
-          def self.parse: (::Array[::Hash[Symbol, untyped]] array) -> ContentList
+          def self.parse: (::Array[::Hash[Symbol, untyped]] array, DateTime? expires) -> ContentList
 
           def find_content: (Configuration::Path path, Configuration::Target target) -> Content?
 

sig/datadog/core/remote/configuration/target.rbs

@@ -3,12 +3,17 @@ module Datadog
     module Remote
       class Configuration
         class TargetMap < Hash[Configuration::Path, Configuration::Target]
+          class ParseError < StandardError
+          end
+
           def self.parse: (::Hash[::String, untyped] hash) -> TargetMap
 
           attr_reader opaque_backend_state: ::String?
 
           attr_reader version: ::Integer?
 
+          attr_reader expires: DateTime?
+
           def initialize: () -> void
         end
 

spec/datadog/appsec/remote_spec.rb

@@ -59,6 +59,7 @@
     end
 
     context 'remote configuration enabled' do
+      let(:expires) { DateTime.now.new_offset.to_date.next_year }
       before do
         expect(Datadog::AppSec).to receive(:default_setting?).with(:ruleset).and_return(true)
       end
@@ -122,7 +123,8 @@
             {
               path: 'datadog/603646/ASM_DD/latest/config',
               content: StringIO.new(rules_data)
-            }
+            },
+            expires
           )
         end
         let(:transaction) do

spec/datadog/core/remote/client_spec.rb

@@ -449,8 +449,8 @@
             }
           end
 
-          it 'raises Path::ParseError' do
-            expect { client.sync }.to raise_error(Datadog::Core::Remote::Configuration::Path::ParseError)
+          it 'raises TargetMap::ParseError' do
+            expect { client.sync }.to raise_error(Datadog::Core::Remote::Configuration::TargetMap::ParseError)
           end
         end
       end

spec/datadog/core/remote/configuration/content_spec.rb

@@ -89,13 +89,15 @@
       rules_override: []
     }
   end
+  let(:expires) { DateTime.now.new_offset.to_date.next_year }
   let(:string_io_content) { StringIO.new(raw.to_json) }
   subject(:content_list) do
     described_class.parse(
       [{
         :path => path.to_s,
         :content => string_io_content
-      }]
+      }],
+      expires
     )
   end
 
@@ -113,7 +115,8 @@
             [{
               :path => 'invalid path',
               :content => string_io_content
-            }]
+            }],
+            expires
           )
         end.to raise_error(Datadog::Core::Remote::Configuration::Path::ParseError)
       end
@@ -178,7 +181,8 @@
         {
           :path => path.to_s,
           :content => updated_string_io
-        }
+        },
+        expires
       )
     end
 
@@ -230,7 +234,8 @@
         {
           :path => path.to_s,
           :content => string_io_content
-        }
+        },
+        expires
       )
     end
 

spec/datadog/core/remote/configuration/digest_spec.rb

@@ -6,12 +6,14 @@
 
 RSpec.describe Datadog::Core::Remote::Configuration::Digest do
   let(:data) { StringIO.new('Hello World') }
+  let(:expires) { DateTime.now.new_offset.to_date.next_year }
   let(:content) do
     Datadog::Core::Remote::Configuration::Content.parse(
       {
         :path => 'datadog/603646/ASM/exclusion_filters/config',
         :content => data
-      }
+      },
+      expires
     )
   end
 

spec/datadog/core/remote/configuration/respository_spec.rb

@@ -15,6 +15,22 @@
       'length' => 645
     }
   end
+  let(:expires) { DateTime.now.new_offset.to_date.next_year }
+  let(:new_content_string_io_content) { StringIO.new('hello world') }
+  let(:new_content) do
+    Datadog::Core::Remote::Configuration::Content.parse(
+      {
+        :path => path.to_s,
+        :content => new_content_string_io_content,
+      },
+      expires
+    )
+  end
+  let(:new_target) do
+    updated_raw_target = raw_target.dup
+    updated_raw_target['custom']['v'] += 1
+    Datadog::Core::Remote::Configuration::Target.parse(updated_raw_target)
+  end
 
   let(:target) { Datadog::Core::Remote::Configuration::Target.parse(raw_target) }
   let(:path) { Datadog::Core::Remote::Configuration::Path.parse('datadog/603646/ASM/exclusion_filters/config') }
@@ -90,28 +106,16 @@
     }
   end
   let(:string_io_content) { StringIO.new(raw.to_json) }
-
   let(:content) do
-    Datadog::Core::Remote::Configuration::Content.parse({ :path => path.to_s, :content => string_io_content })
-  end
-
-  let(:new_content_string_io_content) { StringIO.new('hello world') }
-
-  let(:new_content) do
     Datadog::Core::Remote::Configuration::Content.parse(
       {
         :path => path.to_s,
-        :content => new_content_string_io_content
-      }
+        :content => string_io_content,
+      },
+      expires,
     )
   end
 
-  let(:new_target) do
-    updated_raw_target = raw_target.dup
-    updated_raw_target['custom']['v'] += 1
-    Datadog::Core::Remote::Configuration::Target.parse(updated_raw_target)
-  end
-
   describe '#transaction' do
     it 'yields self and a new Repository::Transaction instance' do
       expect do |b|
@@ -185,8 +189,11 @@
         expect(repository.contents[path]).to eq(content)
 
         new_content = Datadog::Core::Remote::Configuration::Content.parse(
-          { :path => path.to_s,
-            :content => StringIO.new('hello world') }
+          {
+            :path => path.to_s,
+            :content => StringIO.new('hello world')
+          },
+          expires
         )
 
         updated_changes = repository.transaction do |_repository, transaction|
@@ -208,8 +215,11 @@
         expect(repository.contents[path]).to eq(content)
         new_path = Datadog::Core::Remote::Configuration::Path.parse('employee/ASM/exclusion_filters/config')
         new_content = Datadog::Core::Remote::Configuration::Content.parse(
-          { :path => new_path.to_s,
-            :content => StringIO.new('hello world') }
+          {
+            :path => new_path.to_s,
+            :content => StringIO.new('hello world')
+          },
+          expires
         )
 
         changes = repository.transaction do |_repository, transaction|
@@ -420,6 +430,18 @@
             expect(repository.state.config_states).to eq([])
           end
         end
+
+        context 'expired content' do
+          let(:expires) { DateTime.new(2015, 1, 1) }
+
+          it 'do not include expired content' do
+            repository.transaction do |_repository, transaction|
+              transaction.insert(path, target, content)
+            end
+
+            expect(repository.state.config_states).to eq([])
+          end
+        end
       end
     end
   end