From 8097aa30d0cfc2c64a2d21564337ad450f722581 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Fri, 20 Sep 2024 12:48:00 +0300 Subject: [PATCH 01/10] release pool memory on spawn --- pipeline/processor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pipeline/processor.go b/pipeline/processor.go index de4cd0eaa..4d243e672 100644 --- a/pipeline/processor.go +++ b/pipeline/processor.go @@ -414,6 +414,7 @@ func (p *processor) Spawn(parent *Event, nodes []*insaneJSON.Node) { child.stage = eventStageOutput p.output.Out(child) } + child.Root.ReleasePoolMem() } if p.busyActionsTotal == 0 { From 27114a5ae350abd38aff3498cb438e57e6d0c925 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Thu, 26 Sep 2024 19:05:05 +0700 Subject: [PATCH 02/10] Revert "release pool memory on spawn" This reverts commit 977e63ead9063a40f1fcc686742869c3b8be18d0. --- pipeline/processor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/processor.go b/pipeline/processor.go index 4d243e672..de4cd0eaa 100644 --- a/pipeline/processor.go +++ b/pipeline/processor.go @@ -414,7 +414,6 @@ func (p *processor) Spawn(parent *Event, nodes []*insaneJSON.Node) { child.stage = eventStageOutput p.output.Out(child) } - child.Root.ReleasePoolMem() } if p.busyActionsTotal == 0 { From 6e1e85138531bf11965f0c1883333be3b14b37b0 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Thu, 26 Sep 2024 19:58:53 +0700 Subject: [PATCH 03/10] decoderPoolSize metric --- go.mod | 2 ++ go.sum | 4 ++-- pipeline/pipeline.go | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 0712d6598..49cec94fa 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,8 @@ require ( sigs.k8s.io/yaml v1.3.0 ) +replace github.com/vitkovskii/insane-json => github.com/DmitryRomanov/insane-json v0.1.6-0.20240926122634-ddd151e16a6e + require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect diff --git a/go.sum b/go.sum index bc79b75e2..ae3d74efc 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0= github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= +github.com/DmitryRomanov/insane-json v0.1.6-0.20240926122634-ddd151e16a6e h1:RSaL4V1x1McvmpbZDcGF1i03q9ZiwT0yJ6NYEl5ipxA= +github.com/DmitryRomanov/insane-json v0.1.6-0.20240926122634-ddd151e16a6e/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/KimMachineGun/automemlimit v0.2.6 h1:tQFriVTcIteUkV5EgU9iz03eDY36T8JU5RAjP2r6Kt0= github.com/KimMachineGun/automemlimit v0.2.6/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= @@ -389,8 +391,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= -github.com/vitkovskii/insane-json v0.1.7 h1:A84WpQ/xGw8SSzzbLTkTI6PD2IYAVWjTlgaxN/rCAnw= -github.com/vitkovskii/insane-json v0.1.7/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 62a675308..0536304dc 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -116,6 +116,7 @@ type Pipeline struct { // all pipeline`s metrics inUseEventsMetric prometheus.Gauge + decoderPoolSize prometheus.Gauge eventPoolCapacityMetric prometheus.Gauge inputEventsCountMetric prometheus.Counter inputEventSizeMetric prometheus.Counter @@ -225,6 +226,7 @@ func (p *Pipeline) IncMaxEventSizeExceeded() { func (p *Pipeline) registerMetrics() { m := p.actionParams.MetricCtl p.inUseEventsMetric = m.RegisterGauge("event_pool_in_use_events", "Count of pool events which is used for processing") + p.decoderPoolSize = m.RegisterGauge("decoder_pool_size", "Size of decoder pool") p.eventPoolCapacityMetric = m.RegisterGauge("event_pool_capacity", "Pool capacity value") p.inputEventsCountMetric = m.RegisterCounter("input_events_count", "Count of events on pipeline input") p.inputEventSizeMetric = m.RegisterCounter("input_events_size", "Size of events on pipeline input") @@ -838,6 +840,8 @@ func (p *Pipeline) maintenance() { myDeltas := p.incMetrics(inputEvents, inputSize, outputEvents, outputSize, readOps) p.setMetrics(p.eventPool.inUseEvents.Load()) p.logChanges(myDeltas) + + p.decoderPoolSize.Set(float64(insaneJSON.LenghtDecoderPool())) } } From 888f7e52fe290ede60ae9034dc51d2fae2c9abfe Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Mon, 30 Sep 2024 19:15:57 +0700 Subject: [PATCH 04/10] Revert "decoderPoolSize metric" This reverts commit 6e1e85138531bf11965f0c1883333be3b14b37b0. --- go.mod | 2 -- go.sum | 4 ++-- pipeline/pipeline.go | 4 ---- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 49cec94fa..0712d6598 100644 --- a/go.mod +++ b/go.mod @@ -50,8 +50,6 @@ require ( sigs.k8s.io/yaml v1.3.0 ) -replace github.com/vitkovskii/insane-json => github.com/DmitryRomanov/insane-json v0.1.6-0.20240926122634-ddd151e16a6e - require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect diff --git a/go.sum b/go.sum index ae3d74efc..bc79b75e2 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0= github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= -github.com/DmitryRomanov/insane-json v0.1.6-0.20240926122634-ddd151e16a6e h1:RSaL4V1x1McvmpbZDcGF1i03q9ZiwT0yJ6NYEl5ipxA= -github.com/DmitryRomanov/insane-json v0.1.6-0.20240926122634-ddd151e16a6e/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/KimMachineGun/automemlimit v0.2.6 h1:tQFriVTcIteUkV5EgU9iz03eDY36T8JU5RAjP2r6Kt0= github.com/KimMachineGun/automemlimit v0.2.6/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= @@ -391,6 +389,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/vitkovskii/insane-json v0.1.7 h1:A84WpQ/xGw8SSzzbLTkTI6PD2IYAVWjTlgaxN/rCAnw= +github.com/vitkovskii/insane-json v0.1.7/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 0536304dc..62a675308 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -116,7 +116,6 @@ type Pipeline struct { // all pipeline`s metrics inUseEventsMetric prometheus.Gauge - decoderPoolSize prometheus.Gauge eventPoolCapacityMetric prometheus.Gauge inputEventsCountMetric prometheus.Counter inputEventSizeMetric prometheus.Counter @@ -226,7 +225,6 @@ func (p *Pipeline) IncMaxEventSizeExceeded() { func (p *Pipeline) registerMetrics() { m := p.actionParams.MetricCtl p.inUseEventsMetric = m.RegisterGauge("event_pool_in_use_events", "Count of pool events which is used for processing") - p.decoderPoolSize = m.RegisterGauge("decoder_pool_size", "Size of decoder pool") p.eventPoolCapacityMetric = m.RegisterGauge("event_pool_capacity", "Pool capacity value") p.inputEventsCountMetric = m.RegisterCounter("input_events_count", "Count of events on pipeline input") p.inputEventSizeMetric = m.RegisterCounter("input_events_size", "Size of events on pipeline input") @@ -840,8 +838,6 @@ func (p *Pipeline) maintenance() { myDeltas := p.incMetrics(inputEvents, inputSize, outputEvents, outputSize, readOps) p.setMetrics(p.eventPool.inUseEvents.Load()) p.logChanges(myDeltas) - - p.decoderPoolSize.Set(float64(insaneJSON.LenghtDecoderPool())) } } From 98055e1da8d490a932cc5a7beda29c707a9ae243 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Mon, 30 Sep 2024 19:22:31 +0700 Subject: [PATCH 05/10] use sync pool for insane decoder --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 0712d6598..7739c36d1 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,8 @@ require ( sigs.k8s.io/yaml v1.3.0 ) +replace github.com/vitkovskii/insane-json => github.com/DmitryRomanov/insane-json v0.1.6-0.20240930121319-27c156affa37 + require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect diff --git a/go.sum b/go.sum index bc79b75e2..e52a9dcc9 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0= github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= +github.com/DmitryRomanov/insane-json v0.1.6-0.20240930121319-27c156affa37 h1:r4vfnTki2ovL1GpZj9l6tNkh22X9DPR67NzG+1ri6Hc= +github.com/DmitryRomanov/insane-json v0.1.6-0.20240930121319-27c156affa37/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/KimMachineGun/automemlimit v0.2.6 h1:tQFriVTcIteUkV5EgU9iz03eDY36T8JU5RAjP2r6Kt0= github.com/KimMachineGun/automemlimit v0.2.6/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= @@ -389,8 +391,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= -github.com/vitkovskii/insane-json v0.1.7 h1:A84WpQ/xGw8SSzzbLTkTI6PD2IYAVWjTlgaxN/rCAnw= -github.com/vitkovskii/insane-json v0.1.7/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= From 2b67fdcd5815e2170ddefa616e5416fbc2257606 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Tue, 1 Oct 2024 14:46:42 +0700 Subject: [PATCH 06/10] release child root on spawn --- pipeline/processor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pipeline/processor.go b/pipeline/processor.go index de4cd0eaa..d00b1bdf4 100644 --- a/pipeline/processor.go +++ b/pipeline/processor.go @@ -414,6 +414,7 @@ func (p *processor) Spawn(parent *Event, nodes []*insaneJSON.Node) { child.stage = eventStageOutput p.output.Out(child) } + insaneJSON.Release(child.Root) } if p.busyActionsTotal == 0 { From 25d892394fa2dffbe647d45fd9d3c11fb81c5b15 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Mon, 7 Oct 2024 12:02:54 +0300 Subject: [PATCH 07/10] release pool in finalize --- pipeline/pipeline.go | 6 ++++++ pipeline/processor.go | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 62a675308..fc024b42b 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -605,6 +605,12 @@ func (p *Pipeline) finalize(event *Event, notifyInput bool, backEvent bool) { for _, e := range event.children { insaneJSON.Release(e.Root) } + + if event.Root.PoolSize() > DefaultJSONNodePoolSize*4 && len(event.children) > 0 { + event.Root.ReleasePoolMem() + event.Root.ReleaseBufMem() + } + p.eventPool.back(event) } diff --git a/pipeline/processor.go b/pipeline/processor.go index d00b1bdf4..de4cd0eaa 100644 --- a/pipeline/processor.go +++ b/pipeline/processor.go @@ -414,7 +414,6 @@ func (p *processor) Spawn(parent *Event, nodes []*insaneJSON.Node) { child.stage = eventStageOutput p.output.Out(child) } - insaneJSON.Release(child.Root) } if p.busyActionsTotal == 0 { From 25353b0bc8b568c1d4467ed9106ff7ae99a444d0 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Mon, 7 Oct 2024 15:02:46 +0300 Subject: [PATCH 08/10] use sync pool for decoder node pool --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7739c36d1..eb1b7578e 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( sigs.k8s.io/yaml v1.3.0 ) -replace github.com/vitkovskii/insane-json => github.com/DmitryRomanov/insane-json v0.1.6-0.20240930121319-27c156affa37 +replace github.com/vitkovskii/insane-json => github.com/DmitryRomanov/insane-json v0.1.6-0.20241007115731-9f178d5b8948 require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect diff --git a/go.sum b/go.sum index e52a9dcc9..41f05dc8e 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0= github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= -github.com/DmitryRomanov/insane-json v0.1.6-0.20240930121319-27c156affa37 h1:r4vfnTki2ovL1GpZj9l6tNkh22X9DPR67NzG+1ri6Hc= -github.com/DmitryRomanov/insane-json v0.1.6-0.20240930121319-27c156affa37/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= +github.com/DmitryRomanov/insane-json v0.1.6-0.20241007115731-9f178d5b8948 h1:r4QOJe3pi/NDYrcIhUqSdzQ6iprVjbwZHh8+b5HWcIA= +github.com/DmitryRomanov/insane-json v0.1.6-0.20241007115731-9f178d5b8948/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/KimMachineGun/automemlimit v0.2.6 h1:tQFriVTcIteUkV5EgU9iz03eDY36T8JU5RAjP2r6Kt0= github.com/KimMachineGun/automemlimit v0.2.6/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= From f1e5718e242236bc06395cc9b0ae2647d0199d57 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Tue, 8 Oct 2024 09:53:38 +0300 Subject: [PATCH 09/10] resets nodes on back pool --- go.mod | 2 +- go.sum | 2 ++ pipeline/pipeline.go | 5 ----- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index eb1b7578e..454c0b75b 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( sigs.k8s.io/yaml v1.3.0 ) -replace github.com/vitkovskii/insane-json => github.com/DmitryRomanov/insane-json v0.1.6-0.20241007115731-9f178d5b8948 +replace github.com/vitkovskii/insane-json => github.com/DmitryRomanov/insane-json v0.1.6-0.20241008062604-33cf01b0c441 require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect diff --git a/go.sum b/go.sum index 41f05dc8e..ad3934756 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UT github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= github.com/DmitryRomanov/insane-json v0.1.6-0.20241007115731-9f178d5b8948 h1:r4QOJe3pi/NDYrcIhUqSdzQ6iprVjbwZHh8+b5HWcIA= github.com/DmitryRomanov/insane-json v0.1.6-0.20241007115731-9f178d5b8948/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= +github.com/DmitryRomanov/insane-json v0.1.6-0.20241008062604-33cf01b0c441 h1:KgGmx/V+EdmO5CchVk+6XxQh/Ghw+8wU7lpvxrhxdAo= +github.com/DmitryRomanov/insane-json v0.1.6-0.20241008062604-33cf01b0c441/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/KimMachineGun/automemlimit v0.2.6 h1:tQFriVTcIteUkV5EgU9iz03eDY36T8JU5RAjP2r6Kt0= github.com/KimMachineGun/automemlimit v0.2.6/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index fc024b42b..f713646d3 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -606,11 +606,6 @@ func (p *Pipeline) finalize(event *Event, notifyInput bool, backEvent bool) { insaneJSON.Release(e.Root) } - if event.Root.PoolSize() > DefaultJSONNodePoolSize*4 && len(event.children) > 0 { - event.Root.ReleasePoolMem() - event.Root.ReleaseBufMem() - } - p.eventPool.back(event) } From 8bb734a100216ed572ac0ec50b6f53d152bc8e34 Mon Sep 17 00:00:00 2001 From: Dmitry Romanov Date: Tue, 8 Oct 2024 11:08:37 +0300 Subject: [PATCH 10/10] reset only non-empty node pools --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 454c0b75b..f27123134 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( sigs.k8s.io/yaml v1.3.0 ) -replace github.com/vitkovskii/insane-json => github.com/DmitryRomanov/insane-json v0.1.6-0.20241008062604-33cf01b0c441 +replace github.com/vitkovskii/insane-json => github.com/DmitryRomanov/insane-json v0.1.6-0.20241008094754-751b7776020a require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect diff --git a/go.sum b/go.sum index ad3934756..bb8f365f8 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0= github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= -github.com/DmitryRomanov/insane-json v0.1.6-0.20241007115731-9f178d5b8948 h1:r4QOJe3pi/NDYrcIhUqSdzQ6iprVjbwZHh8+b5HWcIA= -github.com/DmitryRomanov/insane-json v0.1.6-0.20241007115731-9f178d5b8948/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= -github.com/DmitryRomanov/insane-json v0.1.6-0.20241008062604-33cf01b0c441 h1:KgGmx/V+EdmO5CchVk+6XxQh/Ghw+8wU7lpvxrhxdAo= -github.com/DmitryRomanov/insane-json v0.1.6-0.20241008062604-33cf01b0c441/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= +github.com/DmitryRomanov/insane-json v0.1.6-0.20241008094754-751b7776020a h1:aIy+Ad9xJKlZs0Q6l7DPQOi9bqoDzB+p+QlmsJdSnMY= +github.com/DmitryRomanov/insane-json v0.1.6-0.20241008094754-751b7776020a/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/KimMachineGun/automemlimit v0.2.6 h1:tQFriVTcIteUkV5EgU9iz03eDY36T8JU5RAjP2r6Kt0= github.com/KimMachineGun/automemlimit v0.2.6/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=