Production - out of memory error during warmup compilation

[vistra]

As seen in this log entry:

{
"_index": "logzioCustomerIndex191115_v7",
"_type": "doc",
"_id": "AW5saKV_YDXMxNyQOa_7.account-65212",
"_version": 1,
"_score": null,
"_source": {
"adapter": "processor-native",
"level": "error",
"source": "/src/services/processor/native/adapter/native_compiler.go:88",
"message": "warm up compilation on init failed",
"error": "could not build a shared object: error building go source: exit status 1, go build output: go tool compile: exit status 2; fatal error: runtime: out of memory; ; runtime stack:; runtime.throw(0xe60e56, 0x16); \t/usr/local/go/src/runtime/panic.go:617 +0x72 fp=0x7ffeb1857350 sp=0x7ffeb1857320 pc=0x42cbe2; runtime.sysMap(0xc000000000, 0x4000000, 0x16cd178); \t/usr/local/go/src/runtime/mem_linux.go:170 +0xc7 fp=0x7ffeb1857390 sp=0x7ffeb1857350 pc=0x417467; runtime.(*mheap).sysAlloc(0x16a57c0, 0x2000, 0x16a57d0, 0x1); \t/usr/local/go/src/runtime/malloc.go:633 +0x1cd fp=0x7ffeb1857438 sp=0x7ffeb1857390 pc=0x40ab8d; runtime.(*mheap).grow(0x16a57c0, 0x1, 0x0); \t/usr/local/go/src/runtime/mheap.go:1222 +0x42 fp=0x7ffeb1857490 sp=0x7ffeb1857438 pc=0x424782; runtime.(*mheap).allocSpanLocked(0x16a57c0, 0x1, 0x16cd188, 0x0); \t/usr/local/go/src/runtime/mheap.go:1150 +0x37f fp=0x7ffeb18574c8 sp=0x7ffeb1857490 pc=0x42466f; runtime.(*mheap).alloc_m(0x16a57c0, 0x1, 0x2a, 0x0); \t/usr/local/go/src/runtime/mheap.go:977 +0xc2 fp=0x7ffeb1857518 sp=0x7ffeb18574c8 pc=0x423cc2; runtime.(*mheap).alloc.func1(); \t/usr/local/go/src/runtime/mheap.go:1048 +0x4c fp=0x7ffeb1857550 sp=0x7ffeb1857518 pc=0x456fbc; runtime.(*mheap).alloc(0x16a57c0, 0x1, 0x1002a, 0x0); \t/usr/local/go/src/runtime/mheap.go:1047 +0x8a fp=0x7ffeb18575a0 sp=0x7ffeb1857550 pc=0x423f9a; runtime.(*mcentral).grow(0x16a65c0, 0x0); \t/usr/local/go/src/runtime/mcentral.go:256 +0x95 fp=0x7ffeb18575e8 sp=0x7ffeb18575a0 pc=0x416ee5; runtime.(*mcentral).cacheSpan(0x16a65c0, 0x7efdd4475000); \t/usr/local/go/src/runtime/mcentral.go:106 +0x2ff fp=0x7ffeb1857648 sp=0x7ffeb18575e8 pc=0x4169ef; runtime.(*mcache).refill(0x7efdd4475008, 0x2a); \t/usr/local/go/src/runtime/mcache.go:135 +0x86 fp=0x7ffeb1857668 sp=0x7ffeb1857648 pc=0x416486; runtime.(*mcache).nextFree(0x7efdd4475008, 0x169c22a, 0x7efdd4475008, 0x7efdd4475000, 0x8); \t/usr/local/go/src/runtime/malloc.go:786 +0x88 fp=0x7ffeb18576a0 sp=0x7ffeb1857668 pc=0x40b3c8; runtime.mallocgc(0x180, 0xe4a580, 0x1, 0x16cd1e0); \t/usr/local/go/src/runtime/malloc.go:939 +0x76e fp=0x7ffeb1857740 sp=0x7ffeb18576a0 pc=0x40bcde; runtime.newobject(0xe4a580, 0x4000); \t/usr/local/go/src/runtime/malloc.go:1068 +0x38 fp=0x7ffeb1857770 sp=0x7ffeb1857740 pc=0x40c0e8; runtime.malg(0x1de99000008000, 0x16a7e30); \t/usr/local/go/src/runtime/proc.go:3220 +0x31 fp=0x7ffeb18577b0 sp=0x7ffeb1857770 pc=0x435f51; runtime.mpreinit(...); \t/usr/local/go/src/runtime/os_linux.go:311; runtime.mcommoninit(0x169d9a0); \t/usr/local/go/src/runtime/proc.go:618 +0xc2 fp=0x7ffeb18577e8 sp=0x7ffeb18577b0 pc=0x42f982; runtime.schedinit(); \t/usr/local/go/src/runtime/proc.go:540 +0x74 fp=0x7ffeb1857840 sp=0x7ffeb18577e8 pc=0x42f614; runtime.rt0_go(0x7ffeb1857878, 0x2, 0x7ffeb1857878, 0x0, 0x0, 0x2, 0x7ffeb1857e0b, 0x7ffeb1857e36, 0x0, 0x7ffeb1857e3e, ...); \t/usr/local/go/src/runtime/asm_amd64.s:195 +0x11a fp=0x7ffeb1857848 sp=0x7ffeb1857840 pc=0x458e2a; go tool compile: exit status 2; fatal error: runtime: out of memory; ; runtime stack:; runtime.throw(0xe60e56, 0x16); \t/usr/local/go/src/runtime/panic.go:617 +0x72 fp=0x7fff7f65ec90 sp=0x7fff7f65ec60 pc=0x42cbe2; runtime.sysMap(0xc000000000, 0x4000000, 0x16cd178); \t/usr/local/go/src/runtime/mem_linux.go:170 +0xc7 fp=0x7fff7f65ecd0 sp=0x7fff7f65ec90 pc=0x417467; runtime.(*mheap).sysAlloc(0x16a57c0, 0x2000, 0x16a57d0, 0x1); \t/usr/local/go/src/runtime/malloc.go:633 +0x1cd fp=0x7fff7f65ed78 sp=0x7fff7f65ecd0 pc=0x40ab8d; runtime.(*mheap).grow(0x16a57c0, 0x1, 0x0); \t/usr/local/go/src/runtime/mheap.go:1222 +0x42 fp=0x7fff7f65edd0 sp=0x7fff7f65ed78 pc=0x424782; runtime.(*mheap).allocSpanLocked(0x16a57c0, 0x1, 0x16cd188, 0x0); \t/usr/local/go/src/runtime/mheap.go:1150 +0x37f fp=0x7fff7f65ee08 sp=0x7fff7f65edd0 pc=0x42466f; runtime.(*mheap).alloc_m(0x16a57c0, 0x1, 0x2a, 0x0); \t/usr/local/go/src/runtime/mheap.go:977 +0xc2 fp=0x7fff7f65ee58 sp=0x7fff7f65ee08 pc=0x423cc2; runtime.(*mheap).alloc.func1(); \t/usr/local/go/src/runtime/mheap.go:1048 +0x4c fp=0x7fff7f65ee90 sp=0x7fff7f65ee58 pc=0x456fbc; runtime.(*mheap).alloc(0x16a57c0, 0x1, 0x1002a, 0x0); \t/usr/local/go/src/runtime/mheap.go:1047 +0x8a fp=0x7fff7f65eee0 sp=0x7fff7f65ee90 pc=0x423f9a; runtime.(*mcentral).grow(0x16a65c0, 0x0); \t/usr/local/go/src/runtime/mcentral.go:256 +0x95 fp=0x7fff7f65ef28 sp=0x7fff7f65eee0 pc=0x416ee5; runtime.(*mcentral).cacheSpan(0x16a65c0, 0x7f0675f81000); \t/usr/local/go/src/runtime/mcentral.go:106 +0x2ff fp=0x7fff7f65ef88 sp=0x7fff7f65ef28 pc=0x4169ef; runtime.(*mcache).refill(0x7f0675f81008, 0x2a); \t/usr/local/go/src/runtime/mcache.go:135 +0x86 fp=0x7fff7f65efa8 sp=0x7fff7f65ef88 pc=0x416486; runtime.(*mcache).nextFree(0x7f0675f81008, 0x169c22a, 0x7f0675f81008, 0x7f0675f81000, 0x8); \t/usr/local/go/src/runtime/malloc.go:786 +0x88 fp=0x7fff7f65efe0 sp=0x7fff7f65efa8 pc=0x40b3c8; runtime.mallocgc(0x180, 0xe4a580, 0x1, 0x16cd1e0); \t/usr/local/go/src/runtime/malloc.go:939 +0x76e fp=0x7fff7f65f080 sp=0x7fff7f65efe0 pc=0x40bcde; runtime.newobject(0xe4a580, 0x4000); \t/usr/local/go/src/runtime/malloc.go:1068 +0x38 fp=0x7fff7f65f0b0 sp=0x7fff7f65f080 pc=0x40c0e8; runtime.malg(0x1de99000008000, 0x16a7e30); \t/usr/local/go/src/runtime/proc.go:3220 +0x31 fp=0x7fff7f65f0f0 sp=0x7fff7f65f0b0 pc=0x435f51; runtime.mpreinit(...); \t/usr/local/go/src/runtime/os_linux.go:311; runtime.mcommoninit(0x169d9a0); \t/usr/local/go/src/runtime/proc.go:618 +0xc2 fp=0x7fff7f65f128 sp=0x7fff7f65f0f0 pc=0x42f982; runtime.schedinit(); \t/usr/local/go/src/runtime/proc.go:540 +0x74 fp=0x7fff7f65f180 sp=0x7fff7f65f128 pc=0x42f614; runtime.rt0_go(0x7fff7f65f1b8, 0x2, 0x7fff7f65f1b8, 0x0, 0x0, 0x2, 0x7fff7f65fe0b, 0x7fff7f65fe36, 0x0, 0x7fff7f65fe3e, ...); \t/usr/local/go/src/runtime/asm_amd64.s:195 +0x11a fp=0x7fff7f65f188 sp=0x7fff7f65f180 pc=0x458e2a; go tool compile: exit status 2; fatal error: runtime: out of memory; ; runtime stack:; runtime.throw(0xe60e56, 0x16); \t/usr/local/go/src/runtime/panic.go:617 +0x72 fp=0x7ffc599fc6b0 sp=0x7ffc599fc680 pc=0x42cbe2; runtime.sysMap(0xc000000000, 0x4000000, 0x16cd178); \t/usr/local/go/src/runtime/mem_linux.go:170 +0xc7 fp=0x7ffc599fc6f0 sp=0x7ffc599fc6b0 pc=0x417467; runtime.(*mheap).sysAlloc(0x16a57c0, 0x2000, 0x16a57d0, 0x1); \t/usr/local/go/src/runtime/malloc.go:633 +0x1cd fp=0x7ffc599fc798 sp=0x7ffc599fc6f0 pc=0x40ab8d; runtime.(*mheap).grow(0x16a57c0, 0x1, 0x0); \t/usr/local/go/src/runtime/mheap.go:1222 +0x42 fp=0x7ffc599fc7f0 sp=0x7ffc599fc798 pc=0x424782; runtime.(*mheap).allocSpanLocked(0x16a57c0, 0x1, 0x16cd188, 0x0); \t/usr/local/go/src/runtime/mheap.go:1150 +0x37f fp=0x7ffc599fc828 sp=0x7ffc599fc7f0 pc=0x42466f; runtime.(*mheap).alloc_m(0x16a57c0, 0x1, 0x2a, 0x0); \t/usr/local/go/src/runtime/mheap.go:977 +0xc2 fp=0x7ffc599fc878 sp=0x7ffc599fc828 pc=0x423cc2; runtime.(*mheap).alloc.func1(); \t/usr/local/go/src/runtime/mheap.go:1048 +0x4c fp=0x7ffc599fc8b0 sp=0x7ffc599fc878 pc=0x456fbc; runtime.(*mheap).alloc(0x16a57c0, 0x1, 0x1002a, 0x0); \t/usr/local/go/src/runtime/mheap.go:1047 +0x8a fp=0x7ffc599fc900 sp=0x7ffc599fc8b0 pc=0x423f9a; runtime.(*mcentral).grow(0x16a65c0, 0x0); \t/usr/local/go/src/runtime/mcentral.go:256 +0x95 fp=0x7ffc599fc948 sp=0x7ffc599fc900 pc=0x416ee5; runtime.(*mcentral).cacheSpan(0x16a65c0, 0x7f5a1bd0f000); \t/usr/local/go/src/runtime/mcentral.go:106 +0x2ff fp=0x7ffc599fc9a8 sp=0x7ffc599fc948 pc=0x4169ef; runtime.(*mcache).refill(0x7f5a1bd0f008, 0x2a); \t/usr/local/go/src/runtime/mcache.go:135 +0x86 fp=0x7ffc599fc9c8 sp=0x7ffc599fc9a8 pc=0x416486; runtime.(*mcache).nextFree(0x7f5a1bd0f008, 0x169c22a, 0x7f5a1bd0f008, 0x7f5a1bd0f000, 0x8); \t/usr/local/go/src/runtime/malloc.go:786 +0x88 fp=0x7ffc599fca00 sp=0x7ffc599fc9c8 pc=0x40b3c8; runtime.mallocgc(0x180, 0xe4a580, 0x1, 0x16cd1e0); \t/usr/local/go/src/runtime/malloc.go:939 +0x76e fp=0x7ffc599fcaa0 sp=0x7ffc599fca00 pc=0x40bcde; runtime.newobject(0xe4a580, 0x4000); \t/usr/local/go/src/runtime/malloc.go:1068 +0x38 fp=0x7ffc599fcad0 sp=0x7ffc599fcaa0 pc=0x40c0e8; runtime.malg(0x1de99000008000, 0x16a7e30); \t/usr/local/go/src/runtime/proc.go:3220 +0x31 fp=0x7ffc599fcb10 sp=0x7ffc599fcad0 pc=0x435f51; runtime.mpreinit(...); \t/usr/local/go/src/runtime/os_linux.go:311; runtime.mcommoninit(0x169d9a0); \t/usr/local/go/src/runtime/proc.go:618 +0xc2 fp=0x7ffc599fcb48 sp=0x7ffc599fcb10 pc=0x42f982; runtime.schedinit(); \t/usr/local/go/src/runtime/proc.go:540 +0x74 fp=0x7ffc599fcba0 sp=0x7ffc599fcb48 pc=0x42f614; runtime.rt0_go(0x7ffc599fcbd8, 0x2, 0x7ffc599fcbd8, 0x0, 0x0, 0x2, 0x7ffc599fde0b, 0x7ffc599fde36, 0x0, 0x7ffc599fde3e, ...); \t/usr/local/go/src/runtime/asm_amd64.s:195 +0x11a fp=0x7ffc599fcba8 sp=0x7ffc599fcba0 pc=0x458e2a; go tool compile: exit status 2; fatal error: runtime: out of memory; ; runtime stack:; runtime.throw(0xe60e56, 0x16); \t/usr/local/go/src/runtime/panic.go:617 +0x72 fp=0x7ffc1e26ffe0 sp=0x7ffc1e26ffb0 pc=0x42cbe2; runtime.sysMap(0xc000000000, 0x4000000, 0x16cd178); \t/usr/local/go/src/runtime/mem_linux.go:170 +0xc7 fp=0x7ffc1e270020 sp=0x7ffc1e26ffe0 pc=0x417467; runtime.(*mheap).sysAlloc(0x16a57c0, 0x2000, 0x16a57d0, 0x1); \t/usr/local/go/src/runtime/malloc.go:633 +0x1cd fp=0x7ffc1e2700c8 sp=0x7ffc1e270020 pc=0x40ab8d; runtime.(*mheap).grow(0x16a57c0, 0x1, 0x0); \t/usr/local/go/src/runtime/mheap.go:1222 +0x42 fp=0x7ffc1e270120 sp=0x7ffc1e2700c8 pc=0x424782; runtime.(*mheap).allocSpanLocked(0x16a57c0, 0x1, 0x16cd188, 0x0); \t/usr/local/go/src/runtime/mheap.go:1150 +0x37f fp=0x7ffc1e270158 sp=0x7ffc1e270120 pc=0x42466f; runtime.(*mheap).alloc_m(0x16a57c0, 0x1, 0x2a, 0x0); \t/usr/local/go/src/runtime/mheap.go:977 +0xc2 fp=0x7ffc1e2701a8 sp=0x7ffc1e270158 pc=0x423cc2; runtime.(*mheap).alloc.func1(); \t/usr/local/go/src/runtime/mheap.go:1048 +0x4c fp=0x7ffc1e2701e0 sp=0x7ffc1e2701a8 pc=0x456fbc; runtime.(*mheap).alloc(0x16a57c0, 0x1, 0x1002a, 0x0); \t/usr/local/go/src/runtime/mheap.go:1047 +0x8a fp=0x7ffc1e270230 sp=0x7ffc1e2701e0 pc=0x423f9a; runtime.(*mcentral).grow(0x16a65c0, 0x0); \t/usr/local/go/src/runtime/mcentral.go:256 +0x95 fp=0x7ffc1e270278 sp=0x7ffc1e270230 pc=0x416ee5; runtime.(*mcentral).cacheSpan(0x16a65c0, 0x7fc1963de000); \t/usr/local/go/src/runtime/mcentral.go:106 +0x2ff fp=0x7ffc1e2702d8 sp=0x7ffc1e270278 pc=0x4169ef; runtime.(*mcache).refill(0x7fc1963de008, 0x2a); \t/usr/local/go/src/runtime/mcache.go:135 +0x86 fp=0x7ffc1e2702f8 sp=0x7ffc1e2702d8 pc=0x416486; runtime.(*mcache).nextFree(0x7fc1963de008, 0x169c22a, 0x7fc1963de008, 0x7fc1963de000, 0x8); \t/usr/local/go/src/runtime/malloc.go:786 +0x88 fp=0x7ffc1e270330 sp=0x7ffc1e2702f8 pc=0x40b3c8; runtime.mallocgc(0x180, 0xe4a580, 0x1, 0x16cd1e0); \t/usr/local/go/src/runtime/malloc.go:939 +0x76e fp=0x7ffc1e2703d0 sp=0x7ffc1e270330 pc=0x40bcde; runtime.newobject(0xe4a580, 0x4000); \t/usr/local/go/src/runtime/malloc.go:1068 +0x38 fp=0x7ffc1e270400 sp=0x7ffc1e2703d0 pc=0x40c0e8; runtime.malg(0x1de99000008000, 0x16a7e30); \t/usr/local/go/src/runtime/proc.go:3220 +0x31 fp=0x7ffc1e270440 sp=0x7ffc1e270400 pc=0x435f51; runtime.mpreinit(...); \t/usr/local/go/src/runtime/os_linux.go:311; runtime.mcommoninit(0x169d9a0); \t/usr/local/go/src/runtime/proc.go:618 +0xc2 fp=0x7ffc1e270478 sp=0x7ffc1e270440 pc=0x42f982; runtime.schedinit(); \t/usr/local/go/src/runtime/proc.go:540 +0x74 fp=0x7ffc1e2704d0 sp=0x7ffc1e270478 pc=0x42f614; runtime.rt0_go(0x7ffc1e270508, 0x2, 0x7ffc1e270508, 0x0, 0x0, 0x2, 0x7ffc1e271e0b, 0x7ffc1e271e36, 0x0, 0x7ffc1e271e3e, ...); \t/usr/local/go/src/runtime/asm_amd64.s:195 +0x11a fp=0x7ffc1e2704d8 sp=0x7ffc1e2704d0 pc=0x458e2a; ",
"type": "prod",
"tags": [
"_logz_http_bulk_json_8070"
],
"node": "c281180f60a969e577409f7b26b4f8079335cbe9",
"@timestamp": "2019-11-15T00:10:23.45095555Z",
"function": "adapter.(*nativeCompiler).warmUpCompilationCache",
"vcid": 1000001
},
"fields": {
"@timestamp": [
1573776623450
]
},
"sort": [
1573776623450
]
}

[noambergIL]

seems to be very low frequency and only on 1000001 ... this could be an artefact and not a production
(in all 7 days we keep i found 1 of these).

consider checking once in a while and prossibly kill this issue.