Skip to content

Commit c9dfe09

Browse files
roi-codefreshroi-codefresh
authored
Saas-549 - logs masking (#43)
* added log masking support
1 parent 0722d00 commit c9dfe09

File tree

9 files changed

+640
-31
lines changed

9 files changed

+640
-31
lines changed

‎lib/ContainerLogger.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ class ContainerLogger extends EventEmitter {
111111
this.handledStreams++;
112112
stdout
113113
.pipe(this._logSizeLimitStream())
114+
.pipe(this.stepLogger.createMaskingStream())
114115
.pipe(this.stepLogger.stepNameTransformStream().once('end', this._handleFinished.bind(this)))
115116
.pipe(stepLoggerWritableStream, {end: false});
116117

@@ -123,6 +124,7 @@ class ContainerLogger extends EventEmitter {
123124
stderr
124125
.pipe(this._logSizeLimitStream())
125126
.pipe(this._errorTransformerStream())
127+
.pipe(this.stepLogger.createMaskingStream())
126128
.pipe(this.stepLogger.stepNameTransformStream().once('end', this._handleFinished.bind(this)))
127129
.pipe(stepLoggerWritableStream, {end: false});
128130

‎lib/addNewMask.js

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
const rp = require('request-promise');
2+
3+
4+
function updateMasks(secret) {
5+
const port = process.env.PORT || 8080;
6+
const host = process.env.HOST || 'localhost';
7+
8+
const opt = {
9+
uri: `http://${host}:${port}/secrets`,
10+
method: 'POST',
11+
json: true,
12+
body: secret,
13+
resolveWithFullResponse: true,
14+
};
15+
16+
rp(opt)
17+
.then((res) => {
18+
if (res.statusCode >= 400) {
19+
console.log(`could not create mask for secret: ${secret.key}, because server responded with: ${res.statusCode}\n\n${res.body}`);
20+
process.exit(1);
21+
}
22+
console.log(`successfully updated masks with secret: ${secret.key}`);
23+
process.exit(0);
24+
})
25+
.catch((err) => {
26+
console.log(`could not create mask for secret: ${secret.key}, due to error: ${err}`);
27+
process.exit(1);
28+
});
29+
}
30+
31+
if (require.main === module) {
32+
// first argument is the secret key second argument is the secret value
33+
if (process.argv.length < 4) {
34+
console.log('not enough arguments, need secret key and secret value');
35+
process.exit(2);
36+
}
37+
const key = process.argv[2];
38+
const value = process.argv[3];
39+
updateMasks({ key, value });
40+
} else {
41+
module.exports = updateMasks;
42+
}

‎lib/logger.js

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const LoggerStrategy = require('./enums').LoggerStrategy;
1313
const ContainerHandlingStatus = require('./enums').ContainerHandlingStatus;
1414
const ContainerLogger = require('./ContainerLogger');
1515
const { TaskLogger } = require('@codefresh-io/task-logger');
16+
const express = require('express');
1617

1718
const initialState = { pid: process.pid, status: 'init', lastLogsDate: new Date() , failedHealthChecks: [] , restartCounter: 0, containers: {} };
1819
class Logger {
@@ -117,6 +118,8 @@ class Logger {
117118
}));
118119
return;
119120
});
121+
122+
this._listenForEngineUpdates();
120123
}
121124

122125
_readState() {
@@ -337,6 +340,33 @@ class Logger {
337340
});
338341
}
339342

343+
_listenForEngineUpdates() {
344+
const app = express();
345+
this._app = app;
346+
const port = process.env.PORT || 8080;
347+
const host = process.env.HOST || 'localhost';
348+
349+
app.use(require('body-parser').json());
350+
351+
app.post('/secrets', (req, res) => {
352+
try {
353+
const secret = req.body;
354+
logger.info(`got request to add new mask: ${JSON.stringify(secret)}`);
355+
356+
// secret must have { key, value } structure
357+
this.taskLogger.addNewMask(secret);
358+
res.status(201).end('secret added');
359+
} catch (err) {
360+
logger.info(`could not create new mask due to error: ${err}`);
361+
res.status(400).end(err);
362+
}
363+
});
364+
365+
app.listen(port, host, () => {
366+
logger.info(`listening for engine updates on ${host}:${port}`);
367+
});
368+
}
369+
340370
_handleContainerStreamEnd() {
341371
this.finishedContainers++;
342372
this.finishedContainersEmitter.emit('end');

‎package.json

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,18 @@
66
"cf-container-logger"
77
],
88
"dependencies": {
9-
"@codefresh-io/task-logger": "1.6.0",
9+
"@codefresh-io/task-logger": "1.7.0",
10+
"body-parser": "^1.19.0",
1011
"cf-errors": "^0.1.11",
1112
"cf-logs": "^1.1.0",
1213
"docker-events": "0.0.2",
1314
"dockerode": "^2.3.0",
15+
"express": "^4.17.1",
1416
"forever": "^0.15.3",
1517
"lodash": "^4.15.0",
16-
"q": "^1.4.1"
18+
"q": "^1.4.1",
19+
"request": "^2.88.2",
20+
"request-promise": "^4.2.5"
1721
},
1822
"devDependencies": {
1923
"chai": "^4.2.0",

‎service.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version: 1.3.0
1+
version: 1.4.0

‎test/ContainerLogger.unit.spec.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -814,6 +814,7 @@ describe('Container Logger tests', () => {
814814
transformSpies.push(transform);
815815
return transform;
816816
},
817+
createMaskingStream: () => new PassThrough(),
817818
opts: {
818819
logsRateLimitConfig: {} // use stream
819820
}

‎test/addNewMask.unit.spec.js

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/* jshint ignore:start */
2+
3+
'use strict';
4+
const Q = require('q');
5+
6+
const chai = require('chai');
7+
const expect = chai.expect;
8+
const sinon = require('sinon');
9+
const sinonChai = require('sinon-chai');
10+
const proxyquire = require('proxyquire').noCallThru();
11+
chai.use(sinonChai);
12+
13+
const originalProcessExit = process.exit;
14+
15+
describe('addNewMask', () => {
16+
before(() => {
17+
process.exit = sinon.spy();
18+
});
19+
20+
beforeEach(() => {
21+
process.exit.resetHistory();
22+
});
23+
24+
after(() => {
25+
process.exit = originalProcessExit;
26+
});
27+
28+
describe('positive', () => {
29+
it('should send a request to add a secret', async () => {
30+
const rpSpy = sinon.spy(() => Q.resolve({ statusCode: 201 }));
31+
const addNewMask = proxyquire('../lib/addNewMask', {
32+
'request-promise': rpSpy
33+
});
34+
35+
process.env.PORT = 1337;
36+
process.env.HOST = '127.0.0.1';
37+
38+
const secret = {
39+
key: '123',
40+
value: 'ABC',
41+
};
42+
43+
addNewMask(secret);
44+
45+
expect(rpSpy).to.have.been.calledOnceWith({
46+
uri: `http://127.0.0.1:1337/secrets`,
47+
method: 'POST',
48+
json: true,
49+
body: secret,
50+
resolveWithFullResponse: true,
51+
});
52+
await Q.delay(10);
53+
expect(process.exit).to.have.been.calledOnceWith(0);
54+
});
55+
});
56+
57+
describe('negative', () => {
58+
it('should send a request to add a secret', async () => {
59+
const rpSpy = sinon.spy(() => Q.reject('could not send request'));
60+
const addNewMask = proxyquire('../lib/addNewMask', {
61+
'request-promise': rpSpy
62+
});
63+
64+
process.env.PORT = 1337;
65+
process.env.HOST = '127.0.0.1';
66+
67+
const secret = {
68+
key: '123',
69+
value: 'ABC',
70+
};
71+
72+
addNewMask(secret);
73+
await Q.delay(10);
74+
expect(process.exit).to.have.been.calledOnceWith(1);
75+
});
76+
});
77+
});
78+

‎test/logger.unit.spec.js

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@ const ContainerStatus = require('../lib/enums').ContainerStatus;
1111
const LoggerStrategy = require('../lib/enums').LoggerStrategy;
1212
const { EventEmitter } = require('events');
1313

14+
const expressMock = function() {
15+
return {
16+
use: sinon.spy(),
17+
listen: sinon.spy(),
18+
post: sinon.spy(),
19+
get: sinon.spy(),
20+
}
21+
}
22+
1423
describe('Logger tests', () => {
1524

1625
let processExit;
@@ -83,7 +92,8 @@ describe('Logger tests', () => {
8392
});
8493

8594
const Logger = proxyquire('../lib/logger', {
86-
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
95+
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
96+
'express': expressMock,
8797
});
8898

8999
const loggerId = 'loggerId';
@@ -126,7 +136,8 @@ describe('Logger tests', () => {
126136
});
127137

128138
const Logger = proxyquire('../lib/logger', {
129-
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
139+
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
140+
'express': expressMock,
130141
});
131142

132143
const loggerId = 'loggerId';
@@ -167,7 +178,8 @@ describe('Logger tests', () => {
167178
});
168179

169180
const Logger = proxyquire('../lib/logger', {
170-
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
181+
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
182+
'express': expressMock,
171183
});
172184

173185
const loggerId = 'loggerId';
@@ -207,7 +219,8 @@ describe('Logger tests', () => {
207219
});
208220

209221
const Logger = proxyquire('../lib/logger', {
210-
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
222+
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
223+
'express': expressMock,
211224
});
212225

213226
const loggerId = 'loggerId';
@@ -252,7 +265,8 @@ describe('Logger tests', () => {
252265
});
253266

254267
const Logger = proxyquire('../lib/logger', {
255-
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
268+
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
269+
'express': expressMock,
256270
});
257271

258272
const loggerId = 'loggerId';
@@ -1365,6 +1379,7 @@ describe('Logger tests', () => {
13651379
'@codefresh-io/task-logger': { TaskLogger: () => Q.resolve(taskLogger) },
13661380
'docker-events': function () { return dockerEvents; },
13671381
'./ContainerLogger': function () { return containerLogger; },
1382+
'express': expressMock,
13681383
});
13691384

13701385
const loggerId = 'loggerId';
@@ -1454,6 +1469,7 @@ describe('Logger tests', () => {
14541469
'@codefresh-io/task-logger': { TaskLogger: () => Q.resolve(taskLogger) },
14551470
'docker-events': function () { return dockerEvents; },
14561471
'./ContainerLogger': function () { return containerLogger; },
1472+
'express': expressMock,
14571473
});
14581474

14591475
const logger = new Logger({
@@ -1475,4 +1491,46 @@ describe('Logger tests', () => {
14751491
expect(logger._updateLastLoggingDate).to.have.been.calledOnce;
14761492
});
14771493
});
1494+
1495+
describe('engine updates', () => {
1496+
it('should listen for engine updates', async () => {
1497+
const taskLogger = {
1498+
on: sinon.spy(),
1499+
restore: sinon.spy(() => Q.resolve()),
1500+
startHealthCheck: sinon.spy(),
1501+
onHealthCheckReported: sinon.spy(),
1502+
getStatus: sinon.spy(),
1503+
};
1504+
const TaskLoggerFactory = sinon.spy(() => {
1505+
return Q.resolve(taskLogger);
1506+
});
1507+
1508+
const Logger = proxyquire('../lib/logger', {
1509+
'@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
1510+
'express': expressMock,
1511+
});
1512+
1513+
const loggerId = 'loggerId';
1514+
const taskLoggerConfig = {task: {}, opts: {}};
1515+
const findExistingContainers = false;
1516+
1517+
const logger = new Logger({
1518+
loggerId,
1519+
taskLoggerConfig,
1520+
findExistingContainers,
1521+
});
1522+
logger._listenForNewContainers = sinon.spy();
1523+
logger._writeNewState = sinon.spy();
1524+
logger._listenForExistingContainers = sinon.spy();
1525+
process.env.PORT = 1337;
1526+
process.env.HOST = '127.0.0.1';
1527+
logger.start();
1528+
1529+
await Q.delay(10);
1530+
1531+
expect(logger._app).to.not.be.undefined;
1532+
expect(logger._app.listen).to.have.been.calledOnce;
1533+
expect(logger._app.listen).to.have.been.calledWithMatch(1337, '127.0.0.1');
1534+
});
1535+
});
14781536
});

0 commit comments

Comments
 (0)