Skip to content

Commit

Permalink
new source redisfile, fix tests, /stats api
Browse files Browse the repository at this point in the history
  • Loading branch information
awick committed Jul 19, 2020
1 parent 6374bac commit 7ffbd1b
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 48 deletions.
97 changes: 52 additions & 45 deletions tests/wise.t
Original file line number Diff line number Diff line change
Expand Up @@ -14,72 +14,77 @@ my $wise;

# IP Query
$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/ip/10.0.0.3")->content;
eq_or_diff(from_json($wise), from_json('[{"field":"tags","len":10,"value":"wisebyip1"},
{"field":"irc.channel","len":17,"value":"wisebyip1channel"},
{"field":"email.x-priority","len":4,"value":"999"},
{"field":"tags","len":7,"value":"ipwise"},
{"field":"tags","len":10,"value":"ipwisecsv"}]
eq_or_diff(from_json($wise), from_json('[{"field":"tags","len":9,"value":"wisebyip1"},
{"field":"irc.channel","len":16,"value":"wisebyip1channel"},
{"field":"email.x-priority","len":3,"value":"999"},
{"field":"tags","len":6,"value":"ipwise"},
{"field":"tags","len":9,"value":"ipwisecsv"}]
'),"All 10.0.0.3");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/file:ip/ip/10.0.0.3")->content;
eq_or_diff(from_json($wise), from_json('[{"field":"tags","len":10,"value":"wisebyip1"},
{"field":"irc.channel","len":17,"value":"wisebyip1channel"},
{"field":"email.x-priority","len":4,"value":"999"},
{"field":"tags","len":7,"value":"ipwise"}]
eq_or_diff(from_json($wise), from_json('[{"field":"tags","len":9,"value":"wisebyip1"},
{"field":"irc.channel","len":16,"value":"wisebyip1channel"},
{"field":"email.x-priority","len":3,"value":"999"},
{"field":"tags","len":6,"value":"ipwise"}]
'),"file:ip 10.0.0.3");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/file:ipblah/ip/10.0.0.3")->content;
eq_or_diff($wise, 'Unknown source file:ipblah',"file:ipblah 10.0.0.3");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/ip/10.0.0.2")->content;
eq_or_diff($wise, '[{"field":"tags","len":10,"value":"ipwisecsv"}]', "All 10.0.0.2");
eq_or_diff($wise, '[{"field":"tags","len":9,"value":"ipwisecsv"}]', "All 10.0.0.2");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/ip/10.0.0.1")->content;
eq_or_diff($wise, '[]',"All 10.0.0.1");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/ip/2001:16d8:ffce:0010:aca8:353c:291d:a9b3")->content;
eq_or_diff($wise, '[{"field":"tags","len":13,"value":"ipwise-array"},{"field":"tags","len":11,"value":"ipwisejson"}]');
eq_or_diff($wise, '[{"field":"tags","len":12,"value":"ipwise-array"},
{"field":"tags","len":10,"value":"ipwisejson"}]');

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/ip/2001:16d8:ffce:0010:aca8:353c:291d:0001")->content;
eq_or_diff($wise, '[{"field":"tags","len":14,"value":"ipwise-normal"},{"field":"tags","len":11,"value":"ipwisejson"}]');
eq_or_diff($wise, '[{"field":"tags","len":13,"value":"ipwise-normal"},
{"field":"tags","len":10,"value":"ipwisejson"}]');

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/ip/2001:16d8:ffce:0010:aca8:353c:291d:0002")->content;
eq_or_diff($wise, '[{"field":"tags","len":13,"value":"ipwise-comma"},{"field":"tags","len":11,"value":"ipwisejson"}]');
eq_or_diff($wise, '[{"field":"tags","len":12,"value":"ipwise-comma"},
{"field":"tags","len":10,"value":"ipwisejson"}]');

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/ip/10.20.30.50")->content;
eq_or_diff($wise, '[{"field":"tags","len":13,"value":"ipwise-array"},{"field":"tags","len":11,"value":"ipwisejson"}]');
eq_or_diff($wise, '[{"field":"tags","len":12,"value":"ipwise-array"},
{"field":"tags","len":10,"value":"ipwisejson"}]');

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/ip/10.20.30.51")->content;
eq_or_diff($wise, '[{"field":"tags","len":13,"value":"ipwise-comma"},{"field":"tags","len":11,"value":"ipwisejson"}]');
eq_or_diff($wise, '[{"field":"tags","len":12,"value":"ipwise-comma"},
{"field":"tags","len":10,"value":"ipwisejson"}]');

# IP File Dump
$wise = "[" . $MolochTest::userAgent->get("http://$MolochTest::host:8081/dump/file:ip")->content . "]";
my @wise = sort { $a->{key} cmp $b->{key}} @{from_json($wise, {relaxed=>1, allow_barekey=>1})};
eq_or_diff(\@wise,
from_json('[
{key:"10.0.0.3",ops:
[{"field":"tags","len":7,"value":"ipwise"},
{"field":"tags","len":10,"value":"wisebyip1"},
{"field":"irc.channel","len":17,"value":"wisebyip1channel"},
{"field":"email.x-priority","len":4,"value":"999"}]
[{"field":"tags","len":6,"value":"ipwise"},
{"field":"tags","len":9,"value":"wisebyip1"},
{"field":"irc.channel","len":16,"value":"wisebyip1channel"},
{"field":"email.x-priority","len":3,"value":"999"}]
},
{key:"128.128.128.0/24",ops:
[{"field":"tags","len":7,"value":"ipwise"},
{"field":"tags","len":10,"value":"wisebyip2"},
{"field":"mysql.ver","len":22,"value":"wisebyip2mysqlversion"},
{"field":"test.ip","len":12,"value":"21.21.21.21"}]
[{"field":"tags","len":6,"value":"ipwise"},
{"field":"tags","len":9,"value":"wisebyip2"},
{"field":"mysql.ver","len":21,"value":"wisebyip2mysqlversion"},
{"field":"test.ip","len":11,"value":"21.21.21.21"}]
},
{key:"192.168.177.160",ops:
[{"field":"tags","len":7,"value":"ipwise"},
{"field":"tags","len":10,"value":"wisebyip2"},
{"field":"mysql.ver","len":22,"value":"wisebyip2mysqlversion"},
{"field":"test.ip","len":12,"value":"21.21.21.21"}]
[{"field":"tags","len":6,"value":"ipwise"},
{"field":"tags","len":9,"value":"wisebyip2"},
{"field":"mysql.ver","len":21,"value":"wisebyip2mysqlversion"},
{"field":"test.ip","len":11,"value":"21.21.21.21"}]
},
{key:"fe80::211:25ff:fe82:95b5",ops:
[{"field":"tags","len":7,"value":"ipwise"},
{"field":"tags","len":10,"value":"wisebyip3"},
{"field":"mysql.ver","len":22,"value":"wisebyip3mysqlversion"},
{"field":"test.ip","len":12,"value":"22.22.22.22"}]
[{"field":"tags","len":6,"value":"ipwise"},
{"field":"tags","len":9,"value":"wisebyip3"},
{"field":"mysql.ver","len":21,"value":"wisebyip3mysqlversion"},
{"field":"test.ip","len":11,"value":"22.22.22.22"}]
}
]', {relaxed=>1, allow_barekey=>1}), "file:ip dump");

Expand All @@ -88,10 +93,10 @@ $wise = "[" . $MolochTest::userAgent->get("http://$MolochTest::host:8081/dump/fi
eq_or_diff(\@wise,
from_json('[
{key:"10.0.0.2",ops:
[{"field":"tags","len":10,"value":"ipwisecsv"}]
[{"field":"tags","len":9,"value":"ipwisecsv"}]
},
{key:"10.0.0.3",ops:
[{"field":"tags","len":10,"value":"ipwisecsv"}]
[{"field":"tags","len":9,"value":"ipwisecsv"}]
}
]', {relaxed=>1, allow_barekey=>1}), "file:ipcsv dump");

Expand All @@ -100,14 +105,14 @@ $wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/email/fudge\@
eq_or_diff($wise, '[]',"ALL fudge\@aol.com");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/email/fudge\@fudge.com")->content;
eq_or_diff($wise, '[{"field":"tags","len":10,"value":"emailwise"}]',"ALL fudge\@fudge.com");
eq_or_diff($wise, '[{"field":"tags","len":9,"value":"emailwise"}]',"ALL fudge\@fudge.com");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/email/12345678\@aol.com")->content;
eq_or_diff(from_json($wise), from_json('[{"field":"email.dst","len":11,"value":"wiseadded1"},
{"field":"tags","len":13,"value":"wisesrcmatch"},
{"field":"wise.str","len":6,"value":"house"},
{"field":"wise.str","len":5,"value":"boat"},
{"field":"tags","len":10,"value":"emailwise"}]
eq_or_diff(from_json($wise), from_json('[{"field":"email.dst","len":10,"value":"wiseadded1"},
{"field":"tags","len":12,"value":"wisesrcmatch"},
{"field":"wise.str","len":5,"value":"house"},
{"field":"wise.str","len":4,"value":"boat"},
{"field":"tags","len":9,"value":"emailwise"}]
'),"ALL 12345678\@aol.com");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/rightClicks")->content;
Expand Down Expand Up @@ -162,20 +167,22 @@ my @wise = sort { $a->{key} cmp $b->{key}} @{from_json($wise, {relaxed=>1, allow
eq_or_diff(\@wise,
from_json('[
{key:"00:12:1e:f2:61:3d",ops:
[{"field":"tags","len":8,"value":"macwise"},
{"field":"tags","len":11,"value":"wisebymac1"}]
[{"field":"tags","len":7,"value":"macwise"},
{"field":"tags","len":10,"value":"wisebymac1"}]
},
{key:"00:19:06:e6:82:c4",ops:
[{"field":"tags","len":8,"value":"macwise"},
{"field":"tags","len":11,"value":"wisebymac2"}]
[{"field":"tags","len":7,"value":"macwise"},
{"field":"tags","len":10,"value":"wisebymac2"}]
}
]', {relaxed=>1, allow_barekey=>1}), "file:mac dump");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/mac/00:12:1e:f2:61:3d")->content;
eq_or_diff($wise, '[{"field":"tags","len":11,"value":"wisebymac1"},{"field":"tags","len":8,"value":"macwise"}]',"mac query");
eq_or_diff($wise, '[{"field":"tags","len":10,"value":"wisebymac1"},
{"field":"tags","len":7,"value":"macwise"}]',"mac query");

$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/file:mac/mac/00:12:1e:f2:61:3d")->content;
eq_or_diff($wise, '[{"field":"tags","len":11,"value":"wisebymac1"},{"field":"tags","len":8,"value":"macwise"}]',"file:mac query");
eq_or_diff($wise, '[{"field":"tags","len":10,"value":"wisebymac1"},
{"field":"tags","len":7,"value":"macwise"}]',"file:mac query");

# Sources
$wise = $MolochTest::userAgent->get("http://$MolochTest::host:8081/sources")->content;
Expand Down
76 changes: 76 additions & 0 deletions wiseService/source.redisfile.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/******************************************************************************/
/*
*
* Copyright 2012-2016 AOL Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this Software except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';

var util = require('util');
var simpleSource = require('./simpleSource.js');

// ----------------------------------------------------------------------------
function RedisFileSource (api, section) {
RedisFileSource.super_.call(this, api, section);
this.key = api.getConfig(section, 'key');
this.reload = +api.getConfig(section, 'reload', -1);
this.headers = {};
var headers = api.getConfig(section, 'headers');
this.cacheTimeout = -1;
this.client = api.createRedisClient(api.getConfig(section, 'redisType', 'redis'), section);

if (this.key === undefined) {
console.log(this.section, '- ERROR not loading since no key specified in config file');
return;
}

if (headers) {
headers.split(';').forEach((header) => {
var parts = header.split(':').map(item => item.trim());
if (parts.length === 2) {
this.headers[parts[0]] = parts[1];
}
});
}

if (!this.initSimple()) {
return;
}

setImmediate(this.load.bind(this));

// Reload key every so often
if (this.reload > 0) {
setInterval(this.load.bind(this), this.reload * 1000 * 60);
}
}
util.inherits(RedisFileSource, simpleSource);
// ----------------------------------------------------------------------------
RedisFileSource.prototype.simpleSourceLoad = function (setFunc, cb) {
this.client.get(this.key, (error, reply) => {
if (reply === null) {
cb(error);
} else {
this.parse(reply, setFunc, cb);
}
});
};
// ----------------------------------------------------------------------------
exports.initSource = function (api) {
var sections = api.getConfigSections().filter((e) => { return e.match(/^redisfile:/); });
sections.forEach((section) => {
return new RedisFileSource(api, section);
});
};
// ----------------------------------------------------------------------------
37 changes: 35 additions & 2 deletions wiseService/wiseService.js
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,7 @@ function addType (type, newSrc) {
foundStats: 0,
cacheHitStats: 0,
cacheSrcHitStats: 0,
cacheSrcMissStats: 0,
cacheSrcRefreshStats: 0,
excludes: [],
globalAllowed: globalAllowed,
Expand Down Expand Up @@ -534,6 +535,7 @@ function processQuery (req, query, cb) {
if (cacheResult[src.section] === undefined || cacheResult[src.section].ts + src.cacheTimeout < now) {
if (cacheResult[src.section] === undefined) {
src.cacheMissStat++;
typeInfo.cacheSrcMissStats++;
} else {
src.cacheRefreshStat++;
typeInfo.cacheSrcRefreshStats++;
Expand Down Expand Up @@ -709,7 +711,7 @@ app.get('/:source/:typeName/:value', [noCacheJson], function (req, res) {
if (err || !result) {
return res.end('Not found');
}
res.end(wiseSource.result2Str(result));
res.send(wiseSource.result2Str(result));
});
});
// ----------------------------------------------------------------------------
Expand Down Expand Up @@ -810,10 +812,41 @@ app.get('/:typeName/:value', [noCacheJson], function (req, res) {
if (err || !result) {
return res.end('Not found');
}
res.end(wiseSource.result2Str(result));
res.send(wiseSource.result2Str(result));
});
});
// ----------------------------------------------------------------------------
app.get('/stats', [noCacheJson], function (req, res) {
let types = Object.keys(internals.types).sort();
let stats = { types: [], sources: [] };

for (var type of types) {
let typeInfo = internals.types[type];
stats.types.push({
type: type,
request: typeInfo.requestStats,
found: typeInfo.foundStats,
cacheHit: typeInfo.cacheHitStats,
cacheSrcHit: typeInfo.cacheSrcHitStats,
cacheSrcMiss: typeInfo.cacheSrcMissStats,
cacheSrcRefresh: typeInfo.cacheSrcRefreshStats
});
}

for (let section in internals.sources) {
let src = internals.sources[section];
stats.sources.push({
source: section,
cacheHit: src.cacheHitStat,
cacheMiss: src.cacheMissStat,
cacheRefresh: src.cacheRefreshStat,
cacheDropped: src.cacheDroppedStat,
average100MS: src.average100MS
});
}
res.send(stats);
});
// ----------------------------------------------------------------------------
if (getConfig('wiseService', 'regressionTests')) {
app.post('/shutdown', (req, res) => {
process.exit(0);
Expand Down
2 changes: 1 addition & 1 deletion wiseService/wiseSource.js
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ WISESource.result2Str = function (result, indent) {
var len = result[offset + 1];
var value = result.toString('utf8', offset + 2, offset + 2 + len - 1);
offset += 2 + len;
collection.push({ field: WISESource.pos2Field[pos], len: len, value: value });
collection.push({ field: WISESource.pos2Field[pos], len: len - 1, value: value });
}

return JSON.stringify(collection).replace(/},{/g, '},\n{');
Expand Down

0 comments on commit 7ffbd1b

Please sign in to comment.