diff --git a/CHANGELOG b/CHANGELOG index 1d0a2163df..289597132a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -9,6 +9,7 @@ ES Versions: * Moloch >= 0.18.1 supports ES 2.4.x, >= 5.3.1 not 6.x or later Node Versions: + * Moloch >= 2.4.0 requires NodeJS 12.x * Moloch >= 2.0.0 requires NodeJS 10.x * Moloch >= 1.6.0 requires NodeJS 8.x, 8.12 or later * Moloch >= 1.0.0 requires NodeJS 8.x @@ -18,7 +19,9 @@ Node Versions: NOTICE: Restart wiseService before capture when upgrading 2.4.0 2020/07/xx - - NOTE - RHEL/Centos 6 is no longer supported + - NOTE - RHEL/Centos 6 is no longer supported, Node 12 required + - release - node 12.18.2, glib 2.64.3 + - viewer - aes256Encryption now defaults to true 2.3.2 2020/06/29 - NOTE - 2.3.x will be the last version to support RHEL/Centos 6 diff --git a/capture/Makefile.in b/capture/Makefile.in index f1985db9d4..d73687be95 100644 --- a/capture/Makefile.in +++ b/capture/Makefile.in @@ -20,6 +20,7 @@ LIB_OTHER = @GLIB2_LIBS@ \ @YARA_LIBS@ \ @MAXMINDDB_LIBS@ \ @CURL_LIBS@ \ + @NGHTTP2_LIBS@ \ @LIBS@ \ thirdparty/http_parser.o \ thirdparty/js0n.o \ diff --git a/capture/parsers.c b/capture/parsers.c index b4ec235991..39f82092f7 100644 --- a/capture/parsers.c +++ b/capture/parsers.c @@ -735,6 +735,7 @@ void moloch_parsers_init() if (!parser) { LOG("ERROR - Couldn't load parser %s from '%s'\n%s", filenames[i], path, g_module_error()); + g_free(filenames[i]); g_free (path); continue; } @@ -744,6 +745,7 @@ void moloch_parsers_init() if (!g_module_symbol(parser, "moloch_parser_init", (gpointer *)(char*)&parser_init) || parser_init == NULL) { LOG("ERROR - Module %s doesn't have a moloch_parser_init", filenames[i]); g_free(filenames[i]); + g_free (path); continue; } diff --git a/capture/parsers/Makefile.in b/capture/parsers/Makefile.in index 630d51952a..cc07efd5e4 100644 --- a/capture/parsers/Makefile.in +++ b/capture/parsers/Makefile.in @@ -1,6 +1,7 @@ INCLUDE_PCAP = @PCAP_CFLAGS@ INCLUDE_OTHER = -I.. -I../thirdparty \ + @NGHTTP2_CFLAGS@ \ @GLIB2_CFLAGS@ install_sh = @install_sh@ @@ -16,6 +17,9 @@ SOS=$(SRCS:.c=.so) all:$(SOS) +sanitize: + $(MAKE) SANITIZE_FLAGS="-DMOLOCH_USE_MALLOC -fno-common -fsanitize=address -fsanitize=integer -fsanitize=nullability" + install: @mkdir -p "$(PARSERSDIR)" $(INSTALL) *.so *.jade $(PARSERSDIR) diff --git a/capture/parsers/http.c b/capture/parsers/http.c index b20e5991f9..7f04631e31 100644 --- a/capture/parsers/http.c +++ b/capture/parsers/http.c @@ -28,7 +28,6 @@ typedef struct { GString *authString; GString *valueString[2]; - gboolean isValueStringLowerCase[2]; char header[2][40]; short pos[2]; @@ -79,6 +78,167 @@ LOCAL int headerReqValue; LOCAL int headerResField; LOCAL int headerResValue; +/******************************************************************************/ +void http_common_parse_cookie(MolochSession_t *session, char *cookie, int len) +{ + char *start = cookie; + char *end = cookie + len; + while (1) { + while (isspace(*start) && start < end) start++; + char *equal = memchr(start, '=', end - start); + if (!equal) + break; + moloch_field_string_add(cookieKeyField, session, start, equal-start, TRUE); + start = memchr(equal+1, ';', end - (equal+1)); + if (config.parseCookieValue) { + equal++; + while (isspace(*equal) && equal < end) equal++; + if (equal < end && equal != start) + moloch_field_string_add(cookieValueField, session, equal, start?start-equal:end-equal, TRUE); + } + + if(!start) + break; + start++; + } +} +/******************************************************************************/ +void http_common_add_header_value(MolochSession_t *session, int pos, const char *s, int l) +{ + while (isspace(*s)) { + s++; + l--; + } + + switch (config.fields[pos]->type) { + case MOLOCH_FIELD_TYPE_INT: + case MOLOCH_FIELD_TYPE_INT_ARRAY: + case MOLOCH_FIELD_TYPE_INT_HASH: + case MOLOCH_FIELD_TYPE_INT_GHASH: + moloch_field_int_add(pos, session, atoi(s)); + break; + case MOLOCH_FIELD_TYPE_STR: + case MOLOCH_FIELD_TYPE_STR_ARRAY: + case MOLOCH_FIELD_TYPE_STR_HASH: + case MOLOCH_FIELD_TYPE_STR_GHASH: + if (pos == headerReqValue || pos == headerResValue) + moloch_field_string_add_lower(pos, session, s, MIN(l, 1024)); + else + moloch_field_string_add(pos, session, s, l, TRUE); + break; + case MOLOCH_FIELD_TYPE_IP: + case MOLOCH_FIELD_TYPE_IP_GHASH: + { + int i; + gchar **parts = g_strsplit(s, ",", 0); + + for (i = 0; parts[i]; i++) { + moloch_field_ip_add_str(pos, session, parts[i]); + + /* Add back maybe + if (ia == 0 || ia == 0xffffffff) { + moloch_session_add_tag(session, "http:bad-xff"); + if (config.debug) + LOG("INFO - Didn't understand ip: %s %s %d", s, ip, ia); + continue; + } + */ + } + + g_strfreev(parts); + break; + } + case MOLOCH_FIELD_TYPE_CERTSINFO: + // Unsupported + break; + } /* SWITCH */ +} +/******************************************************************************/ +void http_common_add_header(MolochSession_t *session, int pos, int isReq, const char *name, int namelen, const char *value, int valuelen) +{ + MolochString_t *hstring; + + char *lower = g_ascii_strdown(name, namelen); + + if (isReq) + moloch_field_string_add(tagsReqField, session, (const char *)lower, namelen, TRUE); + else + moloch_field_string_add(tagsResField, session, (const char *)lower, namelen, TRUE); + + if (pos == 0) { + if (isReq) + HASH_FIND(s_, httpReqHeaders, lower, hstring); + else + HASH_FIND(s_, httpResHeaders, lower, hstring); + } + + if (pos == 0) { + if (isReq && config.parseHTTPHeaderRequestAll) { // Header in request + moloch_field_string_add(headerReqField, session, lower, -1, TRUE); + pos = headerReqValue; + } + else if (!isReq && config.parseHTTPHeaderResponseAll) { // Header in response + moloch_field_string_add(headerResField, session, lower, -1, TRUE); + pos = headerResValue; + } + } + + g_free(lower); + + if (pos == 0) + return; + + http_common_add_header_value(session, pos, (char *)value, valuelen); +} +/******************************************************************************/ +void http_common_parse_url(MolochSession_t *session, char *url, int len) +{ + char *end = url + len; + char *question = memchr(url, '?', len); + + if (question) { + moloch_field_string_add(pathField, session, url, question - url, TRUE); + char *start = question+1; + char *ch; + int field = keyField; + for (ch = start; ch < end; ch++) { + if (*ch == '&') { + if (ch != start && (config.parseQSValue || field == keyField)) { + char *str = g_uri_unescape_segment(start, ch, NULL); + if (!str) { + moloch_field_string_add(field, session, start, ch-start, TRUE); + } else if (!moloch_field_string_add(field, session, str, -1, FALSE)) { + g_free(str); + } + } + start = ch+1; + field = keyField; + continue; + } else if (*ch == '=') { + if (ch != start && (config.parseQSValue || field == keyField)) { + char *str = g_uri_unescape_segment(start, ch, NULL); + if (!str) { + moloch_field_string_add(field, session, start, ch-start, TRUE); + } else if (!moloch_field_string_add(field, session, str, -1, FALSE)) { + g_free(str); + } + } + start = ch+1; + field = valueField; + } + } + if (config.parseQSValue && field == valueField && ch > start) { + char *str = g_uri_unescape_segment(start, ch, NULL); + if (!str) { + moloch_field_string_add(field, session, start, ch-start, TRUE); + } else if (!moloch_field_string_add(field, session, str, -1, FALSE)) { + g_free(str); + } + } + } else { + moloch_field_string_add(pathField, session, url, len, TRUE); + } +} /******************************************************************************/ LOCAL int moloch_hp_cb_on_message_begin (http_parser *parser) { @@ -237,66 +397,17 @@ LOCAL int moloch_hp_cb_on_message_complete (http_parser *parser) return 0; } -/******************************************************************************/ /******************************************************************************/ LOCAL void http_add_value(MolochSession_t *session, HTTPInfo_t *http) { int pos = http->pos[http->which]; char *s = http->valueString[http->which]->str; int l = http->valueString[http->which]->len; - gboolean c = http->isValueStringLowerCase[http->which]; - - while (isspace(*s)) { - s++; - l--; - } - - switch (config.fields[pos]->type) { - case MOLOCH_FIELD_TYPE_INT: - case MOLOCH_FIELD_TYPE_INT_ARRAY: - case MOLOCH_FIELD_TYPE_INT_HASH: - case MOLOCH_FIELD_TYPE_INT_GHASH: - moloch_field_int_add(pos, session, atoi(s)); - break; - case MOLOCH_FIELD_TYPE_STR: - case MOLOCH_FIELD_TYPE_STR_ARRAY: - case MOLOCH_FIELD_TYPE_STR_HASH: - case MOLOCH_FIELD_TYPE_STR_GHASH: - if (c == TRUE) - moloch_field_string_add_lower(pos, session, s, l); - else - moloch_field_string_add(pos, session, s, l, TRUE); - break; - case MOLOCH_FIELD_TYPE_IP: - case MOLOCH_FIELD_TYPE_IP_GHASH: - { - int i; - gchar **parts = g_strsplit(http->valueString[http->which]->str, ",", 0); - - for (i = 0; parts[i]; i++) { - moloch_field_ip_add_str(pos, session, parts[i]); - - /* Add back maybe - if (ia == 0 || ia == 0xffffffff) { - moloch_session_add_tag(session, "http:bad-xff"); - if (config.debug) - LOG("INFO - Didn't understand ip: %s %s %d", http->valueString[http->which]->str, ip, ia); - continue; - } - */ - } - g_strfreev(parts); - break; - } - case MOLOCH_FIELD_TYPE_CERTSINFO: - // Unsupported - break; - } /* SWITCH */ + http_common_add_header_value(session, pos, s, l); g_string_truncate(http->valueString[http->which], 0); http->pos[http->which] = 0; - http->isValueStringLowerCase[http->which] = FALSE; } /******************************************************************************/ LOCAL int moloch_hp_cb_on_header_field (http_parser *parser, const char *at, size_t length) @@ -366,12 +477,10 @@ LOCAL int moloch_hp_cb_on_header_value (http_parser *parser, const char *at, siz if ((http->which == 0) && config.parseHTTPHeaderRequestAll) { // Header in request moloch_field_string_add(headerReqField, session, lower, -1, TRUE); http->pos[http->which] = (long) headerReqValue; - http->isValueStringLowerCase[http->which] = TRUE; } else if ((http->which == 1) && config.parseHTTPHeaderResponseAll) { // Header in response moloch_field_string_add(headerResField, session, lower, -1, TRUE); http->pos[http->which] = (long) headerResValue; - http->isValueStringLowerCase[http->which] = TRUE; } } @@ -421,7 +530,7 @@ LOCAL int moloch_hp_cb_on_headers_complete (http_parser *parser) char version[20]; #ifdef HTTPDEBUG - LOG("HTTPDEBUG: which: %d code: %d method: %d", http->which, parser->status_code, parser->method); + LOG("HTTPDEBUG: which: %d code: %d method: %d upgrade: %d", http->which, parser->status_code, parser->method, parser->upgrade); #endif if (parser->method == HTTP_CONNECT) { @@ -457,25 +566,7 @@ LOCAL int moloch_hp_cb_on_headers_complete (http_parser *parser) } if (http->cookieString && http->cookieString->str[0]) { - char *start = http->cookieString->str; - while (1) { - while (isspace(*start)) start++; - char *equal = strchr(start, '='); - if (!equal) - break; - moloch_field_string_add(cookieKeyField, session, start, equal-start, TRUE); - start = strchr(equal+1, ';'); - if (config.parseCookieValue) { - equal++; - while (isspace(*equal)) equal++; - if (*equal && equal != start) - moloch_field_string_add(cookieValueField, session, equal, start?start-equal:-1, TRUE); - } - - if(!start) - break; - start++; - } + http_common_parse_cookie(session, http->cookieString->str, http->cookieString->len); g_string_truncate(http->cookieString, 0); } @@ -497,49 +588,7 @@ LOCAL int moloch_hp_cb_on_headers_complete (http_parser *parser) moloch_field_string_add(hostField, session, http->hostString->str, http->hostString->len, TRUE); } - char *question = strchr(http->urlString->str, '?'); - if (question) { - moloch_field_string_add(pathField, session, http->urlString->str, question - http->urlString->str, TRUE); - char *start = question+1; - char *ch; - int field = keyField; - for (ch = start; *ch; ch++) { - if (*ch == '&') { - if (ch != start && (config.parseQSValue || field == keyField)) { - char *str = g_uri_unescape_segment(start, ch, NULL); - if (!str) { - moloch_field_string_add(field, session, start, ch-start, TRUE); - } else if (!moloch_field_string_add(field, session, str, -1, FALSE)) { - g_free(str); - } - } - start = ch+1; - field = keyField; - continue; - } else if (*ch == '=') { - if (ch != start && (config.parseQSValue || field == keyField)) { - char *str = g_uri_unescape_segment(start, ch, NULL); - if (!str) { - moloch_field_string_add(field, session, start, ch-start, TRUE); - } else if (!moloch_field_string_add(field, session, str, -1, FALSE)) { - g_free(str); - } - } - start = ch+1; - field = valueField; - } - } - if (config.parseQSValue && field == valueField && ch > start) { - char *str = g_uri_unescape_segment(start, ch, NULL); - if (!str) { - moloch_field_string_add(field, session, start, ch-start, TRUE); - } else if (!moloch_field_string_add(field, session, str, -1, FALSE)) { - g_free(str); - } - } - } else { - moloch_field_string_add(pathField, session, http->urlString->str, http->urlString->len, TRUE); - } + http_common_parse_url(session, http->urlString->str, http->urlString->len); if (http->urlString->str[0] != '/') { char *result = strstr(http->urlString->str, http->hostString->str); diff --git a/capture/parsers/http2.c b/capture/parsers/http2.c new file mode 100644 index 0000000000..9a9c81ca31 --- /dev/null +++ b/capture/parsers/http2.c @@ -0,0 +1,528 @@ +/* Copyright 2012-2017 AOL Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this Software except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * https://http2.github.io/http2-spec/ + */ +#include "moloch.h" +#include +#include +#include "nghttp2/nghttp2.h" + +//#define HTTPDEBUG 1 + +#define MAX_URL_LENGTH 4096 +#define MAX_HTTP2_SIZE 20000 +#define MAX_STREAMS 16 + +typedef struct { + uint32_t id; + uint8_t ended; + const char *magicString[2]; + GChecksum *checksum[4]; +} HTTP2Stream_t; + +typedef enum { + HTTP2_STATE_NORMAL, + HTTP2_STATE_IN_DATA +} HTTP2InfoState_t; + +typedef struct { + nghttp2_hd_inflater *hd_inflater[2]; + unsigned char data[2][MAX_HTTP2_SIZE]; + int used[2]; + uint8_t lastType[2]; + uint8_t dataPadding[2]; + uint8_t isEnd[2]; + int dataNeeded[2]; + uint32_t dataStreamId[2]; + int which; + + int numStreams; + HTTP2Stream_t streams[MAX_STREAMS]; +} HTTP2Info_t; + +#ifdef HTTPDEBUG +LOCAL const char *http2_frameNames[] = {"DATA", "HEADERS", "PRIORITY", "RST_STREAM", "SETTINGS", "PUSH_PROMISE", "PING", "GOAWAY", "WINDOW_UPDATE", "CONTINUATION", "ALTSVC", "ORIGIN"}; +#endif + + +extern MolochConfig_t config; + +LOCAL int statuscodeField; +LOCAL int methodField; +LOCAL int hostField; +LOCAL int magicField; +LOCAL int md5Field; +LOCAL int sha256Field; + + +void http_common_parse_cookie(MolochSession_t *session, char *cookie, int len); +void http_common_add_header(MolochSession_t *session, int pos, int isReq, const char *name, int namelen, const char *value, int valuelen); +void http_common_parse_url(MolochSession_t *session, char *url, int len); + +/******************************************************************************/ +LOCAL int http2_spos_get(HTTP2Info_t *http2, uint32_t streamId, int create) +{ + streamId &= 0x7fffffff; + streamId++; + + for (int i = 0; i < http2->numStreams; i++) { + if (streamId == http2->streams[i].id) + return i; + } + + // Not found, if we aren't creating then return error + if (!create) + return -1; + + // See if any slots are free and use that + for (int i = 0; i < http2->numStreams; i++) { + if (http2->streams[i].id == 0) { + http2->streams[http2->numStreams].id = streamId; + return i; + } + } + + // See if we can add to the end + if (http2->numStreams == MAX_STREAMS) + return -1; + http2->streams[http2->numStreams].id = streamId; + http2->numStreams++; + return http2->numStreams-1; +} +/******************************************************************************/ +LOCAL void http2_spos_free(HTTP2Info_t *http2, uint32_t streamId) +{ + streamId &= 0x7fffffff; + streamId++; + + for (int i = 0; i < http2->numStreams; i++) { + if (streamId == http2->streams[i].id) { + g_checksum_free(http2->streams[i].checksum[0]); + g_checksum_free(http2->streams[i].checksum[1]); + if (config.supportSha256) { + g_checksum_free(http2->streams[i].checksum[2]); + g_checksum_free(http2->streams[i].checksum[3]); + } + memset(&http2->streams[i], 0, sizeof(http2->streams[i])); + return; + } + } +} +/******************************************************************************/ +LOCAL void http2_parse_header_block(MolochSession_t *session, HTTP2Info_t *http2, int which, uint8_t flags, uint32_t streamId, unsigned char *in, int inlen) +{ + int spos = http2_spos_get(http2, streamId, TRUE); + if (spos == -1) + return; + + if (!http2->hd_inflater[which]) + nghttp2_hd_inflate_new(&http2->hd_inflater[which]); + + int final = flags & NGHTTP2_FLAG_END_HEADERS; + +#ifdef HTTPDEBUG + LOG("%d,%d: which:%d inlen:%d final:%d %.*s", streamId, spos, which, inlen, final, inlen, in); + //moloch_print_hex_string(in, inlen); +#endif + + // https://nghttp2.org/documentation/nghttp2_hd_inflate_hd2.html + for(;;) { + nghttp2_nv nv; + int inflate_flags = 0; + + ssize_t rv = nghttp2_hd_inflate_hd2(http2->hd_inflater[which], &nv, &inflate_flags, + in, inlen, final); + + if(rv < 0) { + LOG("inflate failed with error code %zd", rv); + return; + } + + in += rv; + inlen -= rv; + + if(inflate_flags & NGHTTP2_HD_INFLATE_EMIT) { + if (nv.name[0] == ':') { + if (memcmp(nv.name, ":method", 7) == 0) { + moloch_field_string_add(methodField, session, (char *)nv.value, nv.valuelen, TRUE); + } else if (memcmp(nv.name, ":authority", 10) == 0) { + uint8_t *colon = memchr(nv.value, ':', nv.valuelen); + if (colon) { + moloch_field_string_add(hostField, session, (char *)nv.value, colon - nv.value, TRUE); + } else { + moloch_field_string_add(hostField, session, (char *)nv.value, nv.valuelen, TRUE); + } + } else if (memcmp(nv.name, ":path", 5) == 0) { + http_common_parse_url(session, (char *)nv.value, nv.valuelen); + } else if (memcmp(nv.name, ":status", 7) == 0) { + moloch_field_int_add(statuscodeField, session, atoi((const char *)nv.value)); + } + } else { + http_common_add_header(session, 0, which == http2->which, (const char *)nv.name, nv.namelen, (const char *)nv.value, nv.valuelen); + + if (memcmp(nv.name, "cookie", 6) == 0) { + http_common_parse_cookie(session, (char *)nv.value, nv.valuelen); + } + } +#ifdef HTTPDEBUG + fwrite(nv.name, nv.namelen, 1, stderr); + fprintf(stderr, ": "); + fwrite(nv.value, nv.valuelen, 1, stderr); + fprintf(stderr, "\n"); +#endif + } + if(inflate_flags & NGHTTP2_HD_INFLATE_FINAL) { + nghttp2_hd_inflate_end_headers(http2->hd_inflater[which]); + break; + } + if((inflate_flags & NGHTTP2_HD_INFLATE_EMIT) == 0 && + inlen == 0) { + break; + } + } +} +/******************************************************************************/ +/* + * https://http2.github.io/http2-spec/#HEADERS + * +---------------+ + * |Pad Length? (8)| + * +-+-------------+-----------------------------------------------+ + * |E| Stream Dependency? (31) | + * +-+-------------+-----------------------------------------------+ + * | Weight? (8) | + * +-+-------------+-----------------------------------------------+ + * | Header Block Fragment (*) ... + * +---------------------------------------------------------------+ + * | Padding (*) ... + * +---------------------------------------------------------------+ + */ +LOCAL void http2_parse_frame_headers(MolochSession_t *session, HTTP2Info_t *http2, int which, uint8_t flags, uint32_t streamId, unsigned char *in, int inlen) +{ + if (flags & NGHTTP2_FLAG_PADDED) { + uint8_t padding = in[0]; + in++; + inlen -= (1 + padding); + } + + if (flags & NGHTTP2_FLAG_PRIORITY) { + in +=5; + inlen -= 5; + } + + if (inlen < 0) + return; + http2_parse_header_block(session, http2, which, flags, streamId, in, inlen); +} +/******************************************************************************/ +/* https://http2.github.io/http2-spec/#PUSH_PROMISE + * +---------------+ + * |Pad Length? (8)| + * +-+-------------+-----------------------------------------------+ + * |R| Promised Stream ID (31) | + * +-+-----------------------------+-------------------------------+ + * | Header Block Fragment (*) ... + * +---------------------------------------------------------------+ + * | Padding (*) ... + * +---------------------------------------------------------------+ + */ +LOCAL void http2_parse_frame_push_promise(MolochSession_t *session, HTTP2Info_t *http2, int which, uint8_t flags, uint32_t streamId, unsigned char *in, int inlen) +{ + if (flags & NGHTTP2_FLAG_PADDED) { + uint8_t padding = in[0]; + in++; + inlen -= (1 + padding); + } + + // Promised Stream ID + in += 4; + inlen -= 4; + + if (inlen < 0) + return; + + http2_parse_header_block(session, http2, which, flags, streamId, in, inlen); +} +/******************************************************************************/ +/* https://http2.github.io/http2-spec/#DATA + * +---------------+ + * |Pad Length? (8)| + * +---------------+-----------------------------------------------+ + * | Data (*) ... + * +---------------------------------------------------------------+ + * | Padding (*) ... + * +---------------------------------------------------------------+ + */ +LOCAL void http2_parse_frame_data(MolochSession_t *session, HTTP2Info_t *http2, int which, uint8_t flags, uint32_t streamId, const unsigned char *in, int inlen, int initial) +{ + // If first packet check for padding/end and save it for when dataneeded is 0 + if (initial) { + if (flags & NGHTTP2_FLAG_PADDED) { + http2->dataPadding[which] = in[0]; + in++; + inlen--; + } else { + http2->dataPadding[which] = 0; + } + http2->isEnd[which] = (flags & NGHTTP2_FLAG_END_STREAM) != 0; + } + + // If last packet in frame subtract saved padding + if (http2->dataNeeded[which] == 0) { + inlen -= http2->dataPadding[which]; + } + + if (inlen < 0) + return; + + int spos = http2_spos_get(http2, streamId, FALSE); + + // Only get magic string on first frame + if (initial) { + http2->streams[spos].magicString[which] = moloch_parsers_magic(session, magicField, (char *)in, inlen); + } + + // Check if checksums are allocated and update with new data + if (!http2->streams[spos].checksum[which]) { + http2->streams[spos].checksum[which] = g_checksum_new(G_CHECKSUM_MD5); + if (config.supportSha256) { + http2->streams[spos].checksum[which+2] = g_checksum_new(G_CHECKSUM_SHA256); + } + } + + g_checksum_update(http2->streams[spos].checksum[which], (guchar *)in, inlen); + if (config.supportSha256) { + g_checksum_update(http2->streams[spos].checksum[which+2], (guchar *)in, inlen); + } + + // If the first packet in the frame said this is end and we've read them all, set the md5/sha fields + if (http2->isEnd[which] && http2->dataNeeded[which] == 0) { + const char *md5 = g_checksum_get_string(http2->streams[spos].checksum[which]); + moloch_field_string_uw_add(md5Field, session, (char*)md5, 32, (gpointer)http2->streams[spos].magicString[which], TRUE); + g_checksum_reset(http2->streams[spos].checksum[which]); + if (config.supportSha256) { + const char *sha256 = g_checksum_get_string(http2->streams[spos].checksum[which+2]); + moloch_field_string_uw_add(sha256Field, session, (char*)sha256, 64, (gpointer)http2->streams[spos].magicString[which], TRUE); + g_checksum_reset(http2->streams[spos].checksum[which+2]); + } + } +} +/******************************************************************************/ +LOCAL int http2_parse_frame(MolochSession_t *session, HTTP2Info_t *http2, int which) +{ + BSB bsb; + BSB_INIT(bsb, http2->data[which], http2->used[which]); + + uint32_t len = 0; + nghttp2_frame_type type = 0; + uint8_t flags = 0; + uint32_t streamId = 0; + + BSB_IMPORT_u24(bsb, len); + BSB_IMPORT_u08(bsb, type); + BSB_IMPORT_u08(bsb, flags); + BSB_IMPORT_u32(bsb, streamId); + + if (BSB_IS_ERROR(bsb)) + goto cleanup; + + // type will only be DATA if this is the first part of a data frame, anything else will be shortcutted in http_parse + if (type == NGHTTP2_DATA) { + http2->dataStreamId[which] = streamId; + if (len > BSB_REMAINING(bsb)) { + http2->used[which] = 0; + http2->dataNeeded[which] = len - BSB_REMAINING(bsb); + } else { + http2->dataNeeded[which] = 0; + } + http2_parse_frame_data(session, http2, which, flags, streamId, BSB_WORK_PTR(bsb), BSB_REMAINING(bsb), TRUE); + + // Don't need to memmove below + if (http2->dataNeeded[which] != 0) + return 0; + } + + if (len > BSB_REMAINING(bsb) && type != NGHTTP2_DATA) { + return 1; + } + + if (type == NGHTTP2_CONTINUATION) { + type = http2->lastType[which]; + } + +#ifdef HTTPDEBUG + LOG("which: %d len: %u, type: %u (%s), flags: 0x%x, streamId: %u", which, len, type, http2_frameNames[type], flags, streamId); +#endif + switch(type) { + case NGHTTP2_HEADERS: + http2_parse_frame_headers(session, http2, which, flags, streamId, BSB_WORK_PTR(bsb), len); + break; + case NGHTTP2_PUSH_PROMISE: + http2_parse_frame_push_promise(session, http2, which, flags, streamId, BSB_WORK_PTR(bsb), len); + break; + case NGHTTP2_RST_STREAM: + http2_spos_free(http2, streamId); + break; + default: + break; + } + http2->lastType[which] = type; + + if (flags & NGHTTP2_FLAG_END_STREAM) { + int spos = http2_spos_get(http2, streamId, FALSE); + if (spos != -1) { + http2->streams[spos].ended |= (1 << which); + if (http2->streams[spos].ended == 0x3) { + http2_spos_free(http2, streamId); + } + } + } + +cleanup: + http2->used[which] -= (9 + len); + memmove(http2->data[which], http2->data[which] + 9 + len, http2->used[which]); + + return 0; +} +/******************************************************************************/ +LOCAL int http2_parse(MolochSession_t *session, void *uw, const unsigned char *data, int len, int which) +{ + HTTP2Info_t *http2 = uw; + +#ifdef HTTPDEBUG + LOG("HTTPDEBUG which: %d used: %d len: %d", which, http2->used[which], len); +#endif + + + if (http2->dataNeeded[which] > 0) { + int used = MIN(http2->dataNeeded[which], len); + http2->dataNeeded[which] -= used; + http2_parse_frame_data(session, http2, which, 0, http2->dataStreamId[which], data, used, FALSE); + if (used == len) + return 0; + + data += used; + len -= used; + } + + if (len > MAX_HTTP2_SIZE - http2->used[which]) { +#ifdef HTTPDEBUG + moloch_print_hex_string(http2->data[which], http2->used[which]); + LOG("TOO MUCH DATA"); +#endif + return MOLOCH_PARSER_UNREGISTER; + } + memcpy(http2->data[which] + http2->used[which], data, len); + http2->used[which] += len; + + if (http2->used[which] > 24 && http2->data[which][0] == 'P' && memcmp(http2->data[which], "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n", 24) == 0) { + http2->used[which] -= 24; + memmove(http2->data[which], http2->data[which] + 24, http2->used[which]); + } + + while (http2->used[which] >= 9) { + if (http2_parse_frame(session, http2, which)) + break; + } + + return 0; +} +/******************************************************************************/ +void http2_save(MolochSession_t UNUSED(*session), void *UNUSED(uw), int final) +{ + if (!final) + return; + +// HTTP2Info_t *http2 = uw; + +#ifdef HTTPDEBUG + LOG("Save callback %d", final); +#endif +} +/******************************************************************************/ +LOCAL void http2_free(MolochSession_t UNUSED(*session), void *uw) +{ + HTTP2Info_t *http2 = uw; + + if (http2->hd_inflater[0]) { + nghttp2_hd_inflate_del(http2->hd_inflater[0]); + } + if (http2->hd_inflater[1]) { + nghttp2_hd_inflate_del(http2->hd_inflater[1]); + } + for (int i = 0; i < http2->numStreams; i++) { + g_checksum_free(http2->streams[i].checksum[0]); + g_checksum_free(http2->streams[i].checksum[1]); + if (config.supportSha256) { + g_checksum_free(http2->streams[i].checksum[2]); + g_checksum_free(http2->streams[i].checksum[3]); + } + } + MOLOCH_TYPE_FREE(HTTP2Info_t, http2); +} +/******************************************************************************/ +LOCAL void http2_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int which, void *UNUSED(uw)) +{ + if (moloch_session_has_protocol(session, "http2")) + return; + moloch_session_add_protocol(session, "http2"); + + HTTP2Info_t *http2 = MOLOCH_TYPE_ALLOC0(HTTP2Info_t); + http2->which = which; + + moloch_parsers_register2(session, http2_parse, http2, http2_free, http2_save); +} +/******************************************************************************/ +void moloch_parser_init() +{ + moloch_parsers_classifier_register_tcp("http2", NULL, 0, (unsigned char *)"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n", 24, http2_classify); + + methodField = moloch_field_define("http", "termfield", + "http.method", "Request Method", "http.method", + "HTTP Request Method", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + (char *)NULL); + statuscodeField = moloch_field_define("http", "integer", + "http.statuscode", "Status Code", "http.statuscode", + "Response HTTP numeric status code", + MOLOCH_FIELD_TYPE_INT_GHASH, MOLOCH_FIELD_FLAG_CNT, + (char *)NULL); + hostField = moloch_field_define("http", "lotermfield", + "host.http", "Hostname", "http.host", + "HTTP host header field", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + "aliases", "[\"http.host\"]", + "category", "host", + (char *)NULL); + magicField = moloch_field_define("http", "termfield", + "http.bodymagic", "Body Magic", "http.bodyMagic", + "The content type of body determined by libfile/magic", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + (char *)NULL); + md5Field = moloch_field_define("http", "lotermfield", + "http.md5", "Body MD5", "http.md5", + "MD5 of http body response", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + "category", "md5", + (char *)NULL); + + if (config.supportSha256) { + sha256Field = moloch_field_define("http", "lotermfield", + "http.sha256", "Body SHA256", "http.sha256", + "SHA256 of http body response", + MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT, + "category", "sha256", + (char *)NULL); + } +} diff --git a/capture/parsers/tls-cipher.h b/capture/parsers/tls-cipher.h index 18182fa0cb..bc4e367d43 100644 --- a/capture/parsers/tls-cipher.h +++ b/capture/parsers/tls-cipher.h @@ -4132,10 +4132,10 @@ static char *ciphers_c1[256] = { "TLS_GOSTR341112_256_WITH_KUZNYECHIK_CTR_OMAC", "TLS_GOSTR341112_256_WITH_MAGMA_CTR_OMAC", "TLS_GOSTR341112_256_WITH_28147_CNT_IMIT", -NULL, -NULL, -NULL, -NULL, +"TLS_GOSTR341112_256_WITH_KUZNYECHIK_MGM_L", +"TLS_GOSTR341112_256_WITH_MAGMA_MGM_L", +"TLS_GOSTR341112_256_WITH_KUZNYECHIK_MGM_S", +"TLS_GOSTR341112_256_WITH_MAGMA_MGM_S", NULL, NULL, NULL, diff --git a/capture/parsers/tls.c b/capture/parsers/tls.c index bab3037842..8bc2f343ad 100644 --- a/capture/parsers/tls.c +++ b/capture/parsers/tls.c @@ -311,6 +311,12 @@ LOCAL void tls_process_server_hello(MolochSession_t *session, const unsigned cha } } + if (etype == 0x10) { // etype 0x10 is alpn + if (elen == 5 && BSB_REMAINING(ebsb) >= 5 && memcmp(BSB_WORK_PTR(ebsb), "\x00\x03\x02\x68\x32", 5) == 0) { + moloch_session_add_protocol(session, "http2"); + } + } + BSB_IMPORT_skip (ebsb, elen); } BSB_EXPORT_rewind(eja3bsb, 1); // Remove last - diff --git a/configure.ac b/configure.ac index 110a3bc09a..b0d5f11700 100644 --- a/configure.ac +++ b/configure.ac @@ -43,7 +43,7 @@ darwin*) ;; *) SHARED_FLAGS="--shared" - UNDEFINED_FLAGS="-u g_checksum_update -u g_hmac_update -u g_uri_unescape_segment" + UNDEFINED_FLAGS="-u g_checksum_update -u g_hmac_update -u g_uri_unescape_segment -u nghttp2_hd_inflate_hd2" esac AC_SUBST(SHARED_FLAGS) AC_SUBST(UNDEFINED_FLAGS) @@ -472,9 +472,40 @@ AC_ARG_WITH(magic, fi ;; esac ], [ - AC_CHECK_LIB(magic, main,MAGIC_LIBS=-lmagic,AC_MSG_ERROR(please install magic library)) + AC_CHECK_LIB(magic,main,MAGIC_LIBS=-lmagic,AC_MSG_ERROR(please install magic library)) AC_MSG_RESULT(yes) ]) AC_SUBST(MAGIC_CFLAGS) AC_SUBST(MAGIC_LIBS) +dnl Checks for nghttp2 +AC_MSG_CHECKING(for nghttp2) +AC_ARG_WITH(nghttp2, +[ --with-nghttp2=DIR use nghttp2 build directory], +[ case "$withval" in + yes) + AC_CHECK_LIB(nghttp2,main,NGHTTP2_LIBS=-lnghttp2,AC_MSG_ERROR(please install nghttp2 library)) + ;; + no) + AC_MSG_RESULT(no) + ;; + *) + AC_MSG_RESULT($withval) + if test -f $withval/lib/includes/nghttp2/nghttp2.h -a -f $withval/lib/.libs/libnghttp2.a; then + owd=`pwd` + if cd $withval; then + withval=`pwd`; + cd $owd; + fi + NGHTTP2_CFLAGS="-I$withval/lib/includes" + NGHTTP2_LIBS="$withval/lib/.libs/libnghttp2.a" + else + AC_ERROR(nghttp2.h or libnghttp2.a not found in $withval) + fi + ;; +esac ], [ + AC_CHECK_LIB(nghttp2, main,NGHTTP2_LIBS=-lnghttp2,AC_MSG_ERROR(please install nghttp2 library)) +AC_MSG_RESULT(yes) ]) +AC_SUBST(NGHTTP2_CFLAGS) +AC_SUBST(NGHTTP2_LIBS) + AC_OUTPUT diff --git a/easybutton-build.sh b/easybutton-build.sh index d958ca3519..be6214d7fc 100755 --- a/easybutton-build.sh +++ b/easybutton-build.sh @@ -15,10 +15,11 @@ GLIB=2.64.3 YARA=3.11.0 MAXMIND=1.3.2 PCAP=1.9.1 -CURL=7.68.0 +CURL=7.71.1 LUA=5.3.5 DAQ=2.0.7 NODE=12.18.2 +NGHTTP2=1.41.0 TDIR="/data/moloch" DOPFRING=0 @@ -122,17 +123,17 @@ fi if [ "$UNAME" = "Darwin" ]; then if [ -x "/opt/local/bin/port" ]; then - sudo port install libpcap yara glib2 jansson ossp-uuid libmaxminddb libmagic pcre lua libyaml wget + sudo port install libpcap yara glib2 jansson ossp-uuid libmaxminddb libmagic pcre lua libyaml wget libnghttp2 echo "MOLOCH: Building capture" - echo './configure --with-libpcap=/opt/local --with-yara=/opt/local LDFLAGS="-L/opt/local/lib" --with-glib2=no GLIB2_CFLAGS="-I/opt/local/include/glib-2.0 -I/opt/local/lib/glib-2.0/include" GLIB2_LIBS="-L/opt/local/lib -lglib-2.0 -lgmodule-2.0 -lgobject-2.0 -lgio-2.0" --with-pfring=no --with-curl=yes --with-lua=no LUA_CFLAGS="-I/opt/local/include" LUA_LIBS="-L/opt/local/lib -llua"' - ./configure --with-libpcap=/opt/local --with-yara=/opt/local LDFLAGS="-L/opt/local/lib" --with-glib2=no GLIB2_CFLAGS="-I/opt/local/include/glib-2.0 -I/opt/local/lib/glib-2.0/include" GLIB2_LIBS="-L/opt/local/lib -lglib-2.0 -lgmodule-2.0 -lgobject-2.0 -lgio-2.0" --with-pfring=no --with-curl=yes --with-lua=no LUA_CFLAGS="-I/opt/local/include" LUA_LIBS="-L/opt/local/lib -llua" + echo './configure --with-libpcap=/opt/local --with-yara=/opt/local LDFLAGS="-L/opt/local/lib" --with-glib2=no GLIB2_CFLAGS="-I/opt/local/include/glib-2.0 -I/opt/local/lib/glib-2.0/include" GLIB2_LIBS="-L/opt/local/lib -lglib-2.0 -lgmodule-2.0 -lgobject-2.0 -lgio-2.0" --with-pfring=no --with-curl=yes --with-nghttp2=yes --with-lua=no LUA_CFLAGS="-I/opt/local/include" LUA_LIBS="-L/opt/local/lib -llua"' + ./configure --with-libpcap=/opt/local --with-yara=/opt/local LDFLAGS="-L/opt/local/lib" --with-glib2=no GLIB2_CFLAGS="-I/opt/local/include/glib-2.0 -I/opt/local/lib/glib-2.0/include" GLIB2_LIBS="-L/opt/local/lib -lglib-2.0 -lgmodule-2.0 -lgobject-2.0 -lgio-2.0" --with-pfring=no --with-curl=yes --with-nghttp2=yes --with-lua=no LUA_CFLAGS="-I/opt/local/include" LUA_LIBS="-L/opt/local/lib -llua" elif [ -x "/usr/local/bin/brew" ]; then - brew install libpcap yara glib jansson ossp-uuid libmaxminddb libmagic pcre lua libyaml openssl wget autoconf automake + brew install libpcap yara glib jansson ossp-uuid libmaxminddb libmagic pcre lua libyaml openssl wget autoconf automake libnghttp2 echo "MOLOCH: Building capture" - echo './configure --with-libpcap=/usr/local/opt/libpcap --with-yara=/usr/local LDFLAGS="-L/usr/local/lib" --with-glib2=no GLIB2_CFLAGS="-I/usr/local/include/glib-2.0 -I/usr/local/lib/glib-2.0/include -I/usr/local/opt/openssl@1.1/include" GLIB2_LIBS="-L/usr/local/lib -lglib-2.0 -lgmodule-2.0 -lgobject-2.0 -lgio-2.0 -L/usr/local/opt/openssl@1.1/lib" --with-pfring=no --with-curl=yes --with-lua=no LUA_CFLAGS="-I/usr/local/include/lua" LUA_LIBS="-L/usr/local/lib -llua' - ./configure --with-libpcap=/usr/local/opt/libpcap --with-yara=/usr/local LDFLAGS="-L/usr/local/lib" --with-glib2=no GLIB2_CFLAGS="-I/usr/local/include/glib-2.0 -I/usr/local/lib/glib-2.0/include -I/usr/local/opt/openssl@1.1/include" GLIB2_LIBS="-L/usr/local/lib -lglib-2.0 -lgmodule-2.0 -lgobject-2.0 -lgio-2.0 -L/usr/local/opt/openssl@1.1/lib" --with-pfring=no --with-curl=yes --with-lua=no LUA_CFLAGS="-I/usr/local/include/lua" LUA_LIBS="-L/usr/local/lib -llua" + echo './configure --with-libpcap=/usr/local/opt/libpcap --with-yara=/usr/local LDFLAGS="-L/usr/local/lib" --with-glib2=no GLIB2_CFLAGS="-I/usr/local/include/glib-2.0 -I/usr/local/lib/glib-2.0/include -I/usr/local/opt/openssl@1.1/include" GLIB2_LIBS="-L/usr/local/lib -lglib-2.0 -lgmodule-2.0 -lgobject-2.0 -lgio-2.0 -L/usr/local/opt/openssl@1.1/lib" --with-pfring=no --with-curl=yes --with-nghttp2=yes --with-lua=no LUA_CFLAGS="-I/usr/local/include/lua" LUA_LIBS="-L/usr/local/lib -llua' + ./configure --with-libpcap=/usr/local/opt/libpcap --with-yara=/usr/local LDFLAGS="-L/usr/local/lib" --with-glib2=no GLIB2_CFLAGS="-I/usr/local/include/glib-2.0 -I/usr/local/lib/glib-2.0/include -I/usr/local/opt/openssl@1.1/include" GLIB2_LIBS="-L/usr/local/lib -lglib-2.0 -lgmodule-2.0 -lgobject-2.0 -lgio-2.0 -L/usr/local/opt/openssl@1.1/lib" --with-pfring=no --with-curl=yes --with-nghttp2=yes --with-lua=no LUA_CFLAGS="-I/usr/local/include/lua" LUA_LIBS="-L/usr/local/lib -llua" else echo "MOLOCH: Please install MacPorts or Homebrew" @@ -241,6 +242,22 @@ else echo "MOLOCH: Not rebuilding curl" fi + # nghttp2 + if [ ! -f "nghttp2-$NGHTTP2.tar.gz" ]; then + wget https://github.com/nghttp2/nghttp2/releases/download/v$NGHTTP2/nghttp2-$NGHTTP2.tar.gz + fi + + if [ ! -f "nghttp2-$NGHTTP2/lib/.libs/libcurl.a" ]; then + tar zxf nghttp2-$NGHTTP2.tar.gz + ( cd nghttp2-$NGHTTP2; ./configure --enable-lib-only; $MAKE) + if [ $? -ne 0 ]; then + echo "MOLOCH: $MAKE failed" + exit 1 + fi + else + echo "MOLOCH: Not rebuilding nghttp2" + fi + # lua if [ ! -f "lua-$LUA.tar.gz" ]; then wget https://www.lua.org/ftp/lua-$LUA.tar.gz @@ -279,8 +296,8 @@ else # Now build moloch echo "MOLOCH: Building capture" cd .. - echo "./configure --prefix=$TDIR $PCAPBUILD --with-yara=thirdparty/yara/yara-$YARA --with-maxminddb=thirdparty/libmaxminddb-$MAXMIND $WITHGLIB --with-curl=thirdparty/curl-$CURL --with-lua=thirdparty/lua-$LUA" - ./configure --prefix=$TDIR $PCAPBUILD --with-yara=thirdparty/yara/yara-$YARA --with-maxminddb=thirdparty/libmaxminddb-$MAXMIND $WITHGLIB --with-curl=thirdparty/curl-$CURL --with-lua=thirdparty/lua-$LUA + echo "./configure --prefix=$TDIR $PCAPBUILD --with-yara=thirdparty/yara/yara-$YARA --with-maxminddb=thirdparty/libmaxminddb-$MAXMIND $WITHGLIB --with-curl=thirdparty/curl-$CURL --with-nghttp2=thirdparty/nghttp2-$NGHTTP2 --with-lua=thirdparty/lua-$LUA" + ./configure --prefix=$TDIR $PCAPBUILD --with-yara=thirdparty/yara/yara-$YARA --with-maxminddb=thirdparty/libmaxminddb-$MAXMIND $WITHGLIB --with-curl=thirdparty/curl-$CURL --with-nghttp2=thirdparty/nghttp2-$NGHTTP2 --with-lua=thirdparty/lua-$LUA fi if [ $DOCLEAN -eq 1 ]; then diff --git a/screwdriver.yaml b/screwdriver.yaml index d0ad949124..14164a78ec 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -234,7 +234,7 @@ jobs: - build-package: | export MOLOCH_VERSION=`sed 's/.*"\(.*\)\".*$/\1/' /data/moloch/viewer/version.js | tr "-" "_"` if [ "$GIT_BRANCH" = "$MOLOCH_COPY_BRANCH" ]; then - scl enable rh-ruby23 'fpm -s dir -t rpm -n moloch -x data/moloch/logs -x data/moloch/raw -v $MOLOCH_VERSION --iteration $SD_BUILD_ID --template-scripts --after-install "release/afterinstall.sh" --url "https://molo.ch" --description "Moloch Full Packet System" -d perl-libwww-perl -d perl-JSON -d ethtool -d libyaml-devel -d libasan-static -d perl-LWP-Protocol-https /data/moloch' + scl enable rh-ruby23 'fpm -s dir -t rpm -n moloch -x data/moloch/logs -x data/moloch/raw -v $MOLOCH_VERSION --iteration $SD_BUILD_ID --template-scripts --after-install "release/afterinstall.sh" --url "https://molo.ch" --description "Moloch Full Packet System" -d perl-libwww-perl -d perl-JSON -d ethtool -d libyaml-devel -d libasan-static -d libubsan1 -d libasan5 -d perl-LWP-Protocol-https /data/moloch' scl enable devtoolset-9 rh-python36 'aws s3 cp --quiet moloch*.x86_64.rpm s3://files.molo.ch/sanitize/moloch-${MOLOCH_FILE_NAME}.centos7.x86_64.rpm --acl public-read' scl enable devtoolset-9 rh-python36 'aws s3api put-object-acl --bucket files.molo.ch --key sanitize/moloch-${MOLOCH_FILE_NAME}.centos7.x86_64.rpm --acl public-read' fi diff --git a/tests/api-scrub.t b/tests/api-scrub.t index e1fdd82185..429d376582 100644 --- a/tests/api-scrub.t +++ b/tests/api-scrub.t @@ -14,7 +14,7 @@ countTest(0, "date=-1&expression=" . uri_escape("file=$copytest")); system("../db/db.pl --prefix tests $MolochTest::elasticsearch rm $copytest 2>&1 1>/dev/null"); viewerPost("/flushCache"); system("/bin/cp pcap/socks-http-example.pcap copytest.pcap"); -system("../capture/moloch-capture -c config.test.ini -n test -r copytest.pcap --host localhost"); +system("../capture/moloch-capture -c config.test.ini -n test -r copytest.pcap"); esGet("/_flush"); esGet("/_refresh"); diff --git a/tests/pcap/http2-nghttp2.pcap b/tests/pcap/http2-nghttp2.pcap new file mode 100644 index 0000000000..88a62e9402 Binary files /dev/null and b/tests/pcap/http2-nghttp2.pcap differ diff --git a/tests/pcap/http2-nghttp2.test b/tests/pcap/http2-nghttp2.test new file mode 100644 index 0000000000..25ceeca9fc --- /dev/null +++ b/tests/pcap/http2-nghttp2.test @@ -0,0 +1,411 @@ +{ + "sessions2" : [ + { + "body" : { + "communityId" : "1:t8w2y51bTGSv9Z4iWI9ZJSV2gxg=", + "dstASN" : "AS63949", + "dstBytes" : 48832, + "dstDataBytes" : 46136, + "dstGEO" : "JP", + "dstIp" : "139.162.123.134", + "dstPackets" : 48, + "dstPayload8" : "0000180400000000", + "dstPort" : 80, + "dstRIR" : "ARIN", + "fileId" : [], + "firstPacket" : 1593995681271, + "http" : { + "host" : [ + "nghttp2.org" + ], + "hostCnt" : 1, + "md5" : [ + "4b806c16dac801940f7075fd9d9b7bae", + "4bf2f259ee1245e1b0cbe6f0127228b5" + ], + "md5Cnt" : 2, + "method" : [ + "GET" + ], + "methodCnt" : 1, + "path" : [ + "/", + "/stylesheets/screen.css" + ], + "pathCnt" : 2, + "requestHeader" : [ + "accept", + "accept-encoding", + "continuation-test-1", + "continuation-test-2", + "continuation-test-3", + "continuation-test-4", + "continuation-test-5", + "continuation-test-6", + "user-agent" + ], + "requestHeaderCnt" : 9, + "requestHeaderField" : [ + "accept", + "accept-encoding", + "continuation-test-1", + "continuation-test-2", + "continuation-test-3", + "continuation-test-4", + "continuation-test-5", + "continuation-test-6", + "user-agent" + ], + "requestHeaderValue" : [ + "*/*", + "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------", + "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------", + "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------", + "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------", + "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------", + "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------", + "gzip, deflate", + "nghttp2/1.41.0" + ], + "requestHeaderValueCnt" : 9, + "responseHeader" : [ + "accept-encoding", + "accept-ranges", + "alt-svc", + "content-length", + "content-type", + "date", + "etag", + "last-modified", + "server", + "user-agent", + "via", + "x-backend-header-rtt", + "x-content-type-options", + "x-frame-options", + "x-http2-push", + "x-xss-protection" + ], + "responseHeaderCnt" : 16, + "responseHeaderField" : [ + "accept-encoding", + "accept-ranges", + "accept-ranges", + "alt-svc", + "alt-svc", + "content-length", + "content-length", + "content-type", + "content-type", + "date", + "date", + "etag", + "etag", + "last-modified", + "last-modified", + "server", + "server", + "user-agent", + "via", + "via", + "x-backend-header-rtt", + "x-backend-header-rtt", + "x-content-type-options", + "x-content-type-options", + "x-frame-options", + "x-frame-options", + "x-http2-push", + "x-xss-protection", + "x-xss-protection" + ], + "responseHeaderValue" : [ + "\"5ed644c7-19d8\"", + "\"5ed644c7-98aa\"", + "0.001873", + "0.001992", + "1", + "1; mode=block", + "1; mode=block", + "2 nghttpx", + "2 nghttpx", + "39082", + "6616", + "bytes", + "bytes", + "gzip, deflate", + "h3-28=\":4433\"; ma=3600", + "h3-28=\":4433\"; ma=3600", + "mon, 06 jul 2020 00:34:42 gmt", + "mon, 06 jul 2020 00:34:42 gmt", + "nghttp2/1.41.0", + "nghttpx", + "nghttpx", + "nosniff", + "nosniff", + "sameorigin", + "sameorigin", + "text/css", + "text/html", + "tue, 02 jun 2020 12:23:35 gmt", + "tue, 02 jun 2020 12:23:35 gmt" + ], + "responseHeaderValueCnt" : 29, + "sha256" : [ + "80d4404b834c034238ac2e3caeb3a6f24dd093432fd13c5f0e64da9b0d9334bb", + "cfd844f4f94b56e0d93d6ad47530ba9030cc8d8e41ad6ae53b30fe930efa774f" + ], + "sha256Cnt" : 2, + "statuscode" : [ + 200 + ], + "statuscodeCnt" : 1 + }, + "initRTT" : 105, + "ipProtocol" : 6, + "lastPacket" : 1593995683015, + "length" : 1745, + "node" : "test", + "packetLen" : [ + 84, + 80, + 72, + 1426, + 1426, + 1426, + 72, + 72, + 1426, + 114, + 1426, + 1426, + 1426, + 1426, + 1426, + 390, + 72, + 1426, + 72, + 72, + 1426, + 1426, + 1426, + 870, + 72, + 128, + 72, + 1426, + 1426, + 72, + 1426, + 1426, + 72, + 1426, + 1426, + 72, + 1426, + 1426, + 72, + 1426, + 72, + 1426, + 72, + 72, + 1426, + 1426, + 72, + 72, + 1426, + 1426, + 72, + 72, + 1426, + 1426, + 72, + 72, + 1426, + 1426, + 72, + 72, + 1426, + 1426, + 72, + 72, + 1426, + 72, + 1426, + 72, + 1426, + 72, + 1426, + 72, + 1426, + 72, + 1426, + 72, + 85, + 1426, + 1426, + 72, + 1426, + 1426, + 72, + 72, + 85, + 1426, + 1426, + 72, + 1426, + 1426, + 72, + 74, + 72, + 72, + 89, + 72, + 72, + 72, + 72, + 72, + 72 + ], + "packetPos" : [ + 24, + 108, + 188, + 260, + 1686, + 3112, + 4538, + 4610, + 4682, + 6108, + 6222, + 7648, + 9074, + 10500, + 11926, + 13352, + 13742, + 13814, + 15240, + 15312, + 15384, + 16810, + 18236, + 19662, + 20532, + 20604, + 20732, + 20804, + 22230, + 23656, + 23728, + 25154, + 26580, + 26652, + 28078, + 29504, + 29576, + 31002, + 32428, + 32500, + 33926, + 33998, + 35424, + 35496, + 35568, + 36994, + 38420, + 38492, + 38564, + 39990, + 41416, + 41488, + 41560, + 42986, + 44412, + 44484, + 44556, + 45982, + 47408, + 47480, + 47552, + 48978, + 50404, + 50476, + 50548, + 51974, + 52046, + 53472, + 53544, + 54970, + 55042, + 56468, + 56540, + 57966, + 58038, + 59464, + 59536, + 59621, + 61047, + 62473, + 62545, + 63971, + 65397, + 65469, + 65541, + 65626, + 67052, + 68478, + 68550, + 69976, + 71402, + 71474, + 71548, + 71620, + 71692, + 71781, + 71853, + 71925, + 71997, + 72069, + 72141 + ], + "protocol" : [ + "http2", + "tcp" + ], + "protocolCnt" : 2, + "segmentCnt" : 1, + "srcBytes" : 21741, + "srcDataBytes" : 18761, + "srcIp" : "10.89.85.15", + "srcPackets" : 53, + "srcPayload8" : "505249202a204854", + "srcPort" : 49570, + "tcpflags" : { + "ack" : 73, + "dstZero" : 0, + "fin" : 2, + "psh" : 24, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "totBytes" : 70573, + "totDataBytes" : 64897, + "totPackets" : 101 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-200706", + "_type" : "session" + } + } + } + ] +} + diff --git a/tests/pcap/tls-alpn-h2.pcap b/tests/pcap/tls-alpn-h2.pcap new file mode 100644 index 0000000000..74bf35acb5 Binary files /dev/null and b/tests/pcap/tls-alpn-h2.pcap differ diff --git a/tests/pcap/tls-alpn-h2.test b/tests/pcap/tls-alpn-h2.test new file mode 100644 index 0000000000..925126f6f1 --- /dev/null +++ b/tests/pcap/tls-alpn-h2.test @@ -0,0 +1,170 @@ +{ + "sessions2" : [ + { + "body" : { + "cert" : [ + { + "alt" : [ + "cloudflare.com", + "www.cloudflare.com" + ], + "altCnt" : 2, + "curve" : "prime256v1", + "hash" : "6a:4c:b2:49:b7:1b:66:59:da:bb:96:01:15:fd:7a:01:bf:c9:96:8c", + "issuerCN" : [ + "digicert ecc extended validation server ca" + ], + "issuerON" : [ + "DigiCert Inc" + ], + "notAfter" : 1604404800000, + "notBefore" : 1540857600000, + "publicAlgorithm" : "id-ecPublicKey", + "remainingDays" : 1, + "serial" : "0a68bb984a507399f4716e809a44a7b0", + "subjectCN" : [ + "cloudflare.com" + ], + "subjectON" : [ + "Cloudflare, Inc." + ], + "validDays" : 735 + }, + { + "curve" : "secp384r1", + "hash" : "fc:20:b6:0c:ad:ac:38:d1:ff:40:cf:1b:6e:ef:29:0f:10:b5:7a:bd", + "issuerCN" : [ + "digicert high assurance ev root ca" + ], + "issuerON" : [ + "DigiCert Inc" + ], + "notAfter" : 1939812867000, + "notBefore" : 1466513667000, + "publicAlgorithm" : "id-ecPublicKey", + "remainingDays" : 1, + "serial" : "029707560cd4a9ebbfe272f1e096d882", + "subjectCN" : [ + "digicert ecc extended validation server ca" + ], + "subjectON" : [ + "DigiCert Inc" + ], + "validDays" : 5478 + } + ], + "certCnt" : 2, + "communityId" : "1:/wxqZ626SZRx3CanjPRZ6tl4iH8=", + "dstASN" : "AS13335", + "dstBytes" : 3142, + "dstDataBytes" : 2810, + "dstGEO" : "US", + "dstIp" : "2606:4700:0000:0000:0000:0000:6811:d209", + "dstPackets" : 5, + "dstPayload8" : "1603030064020000", + "dstPort" : 443, + "fileId" : [], + "firstPacket" : 1593980608621, + "http" : { + "host" : [ + "www.cloudflare.com" + ], + "hostCnt" : 1 + }, + "initRTT" : 18, + "ipProtocol" : 6, + "lastPacket" : 1593980608696, + "length" : 75, + "node" : "test", + "packetLen" : [ + 104, + 92, + 80, + 317, + 80, + 1426, + 1426, + 80, + 198 + ], + "packetPos" : [ + 24, + 128, + 220, + 300, + 617, + 697, + 2123, + 3549, + 3629 + ], + "protocol" : [ + "http2", + "tcp", + "tls" + ], + "protocolCnt" : 3, + "segmentCnt" : 1, + "srcASN" : "AS10310", + "srcBytes" : 517, + "srcDataBytes" : 237, + "srcGEO" : "US", + "srcIp" : "2001:4998:ef83:0014:8000:0000:0000:100d", + "srcPackets" : 4, + "srcPayload8" : "16030100e8010000", + "srcPort" : 64034, + "tcpflags" : { + "ack" : 5, + "dstZero" : 0, + "fin" : 0, + "psh" : 2, + "rst" : 0, + "srcZero" : 0, + "syn" : 1, + "syn-ack" : 1, + "urg" : 0 + }, + "timestamp" : "SET", + "tls" : { + "cipher" : [ + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256" + ], + "cipherCnt" : 1, + "dstSessionId" : [ + "b89ff1ab695278c69b8a73f76242ef755e0b13dc6d459aaaa784fec9c2dfce34" + ], + "ja3" : [ + "3faa4ad39f690c4ef1c3160caa375465" + ], + "ja3Cnt" : 1, + "ja3s" : [ + "7f76f3e952a1bc7d407366fafe1db7ed" + ], + "ja3sCnt" : 1, + "ja3sstring" : [ + "771,52393,0-65281-11-16" + ], + "ja3sstringCnt" : 1, + "ja3string" : [ + "771,49200-49196-49192-49188-49172-49162-159-107-57-52393-52392-52394-65413-196-136-129-157-61-53-192-132-49199-49195-49191-49187-49171-49161-158-103-51-190-69-156-60-47-186-65-49169-49159-5-4-49170-49160-22-10-255,0-11-10-13-16,29-23-24,0" + ], + "ja3stringCnt" : 1, + "version" : [ + "TLSv1.2" + ], + "versionCnt" : 1 + }, + "totBytes" : 3659, + "totDataBytes" : 3047, + "totPackets" : 9 + }, + "header" : { + "index" : { + "_index" : "tests_sessions2-200705", + "_type" : "session" + } + } + } + ] +} +