From 6d04465e8c98c57a17428bf7aa54cc9e0add30ff Mon Sep 17 00:00:00 2001 From: Paolo Insogna Date: Mon, 31 Jul 2023 15:34:36 +0200 Subject: [PATCH] feat: Drop strict mode in favor of leniency flags. (#234) * feat: Drop strict mode in favor of leniency flags. * chore: Simplified build script. * fix: Removed leftover. * chore: Updated benchmark script. * chore: Updated docs. --- README.md | 54 +++++++- bench/index.ts | 132 +++++++++---------- bin/generate.ts | 124 +++++------------- package.json | 1 + src/llhttp.ts | 3 - src/llhttp/constants.ts | 19 +-- src/llhttp/http.ts | 106 +++++++--------- src/llhttp/url.ts | 16 +-- src/native/api.c | 32 +++++ src/native/api.h | 61 ++++++++- test/fixtures/extra.c | 30 +++++ test/fixtures/index.ts | 50 +++++--- test/md-test.ts | 109 ++++------------ test/request/connection.md | 10 +- test/request/content-length.md | 50 ++++++++ test/request/invalid.md | 65 ++++++++++ test/request/sample.md | 53 +++++++- test/request/transfer-encoding.md | 204 +++++++++++++++++++++++++++++- test/request/uri.md | 28 +--- test/response/connection.md | 51 +------- test/response/invalid.md | 10 +- test/response/sample.md | 82 ++++-------- test/url.md | 34 +---- 23 files changed, 786 insertions(+), 538 deletions(-) diff --git a/README.md b/README.md index 4d886b57..e982ed0a 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,7 @@ protocol support to highly non-compliant clients/server. No `HPE_INVALID_HEADER_TOKEN` will be raised for incorrect header values when lenient parsing is "on". -**USE AT YOUR OWN RISK!** +**Enabling this flag can pose a security issue since you will be exposed to request smuggling attacks. USE WITH CAUTION!** ### `void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled)` @@ -300,23 +300,22 @@ conjunction with `Content-Length`. This error is important to prevent HTTP request smuggling, but may be less desirable for small number of cases involving legacy servers. -**USE AT YOUR OWN RISK!** +**Enabling this flag can pose a security issue since you will be exposed to request smuggling attacks. USE WITH CAUTION!** ### `void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled)` Enables/disables lenient handling of `Connection: close` and HTTP/1.0 requests responses. -Normally `llhttp` would error on (in strict mode) or discard (in loose mode) -the HTTP request/response after the request/response with `Connection: close` -and `Content-Length`. +Normally `llhttp` would error the HTTP request/response +after the request/response with `Connection: close` and `Content-Length`. This is important to prevent cache poisoning attacks, but might interact badly with outdated and insecure clients. With this flag the extra request/response will be parsed normally. -**USE AT YOUR OWN RISK!** +**Enabling this flag can pose a security issue since you will be exposed to poisoning attacks. USE WITH CAUTION!** ### `void llhttp_set_lenient_transfer_encoding(llhttp_t* parser, int enabled)` @@ -331,7 +330,48 @@ avoid request smuggling. With this flag the extra value will be parsed normally. -**USE AT YOUR OWN RISK!** +**Enabling this flag can pose a security issue since you will be exposed to request smuggling attacks. USE WITH CAUTION!** + +### `void llhttp_set_lenient_version(llhttp_t* parser, int enabled)` + +Enables/disables lenient handling of HTTP version. + +Normally `llhttp` would error when the HTTP version in the request or status line +is not `0.9`, `1.0`, `1.1` or `2.0`. +With this flag the extra value will be parsed normally. + +**Enabling this flag can pose a security issue since you will allow unsupported HTTP versions. USE WITH CAUTION!** + +### `void llhttp_set_lenient_data_after_close(llhttp_t* parser, int enabled)` + +Enables/disables lenient handling of additional data received after a message ends +and keep-alive is disabled. + +Normally `llhttp` would error when additional unexpected data is received if the message +contains the `Connection` header with `close` value. +With this flag the extra data will discarded without throwing an error. + +**Enabling this flag can pose a security issue since you will be exposed to poisoning attacks. USE WITH CAUTION!** + +### `void llhttp_set_lenient_optional_lf_after_cr(llhttp_t* parser, int enabled)` + +Enables/disables lenient handling of incomplete CRLF sequences. + +Normally `llhttp` would error when a CR is not followed by LF when terminating the +request line, the status line, the headers or a chunk header. +With this flag only a CR is required to terminate such sections. + +**Enabling this flag can pose a security issue since you will be exposed to request smuggling attacks. USE WITH CAUTION!** + +### `void llhttp_set_lenient_optional_crlf_after_chunk(llhttp_t* parser, int enabled)` + +Enables/disables lenient handling of chunks not separated via CRLF. + +Normally `llhttp` would error when after a chunk data a CRLF is missing before +starting a new chunk. +With this flag the new chunk can start immediately after the previous one. + +**Enabling this flag can pose a security issue since you will be exposed to request smuggling attacks. USE WITH CAUTION!** ## Build Instructions diff --git a/bench/index.ts b/bench/index.ts index 95376614..b3ff2e1e 100644 --- a/bench/index.ts +++ b/bench/index.ts @@ -1,89 +1,71 @@ -// NOTE: run `npm test` to build `./test/tmp/http-loose-request` +import * as assert from "assert"; +import { spawnSync } from "child_process"; +import { existsSync } from "fs"; +import { resolve } from "path"; -import * as assert from 'assert'; -import { spawnSync } from 'child_process'; -import { existsSync } from 'fs'; - -const isURL = !process.argv[2] || process.argv[2] === 'url'; -const isHTTP = !process.argv[2] || process.argv[2] === 'http'; +function request(tpl: TemplateStringsArray): string { + return tpl.raw[0].replace(/^\s+/gm, '').replace(/\n/gm, '').replace(/\\r/gm, '\r').replace(/\\n/gm, '\n') +} -const requests: Map = new Map(); +const urlExecutable = resolve(__dirname, "../test/tmp/url-url-c"); +const httpExecutable = resolve(__dirname, "../test/tmp/http-request-c"); + +const httpRequests: Record = { + "seanmonstar/httparse": request` + GET /wp-content/uploads/2010/03/hello-kitty-darth-vader-pink.jpg HTTP/1.1\r\n + Host: www.kittyhell.com\r\n + User-Agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; ja-JP-mac; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 Pathtraq/0.9\r\n + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n + Accept-Language: ja,en-us;q=0.7,en;q=0.3\r\n + Accept-Encoding: gzip,deflate\r\n + Accept-Charset: Shift_JIS,utf-8;q=0.7,*;q=0.7\r\n + Keep-Alive: 115\r\n + Connection: keep-alive\r\n + Cookie: wp_ozh_wsa_visits=2; wp_ozh_wsa_visit_lasttime=xxxxxxxxxx; __utma=xxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.x; __utmz=xxxxxxxxx.xxxxxxxxxx.x.x.utmccn=(referral)|utmcsr=reader.livedoor.com|utmcct=/reader/|utmcmd=referral\r\n\r\n + `, + "nodejs/http-parser": request` + POST /joyent/http-parser HTTP/1.1\r\n + Host: github.com\r\n + DNT: 1\r\n + Accept-Encoding: gzip, deflate, sdch\r\n + Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4\r\n + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) + AppleWebKit/537.36 (KHTML, like Gecko) + Chrome/39.0.2171.65 Safari/537.36\r\n + Accept: text/html,application/xhtml+xml,application/xml;q=0.9, + image/webp,*/*;q=0.8\r\n + Referer: https://github.com/joyent/http-parser\r\n + Connection: keep-alive\r\n + Transfer-Encoding: chunked\r\n + Cache-Control: max-age=0\r\n\r\nb\r\nhello world\r\n0\r\n\r\n + ` +} +const urlRequest = "http://example.com/path/to/file?query=value#fragment"; -if (!existsSync('./test/tmp/http-loose-request.c')) { - console.error('Run npm test to build ./test/tmp/http-loose-request'); +if (!existsSync(urlExecutable) || !existsSync(urlExecutable)) { + console.error( + "\x1b[31m\x1b[1mPlease run npm test in order to create required executables." + ); process.exit(1); } -requests.set('seanmonstar/httparse', - 'GET /wp-content/uploads/2010/03/hello-kitty-darth-vader-pink.jpg HTTP/1.1\r\n' + - 'Host: www.kittyhell.com\r\n' + - 'User-Agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; ja-JP-mac; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 Pathtraq/0.9\r\n' + - 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n' + - 'Accept-Language: ja,en-us;q=0.7,en;q=0.3\r\n' + - 'Accept-Encoding: gzip,deflate\r\n' + - 'Accept-Charset: Shift_JIS,utf-8;q=0.7,*;q=0.7\r\n' + - 'Keep-Alive: 115\r\n' + - 'Connection: keep-alive\r\n' + - 'Cookie: wp_ozh_wsa_visits=2; wp_ozh_wsa_visit_lasttime=xxxxxxxxxx; __utma=xxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.x; __utmz=xxxxxxxxx.xxxxxxxxxx.x.x.utmccn=(referral)|utmcsr=reader.livedoor.com|utmcct=/reader/|utmcmd=referral\r\n\r\n'); - -requests.set('nodejs/http-parser', - 'POST /joyent/http-parser HTTP/1.1\r\n' + - 'Host: github.com\r\n' + - 'DNT: 1\r\n' + - 'Accept-Encoding: gzip, deflate, sdch\r\n' + - 'Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4\r\n' + - 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) ' + - 'AppleWebKit/537.36 (KHTML, like Gecko) ' + - 'Chrome/39.0.2171.65 Safari/537.36\r\n' + - 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,' + - 'image/webp,*/*;q=0.8\r\n' + - 'Referer: https://github.com/joyent/http-parser\r\n' + - 'Connection: keep-alive\r\n' + - 'Transfer-Encoding: chunked\r\n' + - 'Cache-Control: max-age=0\r\n\r\nb\r\nhello world\r\n0\r\n\r\n'); - -if (process.argv[2] === 'loop') { +if (process.argv[2] === "loop") { const reqName = process.argv[3]; - assert(requests.has(reqName), `Unknown request name: "${reqName}"`); - - const request = requests.get(reqName)!; - spawnSync('./test/tmp/http-loose-request', [ - 'loop', - request - ], { stdio: 'inherit' }); + const request = httpRequests[reqName]!; + + assert(request, `Unknown request name: "${reqName}"`); + spawnSync(httpExecutable, ["loop", request], { stdio: "inherit" }); process.exit(0); } -if (isURL) { - console.log('url loose (C)'); - - spawnSync('./test/tmp/url-loose-url-c', [ - 'bench', - 'http://example.com/path/to/file?query=value#fragment' - ], { stdio: 'inherit' }); - - console.log('url strict (C)'); - - spawnSync('./test/tmp/url-strict-url-c', [ - 'bench', - 'http://example.com/path/to/file?query=value#fragment' - ], { stdio: 'inherit' }); +if (!process.argv[2] || process.argv[2] === "url") { + console.log("url (C)"); + spawnSync(urlExecutable, ["bench", urlRequest], { stdio: "inherit" }); } -if (isHTTP) { - for(const [name, request] of requests) { - console.log('http loose: "%s" (C)', name); - - spawnSync('./test/tmp/http-loose-request-c', [ - 'bench', - request - ], { stdio: 'inherit' }); - - console.log('http strict: "%s" (C)', name); - - spawnSync('./test/tmp/http-strict-request-c', [ - 'bench', - request - ], { stdio: 'inherit' }); +if (!process.argv[2] || process.argv[2] === "http") { + for (const [name, request] of Object.entries(httpRequests)) { + console.log('http: "%s" (C)', name); + spawnSync(httpExecutable, ["bench", request], { stdio: "inherit" }); } } diff --git a/bin/generate.ts b/bin/generate.ts index cb0d8c74..edb7f49f 100755 --- a/bin/generate.ts +++ b/bin/generate.ts @@ -1,101 +1,47 @@ #!/usr/bin/env -S npx ts-node -import * as fs from 'fs'; -import { LLParse } from 'llparse'; -import * as path from 'path'; -import * as semver from 'semver'; - -import * as llhttp from '../src/llhttp'; - -const pkgFile = path.join(__dirname, '..', 'package.json'); -const pkg = JSON.parse(fs.readFileSync(pkgFile).toString()); - -const BUILD_DIR = path.join(__dirname, '..', 'build'); -const C_DIR = path.join(BUILD_DIR, 'c'); -const SRC_DIR = path.join(__dirname, '..', 'src'); - -const C_FILE = path.join(C_DIR, 'llhttp.c'); -const HEADER_FILE = path.join(BUILD_DIR, 'llhttp.h'); - -for (const dir of [ BUILD_DIR, C_DIR ]) { - try { - fs.mkdirSync(dir); - } catch (e) { - // no-op - } -} - -function build(mode: 'strict' | 'loose') { - const llparse = new LLParse('llhttp__internal'); - const instance = new llhttp.HTTP(llparse, mode); - - return llparse.build(instance.build().entry, { - c: { - header: 'llhttp', - }, - debug: process.env.LLPARSE_DEBUG ? 'llhttp__debug' : undefined, - headerGuard: 'INCLUDE_LLHTTP_ITSELF_H_', - }); -} - -function guard(strict: string, loose: string): string { - let out = ''; - if (strict === loose) { - return strict; - } - - out += '#if LLHTTP_STRICT_MODE\n'; - out += '\n'; - out += strict + '\n'; - out += '\n'; - out += '#else /* !LLHTTP_STRICT_MODE */\n'; - out += '\n'; - out += loose + '\n'; - out += '\n'; - out += '#endif /* LLHTTP_STRICT_MODE */\n'; - - return out; -} - -const artifacts = { - loose: build('loose'), - strict: build('strict'), -}; - -let headers = ''; - -headers += '#ifndef INCLUDE_LLHTTP_H_\n'; -headers += '#define INCLUDE_LLHTTP_H_\n'; - -headers += '\n'; - -const version = semver.parse(pkg.version)!; +import { mkdirSync, readFileSync, writeFileSync } from 'fs'; +import { LLParse } from 'llparse'; +import { dirname, resolve } from 'path'; +import { parse } from 'semver'; +import { CHeaders, HTTP } from '../src/llhttp'; -headers += `#define LLHTTP_VERSION_MAJOR ${version.major}\n`; -headers += `#define LLHTTP_VERSION_MINOR ${version.minor}\n`; -headers += `#define LLHTTP_VERSION_PATCH ${version.patch}\n`; -headers += '\n'; +const C_FILE = resolve(__dirname, '../build/c/llhttp.c'); +const HEADER_FILE = resolve(__dirname, '../build/llhttp.h'); -headers += '#ifndef LLHTTP_STRICT_MODE\n'; -headers += '# define LLHTTP_STRICT_MODE 0\n'; -headers += '#endif\n'; -headers += '\n'; +const pkg = JSON.parse( + readFileSync(resolve(__dirname, '..', 'package.json')).toString(), +); +const version = parse(pkg.version)!; +const llparse = new LLParse('llhttp__internal'); -const cHeaders = new llhttp.CHeaders(); +const cHeaders = new CHeaders(); +const nativeHeaders = readFileSync(resolve(__dirname, '../src/native/api.h')); +const generated = llparse.build(new HTTP(llparse).build().entry, { + c: { + header: 'llhttp', + }, + debug: process.env.LLPARSE_DEBUG ? 'llhttp__debug' : undefined, + headerGuard: 'INCLUDE_LLHTTP_ITSELF_H_', +}); -headers += guard(artifacts.strict.header, artifacts.loose.header); +const headers = ` +#ifndef INCLUDE_LLHTTP_H_ +#define INCLUDE_LLHTTP_H_ -headers += '\n'; +#define LLHTTP_VERSION_MAJOR ${version.major} +#define LLHTTP_VERSION_MINOR ${version.minor} +#define LLHTTP_VERSION_PATCH ${version.patch} -headers += cHeaders.build(); +${generated.header} -headers += '\n'; +${cHeaders.build()} -headers += fs.readFileSync(path.join(SRC_DIR, 'native', 'api.h')); +${nativeHeaders} -headers += '\n'; -headers += '#endif /* INCLUDE_LLHTTP_H_ */\n'; +#endif /* INCLUDE_LLHTTP_H_ */ +`; -fs.writeFileSync(C_FILE, - guard(artifacts.strict.c || '', artifacts.loose.c || '')); -fs.writeFileSync(HEADER_FILE, headers); +mkdirSync(dirname(C_FILE), { recursive: true }); +writeFileSync(HEADER_FILE, headers); +writeFileSync(C_FILE, generated.c); diff --git a/package.json b/package.json index d44c8121..a07087e1 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "clean": "rm -rf lib && rm -rf test/tmp", "prepare": "npm run clean && npm run build-ts", "lint": "tslint -c tslint.json bin/*.ts src/*.ts src/**/*.ts test/*.ts test/**/*.ts", + "lint-fix": "tslint --fix -c tslint.json bin/*.ts src/*.ts src/**/*.ts test/*.ts test/**/*.ts", "mocha": "mocha --timeout=10000 -r ts-node/register/type-check --reporter progress test/*-test.ts", "test": "npm run mocha && npm run lint", "postversion": "RELEASE=`node -e \"process.stdout.write(require('./package').version)\"` make -B postversion", diff --git a/src/llhttp.ts b/src/llhttp.ts index 29adbe9c..ba36b014 100644 --- a/src/llhttp.ts +++ b/src/llhttp.ts @@ -1,9 +1,6 @@ import * as constants from './llhttp/constants'; -import HTTPMode = constants.HTTPMode; - export { constants }; -export { HTTPMode }; export { HTTP } from './llhttp/http'; export { URL } from './llhttp/url'; diff --git a/src/llhttp/constants.ts b/src/llhttp/constants.ts index 31c16389..eb563703 100644 --- a/src/llhttp/constants.ts +++ b/src/llhttp/constants.ts @@ -1,7 +1,5 @@ import { enumToMap, IEnumMap } from './utils'; -export type HTTPMode = 'loose' | 'strict'; - // C headers export enum ERROR { @@ -72,6 +70,9 @@ export enum LENIENT_FLAGS { KEEP_ALIVE = 1 << 2, TRANSFER_ENCODING = 1 << 3, VERSION = 1 << 4, + DATA_AFTER_CLOSE = 1 << 5, + OPTIONAL_LF_AFTER_CR = 1 << 6, + OPTIONAL_CRLF_AFTER_CHUNK = 1 << 7, } export enum METHODS { @@ -453,7 +454,7 @@ export const USERINFO_CHARS: CharList = ALPHANUM .concat([ '%', ';', ':', '&', '=', '+', '$', ',' ]); // TODO(indutny): use RFC -export const STRICT_URL_CHAR: CharList = ([ +export const URL_CHAR: CharList = ([ '!', '"', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', @@ -462,14 +463,6 @@ export const STRICT_URL_CHAR: CharList = ([ '{', '|', '}', '~', ] as CharList).concat(ALPHANUM); -export const URL_CHAR: CharList = STRICT_URL_CHAR - .concat(([ '\t', '\f' ] as CharList)); - -// All characters with 0x80 bit set to 1 -for (let i = 0x80; i <= 0xff; i++) { - URL_CHAR.push(i); -} - export const HEX: CharList = NUM.concat( [ 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F' ]); @@ -480,15 +473,13 @@ export const HEX: CharList = NUM.concat( * | "/" | "[" | "]" | "?" | "=" * | "{" | "}" | SP | HT */ -export const STRICT_TOKEN: CharList = ([ +export const TOKEN: CharList = ([ '!', '#', '$', '%', '&', '\'', '*', '+', '-', '.', '^', '_', '`', '|', '~', ] as CharList).concat(ALPHANUM); -export const TOKEN: CharList = STRICT_TOKEN.concat([ ' ' ]); - /* * Verify that a char is a valid visible (printable) US-ASCII * character or %x80-FF diff --git a/src/llhttp/http.ts b/src/llhttp/http.ts index 6b94f65c..7e76dc91 100644 --- a/src/llhttp/http.ts +++ b/src/llhttp/http.ts @@ -8,11 +8,10 @@ import { CharList, CONNECTION_TOKEN_CHARS, ERROR, FINISH, FLAGS, H_METHOD_MAP, HEADER_CHARS, HEADER_STATE, HEX_MAP, - HTTPMode, LENIENT_FLAGS, MAJOR, METHOD_MAP, METHODS, METHODS_HTTP, METHODS_ICE, METHODS_RTSP, MINOR, NUM_MAP, QUOTED_STRING, SPECIAL_HEADERS, - STRICT_TOKEN, TOKEN, TYPE, + TOKEN, TYPE, } from './constants'; import { URL } from './url'; @@ -50,6 +49,7 @@ const NODES: ReadonlyArray = [ 'req_http_minor', 'req_http_end', 'req_http_complete', + 'req_http_complete_crlf', 'req_pri_upgrade', @@ -97,7 +97,6 @@ const NODES: ReadonlyArray = [ 'chunk_extension_quoted_value_done', 'chunk_data', 'chunk_data_almost_done', - 'chunk_data_almost_done_skip', 'chunk_complete', 'body_identity', 'body_identity_eof', @@ -168,12 +167,11 @@ export class HTTP { private readonly callback: ICallbackMap; private readonly nodes: Map = new Map(); - constructor(private readonly llparse: LLParse, - private readonly mode: HTTPMode = 'loose') { + constructor(private readonly llparse: LLParse) { const p = llparse; - this.url = new URL(p, mode); - this.TOKEN = mode === 'strict' ? STRICT_TOKEN : TOKEN; + this.url = new URL(p); + this.TOKEN = TOKEN; this.span = { body: p.span(p.code.span('llhttp__on_body')), @@ -377,17 +375,14 @@ export class HTTP { n('res_status') .peek('\r', span.status.end().skipTo(n('res_line_almost_done'))) - .peek('\n', span.status.end().skipTo(onStatusComplete)) + .peek('\n', span.status.end().skipTo(n('res_line_almost_done'))) .skipTo(n('res_status')); - if (this.mode === 'strict') { - n('res_line_almost_done') - .match('\n', onStatusComplete) - .otherwise(p.error(ERROR.STRICT, 'Expected LF after CR')); - } else { - n('res_line_almost_done') - .skipTo(onStatusComplete); - } + n('res_line_almost_done') + .match(['\r', '\n'], onStatusComplete) + .otherwise(this.testLenientFlags(LENIENT_FLAGS.OPTIONAL_LF_AFTER_CR, { + 1: onStatusComplete, + }, p.error(ERROR.STRICT, 'Expected LF after CR'))); // Request n('start_req').otherwise(this.span.method.start(n('after_start_req'))); @@ -474,9 +469,15 @@ export class HTTP { ); n('req_http_complete') - .match([ '\r\n', '\n' ], n('headers_start')) + .match('\r', n('req_http_complete_crlf')) .otherwise(p.error(ERROR.INVALID_VERSION, 'Expected CRLF after version')); + n('req_http_complete_crlf') + .match('\n', n('headers_start')) + .otherwise(this.testLenientFlags(LENIENT_FLAGS.OPTIONAL_LF_AFTER_CR, { + 1: n('headers_start'), + }, p.error(ERROR.STRICT, 'Expected CRLF after version'))); + n('req_pri_upgrade') .match('\r\n\r\nSM\r\n\r\n', p.error(ERROR.PAUSED_H2_UPGRADE, 'Pause on PRI/Upgrade')) @@ -504,9 +505,6 @@ export class HTTP { n('header_field_start') .match('\r', n('headers_almost_done')) - /* they might be just sending \n instead of \r\n so this would be - * the second \n to denote the end of headers*/ - .peek('\n', n('headers_almost_done')) .otherwise(span.headerField.start(n('header_field'))); n('header_field') @@ -768,14 +766,12 @@ export class HTTP { ERROR.CB_CHUNK_COMPLETE, 'message_done'), }); - if (this.mode === 'strict') { - n('headers_almost_done') - .match('\n', checkTrailing) - .otherwise(p.error(ERROR.STRICT, 'Expected LF after headers')); - } else { - n('headers_almost_done') - .skipTo(checkTrailing); - } + n('headers_almost_done') + .match('\n', checkTrailing) + .otherwise( + this.testLenientFlags(LENIENT_FLAGS.OPTIONAL_LF_AFTER_CR, { + 1: checkTrailing, + }, p.error(ERROR.STRICT, 'Expected LF after headers'))); // Set `upgrade` if needed const beforeHeadersComplete = p.invoke(callback.beforeHeadersComplete); @@ -910,7 +906,7 @@ export class HTTP { .otherwise(this.span.chunkExtensionName.start(n('chunk_extension_name'))); n('chunk_extension_name') - .match(STRICT_TOKEN, n('chunk_extension_name')) + .match(TOKEN, n('chunk_extension_name')) .peek('=', this.span.chunkExtensionName.end().skipTo( this.span.chunkExtensionValue.start( onChunkExtensionNameCompleted(n('chunk_extension_value')), @@ -928,7 +924,7 @@ export class HTTP { n('chunk_extension_value') .match('"', n('chunk_extension_quoted_value')) - .match(STRICT_TOKEN, n('chunk_extension_value')) + .match(TOKEN, n('chunk_extension_value')) .peek(';', this.span.chunkExtensionValue.end().skipTo( onChunkExtensionValueCompleted(n('chunk_size_otherwise')), )) @@ -954,14 +950,13 @@ export class HTTP { .otherwise(p.error(ERROR.STRICT, 'Invalid character in chunk extensions quote value')); - if (this.mode === 'strict') { - n('chunk_size_almost_done') - .match('\n', n('chunk_size_almost_done_lf')) - .otherwise(p.error(ERROR.STRICT, 'Expected LF after chunk size')); - } else { - n('chunk_size_almost_done') - .skipTo(n('chunk_size_almost_done_lf')); - } + n('chunk_size_almost_done') + .match('\n', n('chunk_size_almost_done_lf')) + .otherwise( + this.testLenientFlags(LENIENT_FLAGS.OPTIONAL_LF_AFTER_CR, { + 1: n('chunk_size_almost_done_lf'), + }).otherwise(p.error(ERROR.STRICT, 'Expected LF after chunk size')), + ); const toChunk = this.isEqual('content_length', 0, { equal: this.setFlag(FLAGS.TRAILING, 'header_field_start'), @@ -977,17 +972,13 @@ export class HTTP { .otherwise(p.consume('content_length').otherwise( span.body.end(n('chunk_data_almost_done'))))); - if (this.mode === 'strict') { - n('chunk_data_almost_done') - .match('\r\n', n('chunk_complete')) - .otherwise(p.error(ERROR.STRICT, 'Expected CRLF after chunk')); - } else { - n('chunk_data_almost_done') - .skipTo(n('chunk_data_almost_done_skip')); - } - - n('chunk_data_almost_done_skip') - .skipTo(n('chunk_complete')); + n('chunk_data_almost_done') + .match('\r\n', n('chunk_complete')) + .otherwise( + this.testLenientFlags(LENIENT_FLAGS.OPTIONAL_CRLF_AFTER_CHUNK, { + 1: n('chunk_complete'), + }).otherwise(p.error(ERROR.STRICT, 'Expected LF after chunk data')), + ); n('chunk_complete') .otherwise(this.invokePausable('on_chunk_complete', @@ -1015,16 +1006,13 @@ export class HTTP { 1: this.update('content_length', 0, n('restart')), }, this.update('finish', FINISH.SAFE, lenientClose))); - if (this.mode === 'strict') { - // Error on extra data after `Connection: close` - n('closed') - .match([ '\r', '\n' ], n('closed')) - .skipTo(p.error(ERROR.CLOSED_CONNECTION, - 'Data after `Connection: close`')); - } else { - // Discard all data after `Connection: close` - n('closed').skipTo(n('closed')); - } + const lenientDiscardAfterClose = this.testLenientFlags(LENIENT_FLAGS.DATA_AFTER_CLOSE, { + 1: n('closed'), + }, p.error(ERROR.CLOSED_CONNECTION, 'Data after `Connection: close`')); + + n('closed') + .match([ '\r', '\n' ], n('closed')) + .skipTo(lenientDiscardAfterClose); n('restart') .otherwise( diff --git a/src/llhttp/url.ts b/src/llhttp/url.ts index 49f816c8..c5fced90 100644 --- a/src/llhttp/url.ts +++ b/src/llhttp/url.ts @@ -7,8 +7,6 @@ import { ALPHA, CharList, ERROR, - HTTPMode, - STRICT_URL_CHAR, URL_CHAR, USERINFO_CHARS, } from './constants'; @@ -29,22 +27,16 @@ export interface IURLResult { type SpanTable = Map; export class URL { - private readonly span: source.Span | undefined; private readonly spanTable: SpanTable = new Map(); private readonly errorInvalid: Node; - private readonly errorStrictInvalid: Node; private readonly URL_CHAR: CharList; - constructor(private readonly llparse: LLParse, - private readonly mode: HTTPMode = 'loose', - separateSpans: boolean = false) { + constructor(private readonly llparse: LLParse, separateSpans: boolean = false) { const p = this.llparse; this.errorInvalid = p.error(ERROR.INVALID_URL, 'Invalid characters in url'); - this.errorStrictInvalid = - p.error(ERROR.INVALID_URL, 'Invalid characters in url (strict mode)'); - this.URL_CHAR = mode === 'strict' ? STRICT_URL_CHAR : URL_CHAR; + this.URL_CHAR = URL_CHAR; const table = this.spanTable; if (separateSpans) { @@ -221,9 +213,7 @@ export class URL { private node(name: string): Match { const res = this.llparse.node('url_' + name); - if (this.mode === 'strict') { - res.match([ '\t', '\f' ], this.errorStrictInvalid); - } + res.match([ '\t', '\f' ], this.errorInvalid); return res; } diff --git a/src/native/api.c b/src/native/api.c index 4b687a5d..e0d03850 100644 --- a/src/native/api.c +++ b/src/native/api.c @@ -283,6 +283,38 @@ void llhttp_set_lenient_transfer_encoding(llhttp_t* parser, int enabled) { } } +void llhttp_set_lenient_version(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_VERSION; + } else { + parser->lenient_flags &= ~LENIENT_VERSION; + } +} + +void llhttp_set_lenient_data_after_close(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_DATA_AFTER_CLOSE; + } else { + parser->lenient_flags &= ~LENIENT_DATA_AFTER_CLOSE; + } +} + +void llhttp_set_lenient_optional_lf_after_cr(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_OPTIONAL_LF_AFTER_CR; + } else { + parser->lenient_flags &= ~LENIENT_OPTIONAL_LF_AFTER_CR; + } +} + +void llhttp_set_lenient_optional_crlf_after_chunk(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_OPTIONAL_CRLF_AFTER_CHUNK; + } else { + parser->lenient_flags &= ~LENIENT_OPTIONAL_CRLF_AFTER_CHUNK; + } +} + /* Callbacks */ diff --git a/src/native/api.h b/src/native/api.h index 50a7a2e2..a05ea64c 100644 --- a/src/native/api.h +++ b/src/native/api.h @@ -223,7 +223,8 @@ const char* llhttp_status_name(llhttp_status_t status); * `HPE_INVALID_HEADER_TOKEN` will be raised for incorrect header values when * lenient parsing is "on". * - * **(USE AT YOUR OWN RISK)** + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** */ LLHTTP_EXPORT void llhttp_set_lenient_headers(llhttp_t* parser, int enabled); @@ -237,7 +238,8 @@ void llhttp_set_lenient_headers(llhttp_t* parser, int enabled); * request smuggling, but may be less desirable for small number of cases * involving legacy servers. * - * **(USE AT YOUR OWN RISK)** + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** */ LLHTTP_EXPORT void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled); @@ -252,7 +254,8 @@ void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled); * but might interact badly with outdated and insecure clients. With this flag * the extra request/response will be parsed normally. * - * **(USE AT YOUR OWN RISK)** + * **Enabling this flag can pose a security issue since you will be exposed to + * poisoning attacks. USE WITH CAUTION!** */ LLHTTP_EXPORT void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled); @@ -266,11 +269,61 @@ void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled); * avoid request smuggling. * With this flag the extra value will be parsed normally. * - * **(USE AT YOUR OWN RISK)** + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** */ LLHTTP_EXPORT void llhttp_set_lenient_transfer_encoding(llhttp_t* parser, int enabled); +/* Enables/disables lenient handling of HTTP version. + * + * Normally `llhttp` would error when the HTTP version in the request or status line + * is not `0.9`, `1.0`, `1.1` or `2.0`. + * With this flag the invalid value will be parsed normally. + * + * **Enabling this flag can pose a security issue since you will allow unsupported + * HTTP versions. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_version(llhttp_t* parser, int enabled); + +/* Enables/disables lenient handling of additional data received after a message ends + * and keep-alive is disabled. + * + * Normally `llhttp` would error when additional unexpected data is received if the message + * contains the `Connection` header with `close` value. + * With this flag the extra data will discarded without throwing an error. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * poisoning attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_data_after_close(llhttp_t* parser, int enabled); + +/* Enables/disables lenient handling of incomplete CRLF sequences. + * + * Normally `llhttp` would error when a CR is not followed by LF when terminating the + * request line, the status line, the headers or a chunk header. + * With this flag only a CR is required to terminate such sections. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_optional_lf_after_cr(llhttp_t* parser, int enabled); + +/* Enables/disables lenient handling of chunks not separated via CRLF. + * + * Normally `llhttp` would error when after a chunk data a CRLF is missing before + * starting a new chunk. + * With this flag the new chunk can start immediately after the previous one. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_optional_crlf_after_chunk(llhttp_t* parser, int enabled); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/test/fixtures/extra.c b/test/fixtures/extra.c index c0efa373..238f2a7d 100644 --- a/test/fixtures/extra.c +++ b/test/fixtures/extra.c @@ -121,6 +121,36 @@ void llhttp__test_init_response_lenient_headers(llparse_t* s) { s->lenient_flags |= LENIENT_HEADERS; } +void llhttp__test_init_request_lenient_data_after_close(llparse_t* s) { + llhttp__test_init_request(s); + s->lenient_flags |= LENIENT_DATA_AFTER_CLOSE; +} + +void llhttp__test_init_response_lenient_data_after_close(llparse_t* s) { + llhttp__test_init_response(s); + s->lenient_flags |= LENIENT_DATA_AFTER_CLOSE; +} + +void llhttp__test_init_request_lenient_optional_lf_after_cr(llparse_t* s) { + llhttp__test_init_request(s); + s->lenient_flags |= LENIENT_OPTIONAL_LF_AFTER_CR; +} + +void llhttp__test_init_response_lenient_optional_lf_after_cr(llparse_t* s) { + llhttp__test_init_response(s); + s->lenient_flags |= LENIENT_OPTIONAL_LF_AFTER_CR; +} + +void llhttp__test_init_request_lenient_optional_crlf_after_chunk(llparse_t* s) { + llhttp__test_init_request(s); + s->lenient_flags |= LENIENT_OPTIONAL_CRLF_AFTER_CHUNK; +} + +void llhttp__test_init_response_lenient_optional_crlf_after_chunk(llparse_t* s) { + llhttp__test_init_response(s); + s->lenient_flags |= LENIENT_OPTIONAL_CRLF_AFTER_CHUNK; +} + void llhttp__test_finish(llparse_t* s) { llparse__print(NULL, NULL, "finish=%d", s->finish); diff --git a/test/fixtures/index.ts b/test/fixtures/index.ts index a3cb1f6f..a5b74e37 100644 --- a/test/fixtures/index.ts +++ b/test/fixtures/index.ts @@ -8,14 +8,37 @@ import * as path from 'path'; import * as llhttp from '../../src/llhttp'; -export type TestType = 'request' | 'response' | 'request-lenient-headers' | - 'request-lenient-chunked-length' | 'request-lenient-transfer-encoding' | - 'request-lenient-keep-alive' | 'response-lenient-keep-alive' | - 'response-lenient-headers' | 'request-lenient-version' | 'response-lenient-version' | - 'request-finish' | 'response-finish' | +export { FixtureResult }; + +export type TestType = 'request' | 'response' | 'request-finish' | 'response-finish' | + 'request-lenient-headers' | 'request-lenient-chunked-length' | 'request-lenient-transfer-encoding' | + 'request-lenient-keep-alive' | 'response-lenient-keep-alive' | 'response-lenient-headers' | + 'request-lenient-version' | 'response-lenient-version' | + 'request-lenient-data-after-close' | 'response-lenient-data-after-close' | + 'response-lenient-optional-lf-after-cr' | 'request-lenient-optional-lf-after-cr' | + 'response-lenient-optional-crlf-after-chunk' | 'request-lenient-optional-crlf-after-chunk' | 'none' | 'url'; -export { FixtureResult }; +export const allowedTypes: TestType[] = [ + 'request', + 'response', + 'request-finish', + 'response-finish', + 'request-lenient-headers', + 'response-lenient-headers', + 'request-lenient-keep-alive', + 'response-lenient-keep-alive', + 'request-lenient-chunked-length', + 'request-lenient-transfer-encoding', + 'request-lenient-version', + 'response-lenient-version', + 'request-lenient-data-after-close', + 'response-lenient-data-after-close', + 'request-lenient-optional-lf-after-cr', + 'response-lenient-optional-lf-after-cr', + 'request-lenient-optional-crlf-after-chunk', + 'response-lenient-optional-crlf-after-chunk', +]; const BUILD_DIR = path.join(__dirname, '..', 'tmp'); const CHEADERS_FILE = path.join(BUILD_DIR, 'cheaders.h'); @@ -62,18 +85,13 @@ export async function build( } const extra = options.extra === undefined ? [] : options.extra.slice(); - if (ty === 'request' || ty === 'response' || - ty === 'request-lenient-headers' || - ty === 'request-lenient-chunked-length' || - ty === 'request-lenient-transfer-encoding' || - ty === 'request-lenient-keep-alive' || - ty === 'request-lenient-version' || - ty === 'response-lenient-headers' || - ty === 'response-lenient-keep-alive' || - ty === 'response-lenient-version') { + + if (allowedTypes.includes(ty)) { extra.push( `-DLLPARSE__TEST_INIT=llhttp__test_init_${ty.replace(/-/g, '_')}`); - } else if (ty === 'request-finish' || ty === 'response-finish') { + } + + if (ty === 'request-finish' || ty === 'response-finish') { if (ty === 'request-finish') { extra.push('-DLLPARSE__TEST_INIT=llhttp__test_init_request'); } else { diff --git a/test/md-test.ts b/test/md-test.ts index 55881812..0c24e183 100644 --- a/test/md-test.ts +++ b/test/md-test.ts @@ -8,7 +8,7 @@ import * as vm from 'vm'; import * as llhttp from '../src/llhttp'; import {IHTTPResult} from '../src/llhttp/http'; import {IURLResult} from '../src/llhttp/url'; -import { build, FixtureResult, TestType } from './fixtures'; +import { allowedTypes, build, FixtureResult, TestType } from './fixtures'; // // Cache nodes/llparse instances ahead of time @@ -25,34 +25,18 @@ interface IUrlCacheEntry { entry: IURLResult['entry']['normal']; } -const nodeCache = new Map(); -const urlCache = new Map(); const modeCache = new Map(); -function buildNode(mode: llhttp.HTTPMode) { - let entry = nodeCache.get(mode); - - if (entry) { - return entry; - } - +function buildNode() { const p = new LLParse(); - const instance = new llhttp.HTTP(p, mode); + const instance = new llhttp.HTTP(p); - entry = { llparse: p, entry: instance.build().entry }; - nodeCache.set(mode, entry); - return entry; + return { llparse: p, entry: instance.build().entry }; } -function buildURL(mode: llhttp.HTTPMode) { - let entry = urlCache.get(mode); - - if (entry) { - return entry; - } - +function buildURL() { const p = new LLParse(); - const instance = new llhttp.URL(p, mode, true); + const instance = new llhttp.URL(p, true); const node = instance.build(); @@ -60,19 +44,17 @@ function buildURL(mode: llhttp.HTTPMode) { node.exit.toHTTP.otherwise(node.entry.normal); node.exit.toHTTP09.otherwise(node.entry.normal); - entry = { llparse: p, entry: node.entry.normal }; - urlCache.set(mode, entry); - return entry; + return { llparse: p, entry: node.entry.normal }; } // // Build binaries using cached nodes/llparse // -async function buildMode(mode: llhttp.HTTPMode, ty: TestType, meta: any) +async function buildMode(ty: TestType, meta: any) : Promise { - const cacheKey = `${mode}:${ty}:${JSON.stringify(meta || {})}`; + const cacheKey = `${ty}:${JSON.stringify(meta || {})}`; let entry = modeCache.get(cacheKey); if (entry) { @@ -83,11 +65,11 @@ async function buildMode(mode: llhttp.HTTPMode, ty: TestType, meta: any) let prefix: string; let extra: string[]; if (ty === 'url') { - node = buildURL(mode); + node = buildURL(); prefix = 'url'; extra = []; } else { - node = buildNode(mode); + node = buildNode(); prefix = 'http'; extra = [ '-DLLHTTP__TEST_HTTP', @@ -103,7 +85,7 @@ async function buildMode(mode: llhttp.HTTPMode, ty: TestType, meta: any) extra.push('-DLLHTTP__TEST_SKIP_BODY=1'); } - entry = await build(node.llparse, node.entry, `${prefix}-${mode}-${ty}`, { + entry = await build(node.llparse, node.entry, `${prefix}-${ty}`, { extra, }, ty); @@ -125,11 +107,11 @@ function run(name: string): void { const raw = fs.readFileSync(path.join(__dirname, name + '.md')).toString(); const groups = md.parse(raw); - function runSingleTest(mode: llhttp.HTTPMode, ty: TestType, meta: any, + function runSingleTest(ty: TestType, meta: any, input: string, expected: ReadonlyArray): void { - it(`should pass in mode="${mode}" and for type="${ty}"`, async () => { - const binary = await buildMode(mode, ty, meta); + it(`should pass for type="${ty}"`, async () => { + const binary = await buildMode(ty, meta); await binary.check(input, expected, { noScan: meta.noScan === true, }); @@ -138,8 +120,7 @@ function run(name: string): void { function runTest(test: Test) { describe(test.name + ` at ${name}.md:${test.line + 1}`, () => { - let modes: llhttp.HTTPMode[] = [ 'strict', 'loose' ]; - let types: TestType[] = [ 'none' ]; + let types: TestType[] = []; const isURL = test.values.has('url'); const inputKey = isURL ? 'url' : 'http'; @@ -161,36 +142,13 @@ function run(name: string): void { types = [ 'url' ]; } else { assert(meta.hasOwnProperty('type'), 'Missing required `type` metadata'); - if (meta.type === 'request') { - types.push('request'); - } else if (meta.type === 'response') { - types.push('response'); - } else if (meta.type === 'request-only') { - types = [ 'request' ]; - } else if (meta.type === 'request-lenient-headers') { - types = [ 'request-lenient-headers' ]; - } else if (meta.type === 'request-lenient-chunked-length') { - types = [ 'request-lenient-chunked-length' ]; - } else if (meta.type === 'request-lenient-keep-alive') { - types = [ 'request-lenient-keep-alive' ]; - } else if (meta.type === 'request-lenient-transfer-encoding') { - types = [ 'request-lenient-transfer-encoding' ]; - } else if (meta.type === 'request-lenient-version') { - types = [ 'request-lenient-version' ]; - } else if (meta.type === 'response-lenient-keep-alive') { - types = [ 'response-lenient-keep-alive' ]; - } else if (meta.type === 'response-lenient-headers') { - types = [ 'response-lenient-headers' ]; - } else if (meta.type === 'response-lenient-version') { - types = [ 'response-lenient-version' ]; - } else if (meta.type === 'response-only') { - types = [ 'response' ]; - } else if (meta.type === 'request-finish') { - types = [ 'request-finish' ]; - } else if (meta.type === 'response-finish') { - types = [ 'response-finish' ]; - } else { - throw new Error(`Invalid value of \`type\` metadata: "${meta.type}"`); + + if (meta.type) { + if (!allowedTypes.includes(meta.type)) { + throw new Error(`Invalid value of \`type\` metadata: "${meta.type}"`); + } + + types.push(meta.type); } } @@ -199,15 +157,6 @@ function run(name: string): void { assert.strictEqual(test.values.get('log')!.length, 1, 'Expected just one output'); - if (meta.mode === 'strict') { - modes = [ 'strict' ]; - } else if (meta.mode === 'loose') { - modes = [ 'loose' ]; - } else { - assert(!meta.hasOwnProperty('mode'), - `Invalid value of \`mode\` metadata: "${meta.mode}"`); - } - let input: string = test.values.get(inputKey)![0]; let expected: string = test.values.get('log')![0]; @@ -265,14 +214,12 @@ function run(name: string): void { } }); - for (const mode of modes) { - for (const ty of types) { - if (meta.skip === true || (process.env.ONLY === 'true' && !meta.only)) { - continue; - } - - runSingleTest(mode, ty, meta, input, fullExpected); + for (const ty of types) { + if (meta.skip === true || (process.env.ONLY === 'true' && !meta.only)) { + continue; } + + runSingleTest(ty, meta, input, fullExpected); } }); } diff --git a/test/request/connection.md b/test/request/connection.md index 3c2551e4..a0e2d973 100644 --- a/test/request/connection.md +++ b/test/request/connection.md @@ -72,9 +72,9 @@ off=90 headers complete method=4 v=1/1 flags=1 content_length=0 off=90 message complete ``` -### No restart when keep-alive is off (1.0) and parser is in strict mode +### No restart when keep-alive is off (1.0) - + ```http PUT /url HTTP/1.0 @@ -241,11 +241,11 @@ off=40 headers complete method=4 v=1/1 flags=2 content_length=0 off=40 message complete ``` -### CRLF between requests, explicit `close` (strict mode) +### CRLF between requests, explicit `close` -`close` means closed connection in strict mode. +`close` means closed connection - + ```http POST / HTTP/1.1 Host: www.example.com diff --git a/test/request/content-length.md b/test/request/content-length.md index c8fa9daa..83e09c27 100644 --- a/test/request/content-length.md +++ b/test/request/content-length.md @@ -432,3 +432,53 @@ off=85 headers complete method=3 v=1/1 flags=20 content_length=456 off=85 skip body off=85 message complete ``` + +## Missing CRLF-CRLF before body + + +```http +PUT /url HTTP/1.1 +Content-Length: 3 +\rabc +``` + +```log +off=0 message begin +off=0 len=3 span[method]="PUT" +off=3 method complete +off=4 len=4 span[url]="/url" +off=9 url complete +off=14 len=3 span[version]="1.1" +off=17 version complete +off=19 len=14 span[header_field]="Content-Length" +off=34 header_field complete +off=35 len=1 span[header_value]="3" +off=38 header_value complete +off=39 error code=2 reason="Expected LF after headers" +``` + +## Missing CRLF-CRLF before body in lenient + + +```http +PUT /url HTTP/1.1 +Content-Length: 3 +\rabc +``` + +```log +off=0 message begin +off=0 len=3 span[method]="PUT" +off=3 method complete +off=4 len=4 span[url]="/url" +off=9 url complete +off=14 len=3 span[version]="1.1" +off=17 version complete +off=19 len=14 span[header_field]="Content-Length" +off=34 header_field complete +off=35 len=1 span[header_value]="3" +off=38 header_value complete +off=39 headers complete method=4 v=1/1 flags=20 content_length=3 +off=39 len=3 span[body]="abc" +off=42 message complete +``` \ No newline at end of file diff --git a/test/request/invalid.md b/test/request/invalid.md index 4bdbeb1b..99d8468e 100644 --- a/test/request/invalid.md +++ b/test/request/invalid.md @@ -133,6 +133,71 @@ off=41 len=1 span[header_value]="x" off=42 error code=10 reason="Invalid header value char" ``` +### Headers separated by dummy characters + + +```http +GET / HTTP/1.1 +Connection: close +Host: a +\rZGET /evil: HTTP/1.1 +Host: a + +``` + +```log +off=0 message begin +off=0 len=3 span[method]="GET" +off=3 method complete +off=4 len=1 span[url]="/" +off=6 url complete +off=11 len=3 span[version]="1.1" +off=14 version complete +off=16 len=10 span[header_field]="Connection" +off=27 header_field complete +off=28 len=5 span[header_value]="close" +off=35 header_value complete +off=35 len=4 span[header_field]="Host" +off=40 header_field complete +off=41 len=1 span[header_value]="a" +off=44 header_value complete +off=45 error code=2 reason="Expected LF after headers" +``` + + +### Headers separated by dummy characters in lenient mode + + +```http +GET / HTTP/1.1 +Connection: close +Host: a +\rZGET /evil: HTTP/1.1 +Host: a + +``` + +```log +off=0 message begin +off=0 len=3 span[method]="GET" +off=3 method complete +off=4 len=1 span[url]="/" +off=6 url complete +off=11 len=3 span[version]="1.1" +off=14 version complete +off=16 len=10 span[header_field]="Connection" +off=27 header_field complete +off=28 len=5 span[header_value]="close" +off=35 header_value complete +off=35 len=4 span[header_field]="Host" +off=40 header_field complete +off=41 len=1 span[header_value]="a" +off=44 header_value complete +off=45 headers complete method=1 v=1/1 flags=2 content_length=0 +off=45 message complete +off=46 error code=5 reason="Data after `Connection: close`" +``` + ### Empty headers separated by CR diff --git a/test/request/sample.md b/test/request/sample.md index 32ba3069..a5347e62 100644 --- a/test/request/sample.md +++ b/test/request/sample.md @@ -374,8 +374,9 @@ off=133 message complete ## Line folding in header value with LF + ```http -GET / HTTP/1.1\n\ +GET / HTTP/1.1 Line1: abc\n\ \tdef\n\ ghi\n\ @@ -400,10 +401,52 @@ off=4 len=1 span[url]="/" off=6 url complete off=11 len=3 span[version]="1.1" off=14 version complete -off=15 len=5 span[header_field]="Line1" -off=21 header_field complete -off=24 len=3 span[header_value]="abc" -off=27 error code=10 reason="Invalid header value char" +off=16 len=5 span[header_field]="Line1" +off=22 header_field complete +off=25 len=3 span[header_value]="abc" +off=28 error code=10 reason="Invalid header value char" +``` + +## No LF after CR + + + +```http +GET / HTTP/1.1\rLine: 1 + +``` + +```log +off=0 message begin +off=0 len=3 span[method]="GET" +off=3 method complete +off=4 len=1 span[url]="/" +off=6 url complete +off=11 len=3 span[version]="1.1" +off=14 version complete +off=15 error code=2 reason="Expected CRLF after version" +``` + +## No LF after CR in lenient mode + + + +```http +GET / HTTP/1.1\rLine: 1 + +``` + +```log +off=0 message begin +off=0 len=3 span[method]="GET" +off=3 method complete +off=4 len=1 span[url]="/" +off=6 url complete +off=11 len=3 span[version]="1.1" +off=14 version complete +off=15 len=4 span[header_field]="Line" +off=20 header_field complete +off=21 len=1 span[header_value]="1" ``` ## Request starting with CRLF diff --git a/test/request/transfer-encoding.md b/test/request/transfer-encoding.md index 9fd99742..4be8129f 100644 --- a/test/request/transfer-encoding.md +++ b/test/request/transfer-encoding.md @@ -825,7 +825,7 @@ off=57 error code=12 reason="Invalid character in chunk size" ## Validate chunk parameters - + ```http PUT /url HTTP/1.1 Transfer-Encoding: chunked @@ -854,7 +854,7 @@ off=50 error code=12 reason="Invalid character in chunk size" ## Invalid OBS fold after chunked value - + ```http PUT /url HTTP/1.1 Transfer-Encoding: chunked @@ -883,3 +883,203 @@ off=54 header_value complete off=56 headers complete method=4 v=1/1 flags=200 content_length=0 off=56 error code=15 reason="Request has invalid `Transfer-Encoding`" ``` + +### Chunk header not terminated by CRLF + + + +```http +GET / HTTP/1.1 +Host: a +Connection: close +Transfer-Encoding: chunked + +5\r\r;ABCD +34 +E +0 + +GET / HTTP/1.1 +Host: a +Content-Length: 5 + +0 + +``` + +```log +off=0 message begin +off=0 len=3 span[method]="GET" +off=3 method complete +off=4 len=1 span[url]="/" +off=6 url complete +off=11 len=3 span[version]="1.1" +off=14 version complete +off=16 len=4 span[header_field]="Host" +off=21 header_field complete +off=22 len=1 span[header_value]="a" +off=25 header_value complete +off=25 len=10 span[header_field]="Connection" +off=36 header_field complete +off=37 len=6 span[header_value]="close " +off=45 header_value complete +off=45 len=17 span[header_field]="Transfer-Encoding" +off=63 header_field complete +off=64 len=8 span[header_value]="chunked " +off=74 header_value complete +off=76 headers complete method=1 v=1/1 flags=20a content_length=0 +off=78 error code=2 reason="Expected LF after chunk size" +``` + +### Chunk header not terminated by CRLF in lenient mode + + + +```http +GET / HTTP/1.1 +Host: a +Connection: close +Transfer-Encoding: chunked + +6\r\r;ABCD +33 +E +0 + +GET / HTTP/1.1 +Host: a +Content-Length: 5 +0 + + +``` + +```log +off=0 message begin +off=0 len=3 span[method]="GET" +off=3 method complete +off=4 len=1 span[url]="/" +off=6 url complete +off=11 len=3 span[version]="1.1" +off=14 version complete +off=16 len=4 span[header_field]="Host" +off=21 header_field complete +off=22 len=1 span[header_value]="a" +off=25 header_value complete +off=25 len=10 span[header_field]="Connection" +off=36 header_field complete +off=37 len=6 span[header_value]="close " +off=45 header_value complete +off=45 len=17 span[header_field]="Transfer-Encoding" +off=63 header_field complete +off=64 len=8 span[header_value]="chunked " +off=74 header_value complete +off=76 headers complete method=1 v=1/1 flags=20a content_length=0 +off=78 chunk header len=6 +off=78 len=1 span[body]=cr +off=79 len=5 span[body]=";ABCD" +off=86 chunk complete +off=90 chunk header len=51 +off=90 len=1 span[body]="E" +off=91 len=1 span[body]=cr +off=92 len=1 span[body]=lf +off=93 len=1 span[body]="0" +off=94 len=1 span[body]=cr +off=95 len=1 span[body]=lf +off=96 len=1 span[body]=cr +off=97 len=1 span[body]=lf +off=98 len=15 span[body]="GET / HTTP/1.1 " +off=113 len=1 span[body]=cr +off=114 len=1 span[body]=lf +off=115 len=7 span[body]="Host: a" +off=122 len=1 span[body]=cr +off=123 len=1 span[body]=lf +off=124 len=17 span[body]="Content-Length: 5" +off=143 chunk complete +off=146 chunk header len=0 +off=148 chunk complete +off=148 message complete +``` + +### Chunk data not terminated by CRLF + + + +```http +GET / HTTP/1.1 +Host: a +Connection: close +Transfer-Encoding: chunked + +5 +ABCDE0 + +``` + +```log +off=0 message begin +off=0 len=3 span[method]="GET" +off=3 method complete +off=4 len=1 span[url]="/" +off=6 url complete +off=11 len=3 span[version]="1.1" +off=14 version complete +off=16 len=4 span[header_field]="Host" +off=21 header_field complete +off=22 len=1 span[header_value]="a" +off=25 header_value complete +off=25 len=10 span[header_field]="Connection" +off=36 header_field complete +off=37 len=6 span[header_value]="close " +off=45 header_value complete +off=45 len=17 span[header_field]="Transfer-Encoding" +off=63 header_field complete +off=64 len=8 span[header_value]="chunked " +off=74 header_value complete +off=76 headers complete method=1 v=1/1 flags=20a content_length=0 +off=79 chunk header len=5 +off=79 len=5 span[body]="ABCDE" +off=84 error code=2 reason="Expected LF after chunk data" +``` + +### Chunk data not terminated by CRLF in lenient mode + + + +```http +GET / HTTP/1.1 +Host: a +Connection: close +Transfer-Encoding: chunked + +5 +ABCDE0 + +``` + +```log +off=0 message begin +off=0 len=3 span[method]="GET" +off=3 method complete +off=4 len=1 span[url]="/" +off=6 url complete +off=11 len=3 span[version]="1.1" +off=14 version complete +off=16 len=4 span[header_field]="Host" +off=21 header_field complete +off=22 len=1 span[header_value]="a" +off=25 header_value complete +off=25 len=10 span[header_field]="Connection" +off=36 header_field complete +off=37 len=6 span[header_value]="close " +off=45 header_value complete +off=45 len=17 span[header_field]="Transfer-Encoding" +off=63 header_field complete +off=64 len=8 span[header_value]="chunked " +off=74 header_value complete +off=76 headers complete method=1 v=1/1 flags=20a content_length=0 +off=79 chunk header len=5 +off=79 len=5 span[body]="ABCDE" +off=84 chunk complete +off=87 chunk header len=0 +``` \ No newline at end of file diff --git a/test/request/uri.md b/test/request/uri.md index 356863e6..f7f12b04 100644 --- a/test/request/uri.md +++ b/test/request/uri.md @@ -133,35 +133,9 @@ off=42 headers complete method=1 v=1/1 flags=0 content_length=0 off=42 message complete ``` -## UTF-8 in URI path in loose mode - - -```http -GET /δ¶/δt/космос/pope?q=1#narf HTTP/1.1 -Host: github.com - - -``` - -```log -off=0 message begin -off=0 len=3 span[method]="GET" -off=3 method complete -off=4 len=36 span[url]="/δ¶/δt/космос/pope?q=1#narf" -off=41 url complete -off=46 len=3 span[version]="1.1" -off=49 version complete -off=51 len=4 span[header_field]="Host" -off=56 header_field complete -off=57 len=10 span[header_value]="github.com" -off=69 header_value complete -off=71 headers complete method=1 v=1/1 flags=0 content_length=0 -off=71 message complete -``` - ## Disallow UTF-8 in URI path in strict mode - + ```http GET /δ¶/δt/pope?q=1#narf HTTP/1.1 Host: github.com diff --git a/test/response/connection.md b/test/response/connection.md index 4457743a..b3f4f5e2 100644 --- a/test/response/connection.md +++ b/test/response/connection.md @@ -147,9 +147,9 @@ off=35 version complete off=40 len=2 span[status]="OK" ``` -## HTTP/1.1 with keep-alive disabled and 204 status in strict mode +## HTTP/1.1 with keep-alive disabled and 204 status - + ```http HTTP/1.1 204 No content Connection: close @@ -172,11 +172,11 @@ off=46 message complete off=47 error code=5 reason="Data after `Connection: close`" ``` -## HTTP/1.1 with keep-alive disabled, content-length, and in loose mode +## HTTP/1.1 with keep-alive disabled, content-length, and in lenient mode -Parser should discard extra request in loose mode. +Parser should discard extra request in lenient mode. - + ```http HTTP/1.1 200 No content Content-Length: 5 @@ -204,11 +204,11 @@ off=65 len=5 span[body]="2ad73" off=70 message complete ``` -## HTTP/1.1 with keep-alive disabled, content-length, and in strict mode +## HTTP/1.1 with keep-alive disabled, content-length Parser should discard extra request in strict mode. - + ```http HTTP/1.1 200 No content Content-Length: 5 @@ -237,43 +237,6 @@ off=70 message complete off=71 error code=5 reason="Data after `Connection: close`" ``` -## HTTP/1.1 with keep-alive disabled, content-length, and in lenient mode - -Parser should process extra request in lenient mode. - - -```http -HTTP/1.1 200 No content -Content-Length: 5 -Connection: close - -2ad73HTTP/1.1 200 OK -``` - -```log -off=0 message begin -off=5 len=3 span[version]="1.1" -off=8 version complete -off=13 len=10 span[status]="No content" -off=25 status complete -off=25 len=14 span[header_field]="Content-Length" -off=40 header_field complete -off=41 len=1 span[header_value]="5" -off=44 header_value complete -off=44 len=10 span[header_field]="Connection" -off=55 header_field complete -off=56 len=5 span[header_value]="close" -off=63 header_value complete -off=65 headers complete status=200 v=1/1 flags=22 content_length=5 -off=65 len=5 span[body]="2ad73" -off=70 message complete -off=70 reset -off=70 message begin -off=75 len=3 span[version]="1.1" -off=78 version complete -off=83 len=2 span[status]="OK" -``` - ## HTTP/1.1 with keep-alive disabled and 204 status in lenient mode diff --git a/test/response/invalid.md b/test/response/invalid.md index 40eee9b1..d12e7914 100644 --- a/test/response/invalid.md +++ b/test/response/invalid.md @@ -3,9 +3,7 @@ Invalid responses ### Incomplete HTTP protocol -*TODO(indutny): test `req_or_res` mode too* - - + ```http HTP/1.1 200 OK @@ -152,7 +150,7 @@ off=18 error code=30 reason="Unexpected space after start line" ### Extra space between HTTP version and status code - + ```http HTTP/1.1 200 OK @@ -168,7 +166,7 @@ off=9 error code=13 reason="Invalid status code" ### Extra space between status code and reason - + ```http HTTP/1.1 200 OK @@ -186,7 +184,7 @@ off=20 headers complete status=200 v=1/1 flags=0 content_length=0 ### One-digit status code - + ```http HTTP/1.1 2 OK diff --git a/test/response/sample.md b/test/response/sample.md index e32fc5ef..9cf3bd0c 100644 --- a/test/response/sample.md +++ b/test/response/sample.md @@ -39,7 +39,7 @@ off=73 message complete Every response must start with `HTTP/`. - + ```http HTTPER/1.1 200 OK @@ -281,6 +281,25 @@ Connection: close\n\ these headers are from http://news.ycombinator.com/ ``` +```log +off=0 message begin +off=5 len=3 span[version]="1.1" +off=8 version complete +off=13 len=2 span[status]="OK" +off=16 error code=2 reason="Expected LF after CR" +``` + +## No carriage ret in lenient mode + + +```http +HTTP/1.1 200 OK\n\ +Content-Type: text/html; charset=utf-8\n\ +Connection: close\n\ +\n\ +these headers are from http://news.ycombinator.com/ +``` + ```log off=0 message begin off=5 len=3 span[version]="1.1" @@ -479,9 +498,9 @@ off=370 chunk complete off=370 message complete ``` -## Spaces in header name in strict mode +## Spaces in header name - + ```http HTTP/1.1 200 OK Server: Microsoft-IIS/6.0 @@ -512,63 +531,6 @@ off=67 header_value complete off=72 error code=10 reason="Invalid header token" ``` -## Spaces in header name in loose mode - -`en-US Content-Type` must be treated as a header name - - -```http -HTTP/1.1 200 OK -Server: Microsoft-IIS/6.0 -X-Powered-By: ASP.NET -en-US Content-Type: text/xml -Content-Type: text/xml -Content-Length: 16 -Date: Fri, 23 Jul 2010 18:45:38 GMT -Connection: keep-alive - -hello -``` - -```log -off=0 message begin -off=5 len=3 span[version]="1.1" -off=8 version complete -off=13 len=2 span[status]="OK" -off=17 status complete -off=17 len=6 span[header_field]="Server" -off=24 header_field complete -off=25 len=17 span[header_value]="Microsoft-IIS/6.0" -off=44 header_value complete -off=44 len=12 span[header_field]="X-Powered-By" -off=57 header_field complete -off=58 len=7 span[header_value]="ASP.NET" -off=67 header_value complete -off=67 len=18 span[header_field]="en-US Content-Type" -off=86 header_field complete -off=87 len=8 span[header_value]="text/xml" -off=97 header_value complete -off=97 len=12 span[header_field]="Content-Type" -off=110 header_field complete -off=111 len=8 span[header_value]="text/xml" -off=121 header_value complete -off=121 len=14 span[header_field]="Content-Length" -off=136 header_field complete -off=137 len=2 span[header_value]="16" -off=141 header_value complete -off=141 len=4 span[header_field]="Date" -off=146 header_field complete -off=147 len=29 span[header_value]="Fri, 23 Jul 2010 18:45:38 GMT" -off=178 header_value complete -off=178 len=10 span[header_field]="Connection" -off=189 header_field complete -off=190 len=10 span[header_value]="keep-alive" -off=202 header_value complete -off=204 headers complete status=200 v=1/1 flags=21 content_length=16 -off=204 len=16 span[body]="hello" -off=220 message complete -``` - ## Non ASCII in status line diff --git a/test/url.md b/test/url.md index 6f0a7ed4..13a1b015 100644 --- a/test/url.md +++ b/test/url.md @@ -238,46 +238,24 @@ off=7 len=14 span[url.host]="[fe80::a%eth0]" off=21 len=1 span[url.path]="/" ``` -## Disallow tab in URL in strict mode +## Disallow tab in URL - + ```url /foo\tbar/ ``` ```log -off=5 error code=7 reason="Invalid characters in url (strict mode)" +off=5 error code=7 reason="Invalid characters in url" ``` -## Tab in URL in loose mode +## Disallow form-feed in URL - -```url -/foo\tbar/ -``` - -```log -off=0 len=9 span[url.path]="/foo\tbar/" -``` - -## Disallow form-feed in URL in strict mode - - -```url -/foo\fbar/ -``` - -```log -off=5 error code=7 reason="Invalid characters in url (strict mode)" -``` - -## Form-feed in URL in loose mode - - + ```url /foo\fbar/ ``` ```log -off=0 len=9 span[url.path]="/foo\fbar/" +off=5 error code=7 reason="Invalid characters in url" ```