diff --git a/modules/ngx_http_upstream_check_module/ngx_http_upstream_check_module.c b/modules/ngx_http_upstream_check_module/ngx_http_upstream_check_module.c index fc617e480e..9d16e2bff3 100644 --- a/modules/ngx_http_upstream_check_module/ngx_http_upstream_check_module.c +++ b/modules/ngx_http_upstream_check_module/ngx_http_upstream_check_module.c @@ -8,6 +8,9 @@ #include #include +typedef struct ssl_ctx_st SSL_CTX; +typedef struct ssl_st SSL; +typedef struct bio_st BIO; typedef struct ngx_http_upstream_check_peer_s ngx_http_upstream_check_peer_t; typedef struct ngx_http_upstream_check_srv_conf_s @@ -68,6 +71,7 @@ typedef struct { #error "ngx_http_upstream_check_module needs structure packing pragma support" #endif +typedef struct SSL_Box SSL_Box; typedef struct { ngx_buf_t send; @@ -155,6 +159,8 @@ struct ngx_http_upstream_check_peer_s { ngx_http_upstream_check_peer_shm_t *shm; ngx_http_upstream_check_srv_conf_t *conf; + ngx_int_t phase; + SSL_Box *box; unsigned delete; }; @@ -170,11 +176,34 @@ typedef struct { } ngx_http_upstream_check_peers_t; +typedef void (*ngx_http_upstream_check_send_handler_https_pt) + (ngx_event_t *event, uint8_t *buf, size_t len); +typedef struct SSL_Box { + int fd; + SSL* _ssl; + SSL_CTX* ctx; + + int _is_flush; + int _buff_size; + BIO *_read_bio; + BIO *_write_bio; + uint8_t _buffer_send[1024 * 4]; + uint8_t _buffer_recv[1024 * 4]; + int _buffer_send_size; + int _buffer_recv_size; + + ngx_http_upstream_check_peer_t *peer; + ngx_http_upstream_check_packet_parse_pt parse; + ngx_http_upstream_check_send_handler_https_pt send; + ngx_event_t *event; +} SSL_Box; + #define NGX_HTTP_CHECK_TCP 0x0001 #define NGX_HTTP_CHECK_HTTP 0x0002 #define NGX_HTTP_CHECK_SSL_HELLO 0x0004 #define NGX_HTTP_CHECK_MYSQL 0x0008 #define NGX_HTTP_CHECK_AJP 0x0010 +#define NGX_HTTP_CHECK_HTTPS 0x0011 #define NGX_CHECK_HTTP_1XX 0x0002 #define NGX_CHECK_HTTP_2XX 0x0004 @@ -251,6 +280,7 @@ struct ngx_http_upstream_check_srv_conf_s { ngx_check_conf_t *check_type_conf; ngx_str_t send; + ngx_str_t server_name; union { ngx_uint_t return_code; @@ -365,6 +395,9 @@ static ngx_http_fastcgi_request_start_t ngx_http_fastcgi_request_start = { #define PEER_DELETING 0x01 #define PEER_DELETED 0x02 +static ngx_int_t ngx_http_upstream_check_https_send_hk(ngx_event_t *event); +static void ngx_http_upstream_check_https_send(ngx_event_t *event, uint8_t *buf, size_t len); +static ngx_buf_t *buffer_append_string(ngx_buf_t *b, u_char *s, size_t len, ngx_pool_t *pool); static ngx_uint_t ngx_http_upstream_check_add_dynamic_peer_shm( ngx_pool_t *pool, ngx_http_upstream_check_srv_conf_t *ucscf, @@ -387,6 +420,9 @@ static void ngx_http_upstream_check_peek_handler(ngx_event_t *event); static void ngx_http_upstream_check_send_handler(ngx_event_t *event); static void ngx_http_upstream_check_recv_handler(ngx_event_t *event); +static void ngx_http_upstream_check_https_send_handler(ngx_event_t *event); +static void ngx_http_upstream_check_https_recv_handler(ngx_event_t *event); + static void ngx_http_upstream_check_discard_handler(ngx_event_t *event); static void ngx_http_upstream_check_dummy_handler(ngx_event_t *event); @@ -400,6 +436,13 @@ static ngx_int_t ngx_http_upstream_check_parse_status_line( static void ngx_http_upstream_check_http_reinit( ngx_http_upstream_check_peer_t *peer); +static ngx_int_t ngx_http_upstream_check_https_init( + ngx_http_upstream_check_peer_t *peer); +static ngx_int_t ngx_http_upstream_check_https_parse( + ngx_http_upstream_check_peer_t *peer); +static void ngx_http_upstream_check_https_reinit( + ngx_http_upstream_check_peer_t *peer); + static ngx_buf_t *ngx_http_upstream_check_create_fastcgi_request( ngx_pool_t *pool, ngx_str_t *params, ngx_uint_t num); @@ -713,6 +756,17 @@ static ngx_check_conf_t ngx_check_types[] = { ngx_http_upstream_check_http_reinit, 1, 1 }, + { NGX_HTTP_CHECK_HTTPS, + ngx_string("https"), + ngx_string("GET / HTTP/1.0\r\n\r\n"), + NGX_CONF_BITMASK_SET | NGX_CHECK_HTTP_2XX | NGX_CHECK_HTTP_3XX, + ngx_http_upstream_check_https_send_handler, + ngx_http_upstream_check_https_recv_handler, + ngx_http_upstream_check_https_init, + ngx_http_upstream_check_https_parse, + ngx_http_upstream_check_https_reinit, + 1, + 1 }, { NGX_HTTP_CHECK_HTTP, ngx_string("fastcgi"), @@ -1742,6 +1796,140 @@ ngx_http_upstream_check_dummy_handler(ngx_event_t *event) } +static ngx_buf_t * +buffer_append_string(ngx_buf_t *b, u_char *s, size_t len, ngx_pool_t *pool) +{ + u_char *p; + ngx_uint_t capacity, size; + + if (len > (size_t) (b->end - b->last)) { + + size = b->last - b->pos; + + capacity = b->end - b->start; + capacity <<= 1; + + if (capacity < (size + len)) { + capacity = size + len; + } + + p = ngx_palloc(pool, capacity); + if (p == NULL) { + return NULL; + } + + b->last = ngx_copy(p, b->pos, size); + + b->start = b->pos = p; + b->end = p + capacity; + } + + b->last = ngx_copy(b->last, s, len); + + return b; +} + +static void +ngx_http_upstream_check_https_send(ngx_event_t *event, uint8_t *buf, size_t len) +{ + ssize_t size; + ngx_connection_t *c; + ngx_http_upstream_check_ctx_t *ctx; + ngx_http_upstream_check_peer_t *peer; + + if (ngx_http_upstream_check_need_exit()) { + return; + } + + c = event->data; + peer = c->data; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http check send."); + + if (c->pool == NULL) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "check pool NULL with peer: %V ", + &peer->check_peer_addr->name); + + goto check_send_fail; + } + + if (peer->state != NGX_HTTP_CHECK_CONNECT_DONE) { + if (ngx_handle_write_event(c->write, 0) != NGX_OK) { + + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "check handle write event error with peer: %V ", + &peer->check_peer_addr->name); + + goto check_send_fail; + } + + return; + } + + if (peer->check_data == NULL) { + + peer->check_data = ngx_pcalloc(peer->pool, + sizeof(ngx_http_upstream_check_ctx_t)); + if (peer->check_data == NULL) { + goto check_send_fail; + } + + if (peer->init == NULL || peer->init(peer) != NGX_OK) { + + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "check init error with peer: %V ", + &peer->check_peer_addr->name); + + goto check_send_fail; + } + } + + ctx = peer->check_data; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http check send total: %z", + ctx->send.last - ctx->send.pos); + + buffer_append_string(&ctx->send, buf, len, peer->pool); + + while (ctx->send.pos < ctx->send.last) { + + size = c->send(c, ctx->send.pos, ctx->send.last - ctx->send.pos); + +#if (NGX_DEBUG) + { + ngx_err_t err; + + err = (size >=0) ? 0 : ngx_socket_errno; + ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, err, + "http check send size: %z, total: %z", + size, ctx->send.last - ctx->send.pos); + } +#endif + + if (size >= 0) { + ctx->send.pos += size; + } else if (size == NGX_AGAIN) { + return; + } else { + c->error = 1; + goto check_send_fail; + } + } + + if (ctx->send.pos == ctx->send.last) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http check send done."); + c->requests++; + } + + return; + +check_send_fail: + ngx_http_upstream_check_status_update(peer, 0); + ngx_http_upstream_check_clean_event(peer); +} + static void ngx_http_upstream_check_send_handler(ngx_event_t *event) { @@ -1966,6 +2154,324 @@ ngx_http_upstream_check_recv_handler(ngx_event_t *event) } +static void +flush_rbio(SSL_Box* box) +{ + int total = 0; + int nread = 0; + + uint8_t buffer[1024 * 4] = {0}; + int buf_size = 1024 * 4 - 1; + + do { + nread = SSL_read(box->_ssl, buffer + total, buf_size - total); + if (nread > 0) { + total += nread; + } + } while (nread > 0 && buf_size - total > 0); + + if (!total) { + return; + } + + memcpy(box->_buffer_recv, buffer, total); + box->_buffer_recv_size = total; + + ngx_http_upstream_check_ctx_t *ctx = box->peer->check_data; + ctx->recv.pos = box->_buffer_recv; + ctx->recv.last = box->_buffer_recv + total; + + box->parse(box->peer); + + ctx->recv.pos = ctx->recv.last = ctx->recv.start; + box->_buffer_recv_size = 0; +} + + +static void +flush_wbio(SSL_Box* box) +{ + int total = 0; + int nread = 0; + + uint8_t buffer[1024 * 4] = {0}; + int buf_size = 1024 * 4 - 1; + + do { + nread = BIO_read(box->_write_bio, buffer + total, buf_size - total); + if (nread > 0) { + total += nread; + } + } while (nread > 0 && buf_size - total > 0); + + if (!total) { + return; + } + + box->send(box->event, buffer, total); +} + + +static void +box_flush(SSL_Box* box) +{ + if (box->_is_flush) { + return; + } + + box->_is_flush = 1; + + flush_rbio(box); + + if (!SSL_is_init_finished(box->_ssl)) { + flush_wbio(box); + box->_is_flush = 0; + return; + } + + do { + uint32_t offset = 0; + uint32_t size = box->_buffer_send_size; + while (offset < size) { + int nwrite = SSL_write(box->_ssl, box->_buffer_send + offset, size - offset); + if (nwrite > 0) { + offset += nwrite; + flush_wbio(box); + continue; + } + break; + } + + if (offset != size) { + break; + } + } while(0); + + box->_is_flush = 0; +} + + +static ngx_int_t +ngx_http_upstream_check_https_send_hk(ngx_event_t *event) +{ + ngx_connection_t *c; + ngx_http_upstream_check_peer_t *peer; + + SSL_Box *box = (SSL_Box*)malloc(sizeof(SSL_Box)); + + if (box == NULL) { + return NGX_ERROR; + } + + memset(box, 0x0, sizeof(SSL_Box)); + + box->_read_bio = BIO_new(BIO_s_mem()); + box->_write_bio = BIO_new(BIO_s_mem()); + + box->ctx = SSL_CTX_new(SSLv23_client_method()); + box->_ssl = SSL_new(box->ctx); + + SSL_set_bio(box->_ssl, box->_read_bio, box->_write_bio); + SSL_set_connect_state(box->_ssl); + + box->_buff_size = 1024 * 4; + + memset(box->_buffer_send, 0x0, 1024 * 4); + + c = event->data; + peer = c->data; + peer->box = box; + + box->peer = peer; + box->event = event; + + box->parse = ngx_http_upstream_check_https_parse; + box->send = ngx_http_upstream_check_https_send; + + if (peer->conf->server_name.len > 0) { + SSL_set_tlsext_host_name(box->_ssl, peer->conf->server_name.data); + } + + SSL_connect(box->_ssl); + + box->_is_flush = 0; + + box_flush(box); + + return NGX_OK; +} + + +static void +box_on_send(SSL_Box *box, uint8_t *buf, size_t size) +{ + if (size <= 0) { + return; + } + + memcpy(box->_buffer_send, buf, size); + box->_buffer_send_size = size; + box_flush(box); +} + + +static void +ngx_http_upstream_check_https_send_handler(ngx_event_t *event) +{ + ngx_connection_t *c; + ngx_http_upstream_check_peer_t *peer; + + if (ngx_http_upstream_check_need_exit()) { + return; + } + + c = event->data; + peer = c->data; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "https check send."); + + if (c->pool == NULL) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "check pool NULL with peer: %V ", + &peer->check_peer_addr->name); + + goto check_send_fail; + } + + if (peer->state != NGX_HTTP_CHECK_CONNECT_DONE) { + if (ngx_handle_write_event(c->write, 0) != NGX_OK) { + + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "check handle write event error with peer: %V ", + &peer->check_peer_addr->name); + + goto check_send_fail; + } + + return; + } + + if (peer->phase == 0) { + + ngx_http_upstream_check_srv_conf_t *ucscf; + ucscf = peer->conf; + + peer->phase = 1; + + if (ngx_http_upstream_check_https_send_hk(event) != NGX_OK) { + goto check_send_fail; + } + + box_on_send(peer->box, (u_char *)ucscf->send.data, ucscf->send.len); + } + + return; + +check_send_fail: + ngx_http_upstream_check_status_update(peer, 0); + ngx_http_upstream_check_clean_event(peer); +} + + +static void +box_on_recv(SSL_Box *box, uint8_t *buf, size_t size) +{ + if (size <= 0) { + return; + } + uint32_t offset = 0; + while (offset < size) { + int nwrite = BIO_write(box->_read_bio, buf + offset, size - offset); + if (nwrite > 0) { + offset += nwrite; + box_flush(box); + continue; + } + break; + } +} + + +static void +ngx_http_upstream_check_https_recv_handler(ngx_event_t *event) +{ + u_char *new_buf; + ssize_t size, n; + ngx_connection_t *c; + ngx_http_upstream_check_ctx_t *ctx; + ngx_http_upstream_check_peer_t *peer; + + if (ngx_http_upstream_check_need_exit()) { + return; + } + + c = event->data; + peer = c->data; + + ctx = peer->check_data; + + if (ctx->recv.start == NULL) { + /* half of the page_size, is it enough? */ + ctx->recv.start = ngx_palloc(c->pool, ngx_pagesize / 2); + if (ctx->recv.start == NULL) { + goto check_recv_fail; + } + + ctx->recv.last = ctx->recv.pos = ctx->recv.start; + ctx->recv.end = ctx->recv.start + ngx_pagesize / 2; + } + + while (1) { + n = ctx->recv.end - ctx->recv.last; + + /* buffer not big enough? enlarge it by twice */ + if (n == 0) { + size = ctx->recv.end - ctx->recv.start; + new_buf = ngx_palloc(c->pool, size * 2); + if (new_buf == NULL) { + goto check_recv_fail; + } + + ngx_memcpy(new_buf, ctx->recv.start, size); + + ctx->recv.pos = ctx->recv.start = new_buf; + ctx->recv.last = new_buf + size; + ctx->recv.end = new_buf + size * 2; + + n = ctx->recv.end - ctx->recv.last; + } + + size = c->recv(c, ctx->recv.last, n); + + if (size > 0) { + ctx->recv.last += size; + continue; + } else if (size == 0 || size == NGX_AGAIN) { + break; + } else { + c->error = 1; + goto check_recv_fail; + } + } + + box_on_recv(peer->box, ctx->recv.pos, ctx->recv.last - ctx->recv.pos); + + ctx->recv.pos = ctx->recv.last = ctx->recv.start; + + if (peer->state == NGX_HTTP_CHECK_RECV_DONE) { + ngx_http_upstream_check_clean_event(peer); + } + return; + +check_recv_fail: + + ngx_http_upstream_check_status_update(peer, 0); + ngx_http_upstream_check_clean_event(peer); + + return; +} + + static ngx_int_t ngx_http_upstream_check_http_init(ngx_http_upstream_check_peer_t *peer) { @@ -2052,6 +2558,130 @@ ngx_http_upstream_check_http_parse(ngx_http_upstream_check_peer_t *peer) } +static ngx_int_t +ngx_http_upstream_check_https_init(ngx_http_upstream_check_peer_t *peer) +{ + ngx_buf_t *b; + ngx_http_upstream_check_ctx_t *ctx; + + ctx = peer->check_data; + + b = ngx_create_temp_buf(peer->pool, 1024 * 4); + ctx->send = *b; + + ctx->recv.start = ctx->recv.pos = NULL; + ctx->recv.end = ctx->recv.last = NULL; + + ctx->state = 0; + + ngx_memzero(&ctx->status, sizeof(ngx_http_status_t)); + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_check_https_parse(ngx_http_upstream_check_peer_t *peer) +{ + ngx_int_t rc; + ngx_uint_t code, code_n; + ngx_http_upstream_check_ctx_t *ctx; + ngx_http_upstream_check_srv_conf_t *ucscf; + SSL_Box *box; + + ucscf = peer->conf; + ctx = peer->check_data; + box = peer->box; + + if ((ctx->recv.last - ctx->recv.pos) <= 0) { + rc = NGX_AGAIN; + goto parse_end; + } + + BIO_write(box->_read_bio, ctx->recv.pos, ctx->recv.last - ctx->recv.pos); + box_flush(box); + + if (box->_buffer_recv_size) { + + ctx->recv.pos = box->_buffer_recv; + ctx->recv.last = box->_buffer_recv + box->_buffer_recv_size; + + rc = ngx_http_upstream_check_parse_status_line(ctx, + &ctx->recv, + &ctx->status); + if (rc == NGX_AGAIN) { + goto parse_end; + } + + if (rc == NGX_ERROR) { + ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, + "http parse status line error with peer: %V ", + &peer->check_peer_addr->name); + goto parse_end; + } + + code = ctx->status.code; + + if (code > 0 && code < 200) { + code_n = NGX_CHECK_HTTP_1XX; + } else if (code >= 200 && code < 300) { + code_n = NGX_CHECK_HTTP_2XX; + } else if (code >= 300 && code < 400) { + code_n = NGX_CHECK_HTTP_3XX; + } else if (code >= 400 && code < 500) { + peer->pc.connection->error = 1; + code_n = NGX_CHECK_HTTP_4XX; + } else if (code >= 500 && code < 600) { + peer->pc.connection->error = 1; + code_n = NGX_CHECK_HTTP_5XX; + } else { + peer->pc.connection->error = 1; + code_n = NGX_CHECK_HTTP_ERR; + } + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, + "http_parse: code_n: %ui, conf: %ui", + code_n, ucscf->code.status_alive); + + if (code_n & ucscf->code.status_alive) { + rc = NGX_OK; + } else { + rc = NGX_ERROR; + } + } else { + rc = NGX_AGAIN; + } + +parse_end: + + switch (rc) { + + case NGX_AGAIN: + /* The peer has closed its half side of the connection. */ + /* so bad we will fall through */ + + case NGX_ERROR: + ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, + "check protocol %V error with peer: %V ", + &peer->conf->check_type_conf->name, + &peer->check_peer_addr->name); + + ngx_http_upstream_check_status_update(peer, 0); + break; + + case NGX_OK: + /* fall through */ + + default: + ngx_http_upstream_check_status_update(peer, 1); + break; + } + + peer->state = NGX_HTTP_CHECK_RECV_DONE; + + return NGX_OK; +} + static ngx_int_t ngx_http_upstream_check_fastcgi_process_record( ngx_http_upstream_check_ctx_t *ctx, ngx_buf_t *b, ngx_http_status_t *status) @@ -2782,6 +3412,46 @@ ngx_http_upstream_check_http_reinit(ngx_http_upstream_check_peer_t *peer) } +static void +ngx_http_upstream_check_https_free(SSL_Box *box) +{ + if (box == NULL) { + return; + } + if (box->_ssl) { + SSL_free(box->_ssl); + } + if (box->ctx) { + SSL_CTX_free(box->ctx); + } + + free(box); + return; +} + +static void +ngx_http_upstream_check_https_reinit(ngx_http_upstream_check_peer_t *peer) +{ + ngx_http_upstream_check_ctx_t *ctx; + + ctx = peer->check_data; + + ctx->send.pos = ctx->send.last = ctx->send.start; + + ctx->recv.pos = ctx->recv.last = ctx->recv.start; + + ctx->state = 0; + + peer->phase = 0; + + ngx_http_upstream_check_https_free(peer->box); + + peer->box = NULL; + + ngx_memzero(&ctx->status, sizeof(ngx_http_status_t)); +} + + static ngx_int_t ngx_http_upstream_check_ssl_hello_init(ngx_http_upstream_check_peer_t *peer) { @@ -3911,6 +4581,13 @@ ngx_http_upstream_check(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } + if (ngx_strncmp(value[i].data, "server_name=", 12) == 0) { + ucscf->server_name.len = value[i].len - 12; + ucscf->server_name.data = value[i].data + 12; + + continue; + } + goto invalid_check_parameter; } @@ -3993,8 +4670,9 @@ ngx_http_upstream_check_http_send(ngx_conf_t *cf, ngx_command_t *cmd, if (value[1].len && (ucscf->check_type_conf->name.len != 4 - || ngx_strncmp(ucscf->check_type_conf->name.data, - "http", 4) != 0)) + || ngx_strncmp(ucscf->check_type_conf->name.data, "http", 4) != 0) + && (ucscf->check_type_conf->name.len != 5 + || ngx_strncmp(ucscf->check_type_conf->name.data, "https", 5) != 0)) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid check_http_send for type \"%V\"",