diff --git a/README.md b/README.md index f2d460e..d8fb272 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ SELECT content::json->'headers'->>'Authorization' (1 row) ``` -Read the `status` and `content` fields out of a `http_response` object. +Read the `status` and `content_type` fields out of a `http_response` object. ```sql SELECT status, content_type @@ -248,7 +248,7 @@ As seen in the examples, you can unspool the array of `http_header` tuples into ## CURL Options -Select [CURL options](https://curl.se/libcurl/c/curl_easy_setopt.html) are available to set using the `http_set_curlopt(curlopt VARCHAR, value varchar)` function. +Select [CURL options](https://curl.se/libcurl/c/curl_easy_setopt.html) are available to set using the SQL `SET` command and the appropriate option name. * [CURLOPT_DNS_SERVERS](https://curl.se/libcurl/c/CURLOPT_DNS_SERVERS.html) * [CURLOPT_PROXY](https://curl.se/libcurl/c/CURLOPT_PROXY.html) @@ -279,53 +279,66 @@ Select [CURL options](https://curl.se/libcurl/c/curl_easy_setopt.html) are avail * [CURLOPT_CONNECTTIMEOUT_MS](https://curl.se/libcurl/c/CURLOPT_CONNECTTIMEOUT_MS.html) * [CURLOPT_USERAGENT](https://curl.se/libcurl/c/CURLOPT_USERAGENT.html) - For example, ```sql --- Set the PROXYPORT option -SELECT http_set_curlopt('CURLOPT_PROXYPORT', '12345'); +-- Set the curlopt_proxyport option +SET http.curlopt_proxyport = '12345'; + +-- View the curlopt_proxyport option +SHOW http.curlopt_proxyport; --- List all currently set options +-- View *all* currently set options SELECT * FROM http_list_curlopt(); ``` Will set the proxy port option for the lifetime of the database connection. You can reset all CURL options to their defaults using the `http_reset_curlopt()` function. -Using this extension as a background automated process without supervision (e.g as a trigger) may have unintended consequences for other servers. -It is considered a best practice to share contact information with your requests, -so that administrators can reach you in case your HTTP calls get out of control. +You can permanently set the CURL options for a database or role, using the `ALTER DATABASE` and `ALTER ROLE` commands. -Certain API policies (e.g. [Wikimedia User-Agent policy](https://meta.wikimedia.org/wiki/User-Agent_policy)) may even require sharing specific contact information -with each request. Others may disallow (via `robots.txt`) certain agents they don't recognize. +```sql +-- Applies to all roles in the database +ALTER DATABASE mydb SET http.curlopt_tlsauth_password = 'secret'; + +-- Applies to just one role in the database +ALTER ROLE myapp IN mydb SET http.curlopt_tlsauth_password = 'secret'; +``` + +## User Agents + +Using this extension as a background automated process without supervision (e.g as a trigger) may have unintended consequences for other servers. It is considered a best practice to share contact information with your requests, so that administrators can reach you in case your HTTP calls get out of control. + +Certain API policies (e.g. [Wikimedia User-Agent policy](https://meta.wikimedia.org/wiki/User-Agent_policy)) may even require sharing specific contact information with each request. Others may disallow (via `robots.txt`) certain agents they don't recognize. For such cases you can set the `CURLOPT_USERAGENT` option ```sql -SELECT http_set_curlopt('CURLOPT_USERAGENT', - 'Examplebot/2.1 (+http://www.example.com/bot.html) Contact abuse@example.com'); +SET http.curlopt_useragent = 'PgBot/2.1 (+http://pgbot.com/bot.html) Contact abuse@pgbot.com'; -SELECT status, content::json ->> 'user-agent' FROM http_get('http://httpbun.com/user-agent'); +SELECT status, content::json->'headers'->>'User-Agent' + FROM http_get('http://httpbun.com/headers'); ``` ``` status | user_agent --------+----------------------------------------------------------- - 200 | Examplebot/2.1 (+http://www.example.com/bot.html) Contact abuse@example.com + 200 | PgBot/2.1 (+http://pgbot.com/bot.html) Contact abuse@pgbot.com ``` ## Keep-Alive & Timeouts -*The `http_reset_curlopt()` approach described above is recommended. The global variables below will be deprecated and removed over time.* - By default each request uses a fresh connection and assures that the connection is closed when the request is done. This behavior reduces the chance of consuming system resources (sockets) as the extension runs over extended periods of time. High-performance applications may wish to enable keep-alive and connection persistence to reduce latency and enhance throughput. The following GUC variable changes the behavior of the http extension to maintain connections as long as possible: - http.keepalive = 'on' +```sql +SET http.curlopt_tcp_keepalive = 1; +``` By default a 5 second timeout is set for the completion of a request. If a different timeout is desired the following GUC variable can be used to set it in milliseconds: - http.timeout_msec = 200 +```sql +SET http.curlopt_timeout_msec = 200; +``` ## Installation @@ -364,15 +377,22 @@ PG_CONFIG = /usr/lib/postgresql/14/bin/pg_config There is a build available at [postgresonline](http://www.postgresonline.com/journal/archives/371-http-extension.html), not maintained by me. +### Testing + +The integration tests are run with `make install && make installcheck` and expect to find a running instance of [httpbin](http://httpbin.org) at port 9080. The easiest way to get that is: + +``` +docker run -p 9080:80 kennethreitz/httpbin +``` + ## Why This is a Bad Idea -- "What happens if the web page takes a long time to return?" Your SQL call will just wait there until it does. Make sure your web service fails fast. Or (dangerous in a different way) run your query within [pg_background](https://github.com/vibhorkum/pg_background). +- "What happens if the web page takes a long time to return?" Your SQL call will just wait there until it does. Make sure your web service fails fast. Or (dangerous in a different way) run your query within [pg_background](https://github.com/vibhorkum/pg_background) or on a schedule with [pg_cron](https://github.com/citusdata/pg_cron). - "What if the web page returns junk?" Your SQL call will have to test for junk before doing anything with the payload. - "What if the web page never returns?" Set a short timeout, or send a cancel to the request, or just wait forever. - "What if a user queries a page they shouldn't?" Restrict function access, or just don't install a footgun like this extension where users can access it. ## To Do -- The new http://www.postgresql.org/docs/9.3/static/bgworker.html background worker support could be used to set up an HTTP request queue, so that pgsql-http can register a request and callback and then return immediately. -- Inevitably some web server will return gzip content (Content-Encoding) without being asked for it. Handling that gracefully would be good. +- The [background worker](https://www.postgresql.org/docs/current/bgworker.html) support could be used to set up an HTTP request queue, so that pgsql-http can register a request and callback and then return immediately. diff --git a/http.c b/http.c index e978565..56c7c46 100644 --- a/http.c +++ b/http.c @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -103,27 +102,27 @@ typedef enum { } http_method; /* Components (and postitions) of the http_request tuple type */ -enum http_request_type { +enum { REQ_METHOD = 0, REQ_URI = 1, REQ_HEADERS = 2, REQ_CONTENT_TYPE = 3, REQ_CONTENT = 4 -}; +} http_request_type; /* Components (and postitions) of the http_response tuple type */ -enum http_response_type { +enum { RESP_STATUS = 0, RESP_CONTENT_TYPE = 1, RESP_HEADERS = 2, RESP_CONTENT = 3 -}; +} http_response_type; /* Components (and postitions) of the http_header tuple type */ -enum http_header_type { +enum { HEADER_FIELD = 0, HEADER_VALUE = 1 -}; +} http_header_type; /* * String/Long for strings and numbers, blob only for @@ -137,68 +136,70 @@ typedef enum { /* CURLOPT string/enum value mapping */ typedef struct { - char *curlopt_str; - char *curlopt_val; CURLoption curlopt; http_curlopt_type curlopt_type; bool superuser_only; + char *curlopt_str; + char *curlopt_val; + char *curlopt_guc; } http_curlopt; /* CURLOPT values we allow user to set at run-time */ /* Be careful adding these, as they can be a security risk */ static http_curlopt settable_curlopts[] = { - { "CURLOPT_CAINFO", NULL, CURLOPT_CAINFO, CURLOPT_STRING, false }, - { "CURLOPT_TIMEOUT", NULL, CURLOPT_TIMEOUT, CURLOPT_LONG, false }, - { "CURLOPT_TIMEOUT_MS", NULL, CURLOPT_TIMEOUT_MS, CURLOPT_LONG, false }, - { "CURLOPT_CONNECTTIMEOUT", NULL, CURLOPT_CONNECTTIMEOUT, CURLOPT_LONG, false }, - { "CURLOPT_CONNECTTIMEOUT_MS", NULL, CURLOPT_CONNECTTIMEOUT_MS, CURLOPT_LONG, false }, - { "CURLOPT_USERAGENT", NULL, CURLOPT_USERAGENT, CURLOPT_STRING, false }, - { "CURLOPT_USERPWD", NULL, CURLOPT_USERPWD, CURLOPT_STRING, false }, - { "CURLOPT_IPRESOLVE", NULL, CURLOPT_IPRESOLVE, CURLOPT_LONG, false }, + { CURLOPT_CAINFO, CURLOPT_STRING, false, "CURLOPT_CAINFO", NULL, NULL }, + { CURLOPT_TIMEOUT, CURLOPT_LONG, false, "CURLOPT_TIMEOUT", NULL, NULL }, + { CURLOPT_TIMEOUT_MS, CURLOPT_LONG, false, "CURLOPT_TIMEOUT_MS", NULL, NULL }, + { CURLOPT_CONNECTTIMEOUT, CURLOPT_LONG, false, "CURLOPT_CONNECTTIMEOUT", NULL, NULL }, + { CURLOPT_CONNECTTIMEOUT_MS, CURLOPT_LONG, false, "CURLOPT_CONNECTTIMEOUT_MS", NULL, NULL }, + { CURLOPT_USERAGENT, CURLOPT_STRING, false, "CURLOPT_USERAGENT", NULL, NULL }, + { CURLOPT_USERPWD, CURLOPT_STRING, false, "CURLOPT_USERPWD", NULL, NULL }, + { CURLOPT_IPRESOLVE, CURLOPT_LONG, false, "CURLOPT_IPRESOLVE", NULL, NULL }, #if LIBCURL_VERSION_NUM >= 0x070903 /* 7.9.3 */ - { "CURLOPT_SSLCERTTYPE", NULL, CURLOPT_SSLCERTTYPE, CURLOPT_STRING, false }, + { CURLOPT_SSLCERTTYPE, CURLOPT_STRING, false, "CURLOPT_SSLCERTTYPE", NULL, NULL }, #endif #if LIBCURL_VERSION_NUM >= 0x070e01 /* 7.14.1 */ - { "CURLOPT_PROXY", NULL, CURLOPT_PROXY, CURLOPT_STRING, false }, - { "CURLOPT_PROXYPORT", NULL, CURLOPT_PROXYPORT, CURLOPT_LONG, false }, + { CURLOPT_PROXY, CURLOPT_STRING, false, "CURLOPT_PROXY", NULL, NULL }, + { CURLOPT_PROXYPORT, CURLOPT_LONG, false, "CURLOPT_PROXYPORT", NULL, NULL }, #endif #if LIBCURL_VERSION_NUM >= 0x071301 /* 7.19.1 */ - { "CURLOPT_PROXYUSERNAME", NULL, CURLOPT_PROXYUSERNAME, CURLOPT_STRING, false }, - { "CURLOPT_PROXYPASSWORD", NULL, CURLOPT_PROXYPASSWORD, CURLOPT_STRING, false }, + { CURLOPT_PROXYUSERNAME, CURLOPT_STRING, false, "CURLOPT_PROXYUSERNAME", NULL, NULL }, + { CURLOPT_PROXYPASSWORD, CURLOPT_STRING, false, "CURLOPT_PROXYPASSWORD", NULL, NULL }, #endif #if LIBCURL_VERSION_NUM >= 0x071504 /* 7.21.4 */ - { "CURLOPT_TLSAUTH_USERNAME", NULL, CURLOPT_TLSAUTH_USERNAME, CURLOPT_STRING, false }, - { "CURLOPT_TLSAUTH_PASSWORD", NULL, CURLOPT_TLSAUTH_PASSWORD, CURLOPT_STRING, false }, - { "CURLOPT_TLSAUTH_TYPE", NULL, CURLOPT_TLSAUTH_TYPE, CURLOPT_STRING, false }, + { CURLOPT_TLSAUTH_USERNAME, CURLOPT_STRING, false, "CURLOPT_TLSAUTH_USERNAME", NULL, NULL }, + { CURLOPT_TLSAUTH_PASSWORD, CURLOPT_STRING, false, "CURLOPT_TLSAUTH_PASSWORD", NULL, NULL }, + { CURLOPT_TLSAUTH_TYPE, CURLOPT_STRING, false, "CURLOPT_TLSAUTH_TYPE", NULL, NULL }, #endif #if LIBCURL_VERSION_NUM >= 0x071800 /* 7.24.0 */ - { "CURLOPT_DNS_SERVERS", NULL, CURLOPT_DNS_SERVERS, CURLOPT_STRING, false }, + { CURLOPT_DNS_SERVERS, CURLOPT_STRING, false, "CURLOPT_DNS_SERVERS", NULL, NULL }, #endif #if LIBCURL_VERSION_NUM >= 0x071900 /* 7.25.0 */ - { "CURLOPT_TCP_KEEPALIVE", NULL, CURLOPT_TCP_KEEPALIVE, CURLOPT_LONG, false }, - { "CURLOPT_TCP_KEEPIDLE", NULL, CURLOPT_TCP_KEEPIDLE, CURLOPT_LONG, false }, + { CURLOPT_TCP_KEEPALIVE, CURLOPT_LONG, false, "CURLOPT_TCP_KEEPALIVE", NULL, NULL }, + { CURLOPT_TCP_KEEPIDLE, CURLOPT_LONG, false, "CURLOPT_TCP_KEEPIDLE", NULL, NULL }, #endif #if LIBCURL_VERSION_NUM >= 0x072500 /* 7.37.0 */ - { "CURLOPT_SSL_VERIFYHOST", NULL, CURLOPT_SSL_VERIFYHOST, CURLOPT_LONG, false }, - { "CURLOPT_SSL_VERIFYPEER", NULL, CURLOPT_SSL_VERIFYPEER, CURLOPT_LONG, false }, + { CURLOPT_SSL_VERIFYHOST, CURLOPT_LONG, false, "CURLOPT_SSL_VERIFYHOST", NULL, NULL }, + { CURLOPT_SSL_VERIFYPEER, CURLOPT_LONG, false, "CURLOPT_SSL_VERIFYPEER", NULL, NULL }, #endif - { "CURLOPT_SSLCERT", NULL, CURLOPT_SSLCERT, CURLOPT_STRING, false }, - { "CURLOPT_SSLKEY", NULL, CURLOPT_SSLKEY, CURLOPT_STRING, false }, + { CURLOPT_SSLCERT, CURLOPT_STRING, false, "CURLOPT_SSLCERT", NULL, NULL }, + { CURLOPT_SSLKEY, CURLOPT_STRING, false, "CURLOPT_SSLKEY", NULL, NULL }, #if LIBCURL_VERSION_NUM >= 0x073400 /* 7.52.0 */ - { "CURLOPT_PRE_PROXY", NULL, CURLOPT_PRE_PROXY, CURLOPT_STRING, false }, - { "CURLOPT_PROXY_CAINFO", NULL, CURLOPT_PROXY_TLSAUTH_USERNAME, CURLOPT_STRING, false }, - { "CURLOPT_PROXY_TLSAUTH_USERNAME", NULL, CURLOPT_PROXY_TLSAUTH_USERNAME, CURLOPT_STRING, false }, - { "CURLOPT_PROXY_TLSAUTH_PASSWORD", NULL, CURLOPT_PROXY_TLSAUTH_PASSWORD, CURLOPT_STRING, false }, - { "CURLOPT_PROXY_TLSAUTH_TYPE", NULL, CURLOPT_PROXY_TLSAUTH_TYPE, CURLOPT_STRING, false }, + { CURLOPT_PRE_PROXY, CURLOPT_STRING, false, "CURLOPT_PRE_PROXY", NULL, NULL }, + { CURLOPT_PROXY_TLSAUTH_USERNAME, CURLOPT_STRING, false, "CURLOPT_PROXY_CAINFO", NULL, NULL }, + { CURLOPT_PROXY_TLSAUTH_USERNAME, CURLOPT_STRING, false, "CURLOPT_PROXY_TLSAUTH_USERNAME", NULL, NULL }, + { CURLOPT_PROXY_TLSAUTH_PASSWORD, CURLOPT_STRING, false, "CURLOPT_PROXY_TLSAUTH_PASSWORD", NULL, NULL }, + { CURLOPT_PROXY_TLSAUTH_TYPE, CURLOPT_STRING, false, "CURLOPT_PROXY_TLSAUTH_TYPE", NULL, NULL }, #endif #if LIBCURL_VERSION_NUM >= 0x074700 /* 7.71.0 */ - { "CURLOPT_SSLKEY_BLOB", NULL, CURLOPT_SSLKEY_BLOB, CURLOPT_BLOB, false }, - { "CURLOPT_SSLCERT_BLOB", NULL, CURLOPT_SSLCERT_BLOB, CURLOPT_BLOB, false }, + { CURLOPT_SSLKEY_BLOB, CURLOPT_BLOB, false, "CURLOPT_SSLKEY_BLOB", NULL, NULL }, + { CURLOPT_SSLCERT_BLOB, CURLOPT_BLOB, false, "CURLOPT_SSLCERT_BLOB", NULL, NULL }, #endif - { NULL, NULL, 0, 0, false } /* Array null terminator */ + { 0, 0, false, NULL, NULL, NULL } /* Array null terminator */ }; + /* Function signatures */ void _PG_init(void); void _PG_fini(void); @@ -206,11 +207,7 @@ static size_t http_writeback(void *contents, size_t size, size_t nmemb, void *us static size_t http_readback(void *buffer, size_t size, size_t nitems, void *instream); /* Global variables */ -static bool g_use_keepalive; -static int g_timeout_msec; - -static CURL * g_http_handle = NULL; -// static List * g_curl_opts = NIL; +CURL * g_http_handle = NULL; /* * Interrupt support is dependent on CURLOPT_XFERINFOFUNCTION which is @@ -218,19 +215,43 @@ static CURL * g_http_handle = NULL; */ #if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ +pqsigfunc pgsql_interrupt_handler = NULL; +int http_interrupt_requested = 0; + /* * To support request interruption, we have libcurl run the progress meter -* callback frequently, and here we watch to see if PgSQL has flipped -* the global QueryCancelPending || ProcDiePending flags. -* Curl should then return CURLE_ABORTED_BY_CALLBACK -* to the curl_easy_perform() call. +* callback frequently, and here we watch to see if PgSQL has flipped our +* global 'http_interrupt_requested' flag. If it has been flipped, +* the non-zero return value will cue libcurl to abort the transfer, +* leading to a CURLE_ABORTED_BY_CALLBACK return on the curl_easy_perform() */ static int http_progress_callback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { - return QueryCancelPending || ProcDiePending; +#ifdef WIN32 + if (UNBLOCKED_SIGNAL_QUEUE()) + { + pgwin32_dispatch_queued_signals(); + } +#endif + /* elog(DEBUG3, "http_interrupt_requested = %d", http_interrupt_requested); */ + return http_interrupt_requested; } +/* +* We register this callback with the PgSQL signal handler to +* capture SIGINT and set our local interupt flag so that +* libcurl will eventually notice that a cancel is requested +*/ +static void +http_interrupt_handler(int sig) +{ + /* Handle the signal here */ + elog(DEBUG2, "http_interrupt_handler: sig=%d", sig); + http_interrupt_requested = sig; + pgsql_interrupt_handler(sig); + return; +} #endif /* 7.39.0 */ #undef HTTP_MEM_CALLBACKS @@ -270,32 +291,146 @@ http_malloc(size_t sz) #endif +static char * +http_strtolower(const char *input) +{ + char *ptr, *output; + if (input == NULL) + return NULL; + + /* Allocate memory for the output string */ + output = palloc(strlen(input) + 1); + ptr = output; + + while (*input) + { + *ptr++ = tolower((unsigned char) *input); + input++; + } + + *ptr = '\0'; // Null-terminate the string + return output; +} + + +#if PG_VERSION_NUM < 160000 +static void * +guc_malloc(int elevel, size_t size) +{ + void *data; + + /* Avoid unportable behavior of malloc(0) */ + if (size == 0) + size = 1; + data = malloc(size); + if (data == NULL) + ereport(elevel, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); + return data; +} + +static char * +guc_strdup(int elevel, const char *src) +{ + size_t len = strlen(src) + 1; + char *dup = guc_malloc(elevel, len); + memcpy(dup, src, len); + return dup; +} + +static void +guc_free(void *ptr) +{ + free(ptr); +} +#endif + + +static void +http_guc_init_opt(http_curlopt *opt) +{ + char *opt_name_lower = http_strtolower(opt->curlopt_str); + char *opt_name = psprintf("http.%s", opt_name_lower); + + const char *url_tmpl = "https://curl.se/libcurl/c/%s.html"; + char *opt_url = psprintf(url_tmpl, opt->curlopt_str); + opt->curlopt_guc = guc_strdup(ERROR, opt_name); + + DefineCustomStringVariable( + opt_name, // const char *name + guc_strdup(ERROR, opt_url), // const char *short_desc + NULL, // const char *long_desc + &(opt->curlopt_val), // char **valueAddr + NULL, // const char *bootValue + opt->superuser_only ? PGC_SUSET : PGC_USERSET, // GucContext context + 0, // int flags + NULL, // GucStringCheckHook check_hook + NULL, // GucStringAssignHook assign_hook + NULL // GucShowHook show_hook + ); + + pfree(opt_name_lower); + pfree(opt_name); + pfree(opt_url); + + /* + * Backwards compatibility, retain the old GUC + * http.keepalive name for now. + */ + if (opt->curlopt == CURLOPT_TCP_KEEPALIVE) + { + DefineCustomStringVariable( + "http.keepalive", + guc_strdup(ERROR, "https://curl.se/libcurl/c/CURLOPT_TCP_KEEPALIVE.html"), + NULL, + &(opt->curlopt_val), + NULL, + opt->superuser_only ? PGC_SUSET : PGC_USERSET, + 0, NULL, NULL, NULL + ); + } + + /* + * Backwards compatibility, retain the old GUC + * http.timeout_msec name for now. + */ + if (opt->curlopt == CURLOPT_TIMEOUT_MS) + { + DefineCustomStringVariable( + "http.timeout_msec", + guc_strdup(ERROR, "https://curl.se/libcurl/c/CURLOPT_TIMEOUT_MS.html"), + NULL, + &(opt->curlopt_val), + NULL, + opt->superuser_only ? PGC_SUSET : PGC_USERSET, + 0, NULL, NULL, NULL + ); + } +} + +static void +http_guc_init() +{ + http_curlopt *opt = settable_curlopts; + while (opt->curlopt) + { + http_guc_init_opt(opt); + opt++; + } +} + + /* Startup */ void _PG_init(void) { - DefineCustomBoolVariable("http.keepalive", - "reuse existing connections with keepalive", - NULL, - &g_use_keepalive, - false, - PGC_USERSET, - GUC_NOT_IN_SAMPLE, - NULL, - NULL, - NULL); - - DefineCustomIntVariable("http.timeout_msec", - "request completion timeout in milliseconds", - NULL, - &g_timeout_msec, - 0, - 0, - INT_MAX, - PGC_USERSET, - GUC_NOT_IN_SAMPLE | GUC_UNIT_MS, - NULL, - NULL, - NULL); + + /* + * Initialize the DefineCustomStringVariable GUC + * functions to allow "SET http.curlopt_var = value" + * to manipulate CURL options. + */ + http_guc_init(); #ifdef HTTP_MEM_CALLBACKS /* @@ -315,11 +450,23 @@ void _PG_init(void) #endif +#if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ + /* Register our interrupt handler (http_handle_interrupt) */ + /* and store the existing one so we can call it when we're */ + /* through with our work */ + pgsql_interrupt_handler = pqsignal(SIGINT, http_interrupt_handler); + http_interrupt_requested = 0; +#endif } /* Tear-down */ void _PG_fini(void) { +#if LIBCURL_VERSION_NUM >= 0x072700 + /* Re-register the original signal handler */ + pqsignal(SIGINT, pgsql_interrupt_handler); +#endif + if (g_http_handle) { curl_easy_cleanup(g_http_handle); @@ -640,12 +787,12 @@ typname_get_tupledesc(const char *extname, const char *typname) #if PG_VERSION_NUM >= 120000 typoid = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid, - PointerGetDatum(typname), - ObjectIdGetDatum(extschemaoid)); + PointerGetDatum(typname), + ObjectIdGetDatum(extschemaoid)); #else typoid = GetSysCacheOid2(TYPENAMENSP, - PointerGetDatum(typname), - ObjectIdGetDatum(extschemaoid)); + PointerGetDatum(typname), + ObjectIdGetDatum(extschemaoid)); #endif if ( OidIsValid(typoid) ) @@ -752,6 +899,21 @@ http_check_curl_version(const curl_version_info_data *version_info) } } + +static bool +curlopt_is_set(CURLoption curlopt) +{ + http_curlopt *opt = settable_curlopts; + while (opt->curlopt) + { + if (opt->curlopt == curlopt && opt->curlopt_val) + return true; + opt++; + } + return false; +} + + static bool set_curlopt(CURL* handle, const http_curlopt *opt) { @@ -764,7 +926,7 @@ set_curlopt(CURL* handle, const http_curlopt *opt) if (opt->curlopt_type == CURLOPT_STRING) { err = curl_easy_setopt(handle, opt->curlopt, opt->curlopt_val); - elog(DEBUG2, "pgsql-http: set '%s' to value '%s', return value = %d", opt->curlopt_str, opt->curlopt_val, err); + elog(DEBUG2, "pgsql-http: set '%s' to value '%s', return value = %d", opt->curlopt_guc, opt->curlopt_val, err); } /* Argument is a long */ else if (opt->curlopt_type == CURLOPT_LONG) @@ -773,12 +935,11 @@ set_curlopt(CURL* handle, const http_curlopt *opt) errno = 0; value_long = strtol(opt->curlopt_val, NULL, 10); if ( errno == EINVAL || errno == ERANGE ) - elog(ERROR, "invalid integer provided for '%s'", opt->curlopt_str); + elog(ERROR, "invalid integer provided for '%s'", opt->curlopt_guc); err = curl_easy_setopt(handle, opt->curlopt, value_long); - elog(DEBUG2, "pgsql-http: set '%s' to value '%ld', return value = %d", opt->curlopt_str, value_long, err); + elog(DEBUG2, "pgsql-http: set '%s' to value '%ld', return value = %d", opt->curlopt_guc, value_long, err); } -#if LIBCURL_VERSION_NUM >= 0x074700 /* 7.71.0 */ /* Only used for CURLOPT_SSLKEY_BLOB and CURLOPT_SSLCERT_BLOB */ else if (opt->curlopt_type == CURLOPT_BLOB) { @@ -791,13 +952,12 @@ set_curlopt(CURL* handle, const http_curlopt *opt) elog(DEBUG2, "pgsql-http: set 'CURLOPT_SSLKEYTYPE' to value 'PEM', return value = %d", err); err = curl_easy_setopt(handle, opt->curlopt, &blob); - elog(DEBUG2, "pgsql-http: set '%s' to value '%s', return value = %d", opt->curlopt_str, opt->curlopt_val, err); + elog(DEBUG2, "pgsql-http: set '%s' to value '%s', return value = %d", opt->curlopt_guc, opt->curlopt_val, err); } -#endif else { /* Never get here */ - elog(ERROR, "invalid curlopt_type"); + elog(ERROR, "invalid curlopt_type, '%d'", opt->curlopt_type); } if ( err != CURLE_OK ) @@ -812,9 +972,8 @@ set_curlopt(CURL* handle, const http_curlopt *opt) static CURL * http_get_handle() { - http_curlopt opt; CURL *handle = g_http_handle; - size_t i = 0; + http_curlopt *opt = settable_curlopts; /* Initialize the global handle if needed */ if (!handle) @@ -833,20 +992,19 @@ http_get_handle() curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 1000); curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, 5000); - /* Set the user agent. If not set, use PG_VERSION as default */ - curl_easy_setopt(handle, CURLOPT_USERAGENT, PG_VERSION_STR); + /* Set the user agent. If not set, use PG_VERSION as default */ + curl_easy_setopt(handle, CURLOPT_USERAGENT, PG_VERSION_STR); if (!handle) ereport(ERROR, (errmsg("Unable to initialize CURL"))); /* Bring in any options the user has set this session */ - while (1) + while (opt->curlopt) { - opt = settable_curlopts[i++]; - if (!opt.curlopt_str) break; /* Option value is already set */ - if (opt.curlopt_val) - set_curlopt(handle, &opt); + if (opt->curlopt_val) + set_curlopt(handle, opt); + opt++; } g_http_handle = handle; @@ -861,18 +1019,17 @@ Datum http_reset_curlopt(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(http_reset_curlopt); Datum http_reset_curlopt(PG_FUNCTION_ARGS) { - size_t i = 0; + http_curlopt *opt = settable_curlopts; /* Set up global HTTP handle */ CURL * handle = http_get_handle(); curl_easy_reset(handle); /* Clean out the settable_curlopts global cache */ - while (1) + while (opt->curlopt) { - http_curlopt *opt = settable_curlopts + i++; - if (!opt->curlopt_str) break; - if (opt->curlopt_val) pfree(opt->curlopt_val); + if (opt->curlopt_val) guc_free(opt->curlopt_val); opt->curlopt_val = NULL; + opt++; } PG_RETURN_BOOL(true); @@ -883,7 +1040,7 @@ PG_FUNCTION_INFO_V1(http_list_curlopt); Datum http_list_curlopt(PG_FUNCTION_ARGS) { struct list_state { - size_t i; /* read position */ + size_t i; /* read position */ }; MemoryContext oldcontext, newcontext; @@ -894,25 +1051,25 @@ Datum http_list_curlopt(PG_FUNCTION_ARGS) if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); + funcctx = SRF_FIRSTCALL_INIT(); newcontext = funcctx->multi_call_memory_ctx; oldcontext = MemoryContextSwitchTo(newcontext); state = palloc0(sizeof(*state)); funcctx->user_fctx = state; if(get_call_result_type(fcinfo, 0, &funcctx->tuple_desc) != TYPEFUNC_COMPOSITE) - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("composite-returning function called in context that cannot accept a composite"))); + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("composite-returning function called in context that cannot accept a composite"))); - BlessTupleDesc(funcctx->tuple_desc); - MemoryContextSwitchTo(oldcontext); - } + BlessTupleDesc(funcctx->tuple_desc); + MemoryContextSwitchTo(oldcontext); + } - funcctx = SRF_PERCALL_SETUP(); - state = funcctx->user_fctx; + funcctx = SRF_PERCALL_SETUP(); + state = funcctx->user_fctx; while (1) { - Datum result; + Datum result; HeapTuple tuple; text *option, *value; http_curlopt *opt = settable_curlopts + state->i++; @@ -925,8 +1082,8 @@ Datum http_list_curlopt(PG_FUNCTION_ARGS) vals[0] = PointerGetDatum(option); vals[1] = PointerGetDatum(value); nulls[0] = nulls[1] = 0; - tuple = heap_form_tuple(funcctx->tuple_desc, vals, nulls); - result = HeapTupleGetDatum(tuple); + tuple = heap_form_tuple(funcctx->tuple_desc, vals, nulls); + result = HeapTupleGetDatum(tuple); SRF_RETURN_NEXT(funcctx, result); } @@ -940,10 +1097,10 @@ Datum http_set_curlopt(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(http_set_curlopt); Datum http_set_curlopt(PG_FUNCTION_ARGS) { - size_t i = 0; char *curlopt, *value; text *curlopt_txt, *value_txt; CURL *handle; + http_curlopt *opt = settable_curlopts; /* Version check */ http_check_curl_version(curl_version_info(CURLVERSION_NOW)); @@ -961,16 +1118,15 @@ Datum http_set_curlopt(PG_FUNCTION_ARGS) curlopt = text_to_cstring(curlopt_txt); value = text_to_cstring(value_txt); - while (1) + while (opt->curlopt) { - http_curlopt *opt = settable_curlopts + i++; - if (!opt->curlopt_str) break; if (strcasecmp(opt->curlopt_str, curlopt) == 0) { - if (opt->curlopt_val) pfree(opt->curlopt_val); - opt->curlopt_val = MemoryContextStrdup(CacheMemoryContext, value); + if (opt->curlopt_val) guc_free(opt->curlopt_val); + opt->curlopt_val = guc_strdup(ERROR, value); PG_RETURN_BOOL(set_curlopt(handle, opt)); } + opt++; } elog(ERROR, "curl option '%s' is not available for run-time configuration", curlopt); @@ -1018,6 +1174,11 @@ Datum http_request(PG_FUNCTION_ARGS) /* Output */ HeapTuple tuple_out; +#if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ + /* Set up the interrupt flag */ + http_interrupt_requested = 0; +#endif + /* Version check */ http_check_curl_version(curl_version_info(CURLVERSION_NOW)); @@ -1087,8 +1248,7 @@ Datum http_request(PG_FUNCTION_ARGS) CURL_SETOPT(g_http_handle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); #endif - - if ( g_use_keepalive ) + if ( curlopt_is_set(CURLOPT_TCP_KEEPALIVE) ) { /* Keep sockets held open */ CURL_SETOPT(g_http_handle, CURLOPT_FORBID_REUSE, 0); @@ -1114,10 +1274,6 @@ Datum http_request(PG_FUNCTION_ARGS) CURL_SETOPT(g_http_handle, CURLOPT_NOPROGRESS, 0); #endif - /* Set up the HTTP timeout */ - if (g_timeout_msec > 0) - CURL_SETOPT(g_http_handle, CURLOPT_TIMEOUT_MS, g_timeout_msec); - /* Set the HTTP content encoding to all curl supports */ CURL_SETOPT(g_http_handle, CURLOPT_ACCEPT_ENCODING, ""); @@ -1128,7 +1284,7 @@ Datum http_request(PG_FUNCTION_ARGS) CURL_SETOPT(g_http_handle, CURLOPT_MAXREDIRS, 5); } - if ( g_use_keepalive ) + if ( curlopt_is_set(CURLOPT_TCP_KEEPALIVE) ) { /* Add a keep alive option to the headers to reuse network sockets */ headers = curl_slist_append(headers, "Connection: Keep-Alive"); @@ -1256,8 +1412,23 @@ Datum http_request(PG_FUNCTION_ARGS) curl_easy_cleanup(g_http_handle); g_http_handle = NULL; - if (http_return == CURLE_ABORTED_BY_CALLBACK) - elog(ERROR, "canceling statement due to user request"); +#if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ + /* + * If the request was aborted by an interrupt request + * we need to ensure that the interrupt signal + * is in turn sent to the downstream interrupt handler + * that we stored when we set up our own handler. + */ + if (http_return == CURLE_ABORTED_BY_CALLBACK && + pgsql_interrupt_handler && + http_interrupt_requested) + { + elog(DEBUG2, "calling pgsql_interrupt_handler"); + (*pgsql_interrupt_handler)(http_interrupt_requested); + http_interrupt_requested = 0; + elog(ERROR, "HTTP request cancelled"); + } +#endif http_error(http_return, http_error_buffer); } @@ -1273,10 +1444,10 @@ Datum http_request(PG_FUNCTION_ARGS) } /* Prepare our return object */ - if (get_call_result_type(fcinfo, 0, &tup_desc) != TYPEFUNC_COMPOSITE) { - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("%s called with incompatible return type", __func__))); - } + if (get_call_result_type(fcinfo, 0, &tup_desc) != TYPEFUNC_COMPOSITE) { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("%s called with incompatible return type", __func__))); + } ncolumns = tup_desc->natts; values = palloc0(sizeof(Datum)*ncolumns); @@ -1369,7 +1540,7 @@ Datum http_request(PG_FUNCTION_ARGS) /* Clean up */ ReleaseTupleDesc(tup_desc); - if ( !g_use_keepalive ) + if ( ! curlopt_is_set(CURLOPT_TCP_KEEPALIVE) ) { curl_easy_cleanup(g_http_handle); g_http_handle = NULL; @@ -1527,11 +1698,11 @@ Datum urlencode_jsonb(PG_FUNCTION_ARGS) /* Read the value for this key */ #if PG_VERSION_NUM < 130000 { - JsonbValue k; - k.type = jbvString; - k.val.string.val = key; - k.val.string.len = strlen(key); - v = *findJsonbValueFromContainer(&jb->root, JB_FOBJECT, &k); + JsonbValue k; + k.type = jbvString; + k.val.string.val = key; + k.val.string.len = strlen(key); +v = *findJsonbValueFromContainer(&jb->root, JB_FOBJECT, &k); } #else getKeyJsonValueFromContainer(&jb->root, key, strlen(key), &v);