pax_global_header00006660000000000000000000000064144652066320014522gustar00rootroot0000000000000052 comment=d0ca28df6121f10cd3868744d219904c7923c5be pgsql-http-1.6.0/000077500000000000000000000000001446520663200136315ustar00rootroot00000000000000pgsql-http-1.6.0/.editorconfig000066400000000000000000000005211446520663200163040ustar00rootroot00000000000000# http://editorconfig.org # top-most EditorConfig file root = true # these are the defaults [*] charset = utf-8 end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true # C files want tab indentation [*.{c,h}] indent_style = tab # YAML files want space indentation [*.{yml}] indent_style = space indent_size = 4 pgsql-http-1.6.0/.github/000077500000000000000000000000001446520663200151715ustar00rootroot00000000000000pgsql-http-1.6.0/.github/workflows/000077500000000000000000000000001446520663200172265ustar00rootroot00000000000000pgsql-http-1.6.0/.github/workflows/ci.yml000066400000000000000000000034041446520663200203450ustar00rootroot00000000000000# GitHub Actions for PostGIS # # Paul Ramsey name: "CI" on: [push, pull_request] jobs: linux: runs-on: ubuntu-latest name: "CI" strategy: matrix: ci: - { PGVER: 12 } - { PGVER: 13 } - { PGVER: 14 } - { PGVER: 15 } - { PGVER: 16 } steps: - name: 'Check Out' uses: actions/checkout@v3 - name: 'Install PostgreSQL' run: | sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg-snapshot main ${{ matrix.ci.PGVER }}" > /etc/apt/sources.list.d/pgdg.list' curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null sudo apt-get update sudo apt-get -y install postgresql-${{ matrix.ci.PGVER }} postgresql-server-dev-${{ matrix.ci.PGVER }} - name: 'Install Curl' run: | sudo apt-get -y install libcurl4-gnutls-dev - name: 'Start PostgreSQL' run: | export PGDATA=/var/lib/postgresql/${{ matrix.ci.PGVER }}/main export PGETC=/etc/postgresql/${{ matrix.ci.PGVER }}/main export PGBIN=/usr/lib/postgresql/${{ matrix.ci.PGVER }}/bin sudo cp ./ci/pg_hba.conf $PGETC/pg_hba.conf sudo su postgres -c "$PGBIN/pg_ctl --pgdata $PGDATA start -o '-c config_file=$PGETC/postgresql.conf -p 5432'" - name: 'Build & Test' run: | export PATH=/usr/lib/postgresql/${{ matrix.ci.PGVER }}/bin/:$PATH export PG_CONFIG=/usr/lib/postgresql/${{ matrix.ci.PGVER }}/bin/pg_config export PG_CFLAGS=-Werror make sudo -E make PG_CONFIG=$PG_CONFIG install PGUSER=postgres make installcheck || (cat regression.diffs && /bin/false) pgsql-http-1.6.0/.gitignore000066400000000000000000000001561446520663200156230ustar00rootroot00000000000000*.o *.so *.a *.pc *.dylib regression.diffs regression.out results/ tmp_check/ tmp_check_iso/ output_iso/ log/ pgsql-http-1.6.0/LICENSE.md000066400000000000000000000020731446520663200152370ustar00rootroot00000000000000Copyright (C) 2012 Paul Ramsey Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. pgsql-http-1.6.0/META.json000066400000000000000000000023331446520663200152530ustar00rootroot00000000000000{ "name": "http", "abstract": "HTTP client for PostgreSQL", "description": "HTTP allows you to get the content of a web page in a SQL function call.", "version": "1.6.0", "maintainer": [ "Paul Ramsey " ], "license": { "mit": "http://en.wikipedia.org/wiki/MIT_License" }, "prereqs": { "runtime": { "requires": { "PostgreSQL": "9.1.0" }, "recommends": { "PostgreSQL": "9.1.3" } } }, "provides": { "http": { "file": "http--1.6.sql", "docfile": "README.md", "version": "1.6.0", "abstract": "HTTP client for PostgreSQL" } }, "resources": { "homepage": "https://github.com/pramsey/pgsql-http/", "bugtracker": { "web": "https://github.com/pramsey/pgsql-http/issues" }, "repository": { "url": "https://github.com/pramsey/pgsql-http.git", "web": "https://github.com/pramsey/pgsql-http/", "type": "git" } }, "generated_by": "Paul Ramsey", "meta-spec": { "version": "1.0.0", "url": "http://pgxn.org/meta/spec.txt" }, "tags": [ "http", "curl", "web" ] } pgsql-http-1.6.0/Makefile000066400000000000000000000005471446520663200152770ustar00rootroot00000000000000 MODULE_big = http OBJS = http.o EXTENSION = http DATA = $(wildcard *.sql) REGRESS = http EXTRA_CLEAN = CURL_CONFIG = curl-config PG_CONFIG = pg_config CFLAGS += $(shell $(CURL_CONFIG) --cflags) LIBS += $(shell $(CURL_CONFIG) --libs) SHLIB_LINK := $(LIBS) ifdef DEBUG COPT += -O0 -Werror -g endif PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) pgsql-http-1.6.0/README.md000066400000000000000000000337771446520663200151310ustar00rootroot00000000000000# PostgreSQL HTTP Client [![CI](https://github.com/pramsey/pgsql-http/workflows/CI/badge.svg?branch=master)](https://github.com/pramsey/pgsql-http/actions?query=branch%3Amaster) ## Motivation Wouldn't it be nice to be able to write a trigger that called a web service? Either to get back a result, or to poke that service into refreshing itself against the new state of the database? This extension is for that. ## Examples URL encode a string. ```sql SELECT urlencode('my special string''s & things?'); ``` ``` urlencode ------------------------------------- my+special+string%27s+%26+things%3F (1 row) ``` URL encode a JSON associative array. ```sql SELECT urlencode(jsonb_build_object('name','Colin & James','rate','50%')); ``` ``` urlencode ------------------------------------- name=Colin+%26+James&rate=50%25 (1 row) ``` Run a GET request and see the content. ```sql SELECT content FROM http_get('http://httpbun.org/ip'); ``` ``` content ----------------------------- {"origin":"24.69.186.43"} (1 row) ``` Run a GET request with an Authorization header. ```sql SELECT content::json->'headers'->>'Authorization' FROM http(( 'GET', 'http://httpbun.org/headers', ARRAY[http_header('Authorization','Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9')], NULL, NULL )::http_request); ``` ``` content ---------------------------------------------- Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9 (1 row) ``` Read the `status` and `content` fields out of a `http_response` object. ```sql SELECT status, content_type FROM http_get('http://httpbun.org/'); ``` ``` status | content_type --------+-------------------------- 200 | text/html; charset=utf-8 (1 row) ``` Show all the `http_header` in an `http_response` object. ```sql SELECT (unnest(headers)).* FROM http_get('http://httpbun.org/'); ``` ``` field | value ------------------+-------------------------------------------------- Server | nginx Date | Wed, 26 Jul 2023 19:52:51 GMT Content-Type | text/html Content-Length | 162 Connection | close Location | https://httpbun.org server | nginx date | Wed, 26 Jul 2023 19:52:51 GMT content-type | text/html x-powered-by | httpbun/3c0dc05883dd9212ac38b04705037d50b02f2596 content-encoding | gzip ``` Use the PUT command to send a simple text document to a server. ```sql SELECT status, content_type, content::json->>'data' AS data FROM http_put('http://httpbun.org/put', 'some text', 'text/plain'); ``` ``` status | content_type | data --------+------------------+----------- 200 | application/json | some text ``` Use the PATCH command to send a simple JSON document to a server. ```sql SELECT status, content_type, content::json->>'data' AS data FROM http_patch('http://httpbun.org/patch', '{"this":"that"}', 'application/json'); ``` ``` status | content_type | data --------+------------------+------------------ 200 | application/json | '{"this":"that"}' ``` Use the DELETE command to request resource deletion. ```sql SELECT status, content_type, content::json->>'url' AS url FROM http_delete('http://httpbun.org/delete'); ``` ``` status | content_type | url --------+------------------+--------------------------- 200 | application/json | http://httpbun.org/delete ``` As a shortcut to send data to a GET request, pass a JSONB data argument. ```sql SELECT status, content::json->'args' AS args FROM http_get('http://httpbun.org/get', jsonb_build_object('myvar','myval','foo','bar')); ``` To POST to a URL using a data payload instead of parameters embedded in the URL, encode the data in a JSONB as a data payload. ```sql SELECT status, content::json->'form' AS form FROM http_post('http://httpbun.org/post', jsonb_build_object('myvar','myval','foo','bar')); ``` To access binary content, you must coerce the content from the default `varchar` representation to a `bytea` representation using the `text_to_bytea` function. Using the default `varchar::bytea` cast will **not work**, as the cast will stop the first time it hits a zero-valued byte (common in binary data). ```sql WITH http AS ( SELECT * FROM http_get('https://httpbingo.org/image/png') ), headers AS ( SELECT (unnest(headers)).* FROM http ) SELECT http.content_type, length(text_to_bytea(http.content)) AS length_binary FROM http, headers WHERE field ilike 'Content-Type'; ``` ``` content_type | length_binary --------------+--------------- image/png | 8090 ``` Similarly, when using POST to send `bytea` binary content to a service, use the `bytea_to_text` function to prepare the content. To access only the headers you can do a HEAD-Request. This will not follow redirections. ```sql SELECT http.status, headers.value AS location FROM http_head('http://google.com') AS http LEFT OUTER JOIN LATERAL (SELECT value FROM unnest(http.headers) WHERE field = 'Location') AS headers ON true; ``` ``` status | location --------+------------------------ 301 | http://www.google.com/ ``` ## Concepts Every HTTP call is a made up of an `http_request` and an `http_response`. Composite type "public.http_request" Column | Type | Modifiers --------------+-------------------+----------- method | http_method | uri | character varying | headers | http_header[] | content_type | character varying | content | character varying | Composite type "public.http_response" Column | Type | Modifiers --------------+-------------------+----------- status | integer | content_type | character varying | headers | http_header[] | content | character varying | The utility functions, `http_get()`, `http_post()`, `http_put()`, `http_delete()` and `http_head()` are just wrappers around a master function, `http(http_request)` that returns `http_response`. The `headers` field for requests and response is a PostgreSQL array of type `http_header` which is just a simple tuple. Composite type "public.http_header" Column | Type | Modifiers --------+-------------------+----------- field | character varying | value | character varying | As seen in the examples, you can unspool the array of `http_header` tuples into a result set using the PostgreSQL `unnest()` function on the array. From there you select out the particular header you are interested in. ## Functions * `http_header(field VARCHAR, value VARCHAR)` returns `http_header` * `http(request http_request)` returns `http_response` * `http_get(uri VARCHAR)` returns `http_response` * `http_get(uri VARCHAR, data JSONB)` returns `http_response` * `http_post(uri VARCHAR, content VARCHAR, content_type VARCHAR)` returns `http_response` * `http_post(uri VARCHAR, data JSONB)` returns `http_response` * `http_put(uri VARCHAR, content VARCHAR, content_type VARCHAR)` returns `http_response` * `http_patch(uri VARCHAR, content VARCHAR, content_type VARCHAR)` returns `http_response` * `http_delete(uri VARCHAR, content VARCHAR, content_type VARCHAR))` returns `http_response` * `http_head(uri VARCHAR)` returns `http_response` * `http_set_curlopt(curlopt VARCHAR, value varchar)` returns `boolean` * `http_reset_curlopt()` returns `boolean` * `http_list_curlopt()` returns `setof(curlopt text, value text)` * `urlencode(string VARCHAR)` returns `text` * `urlencode(data JSONB)` returns `text` ## CURL Options Select [CURL options](https://curl.haxx.se/libcurl/c/curl_easy_setopt.html) are available to set using the `http_set_curlopt(curlopt VARCHAR, value varchar)` function. * [CURLOPT_DNS_SERVERS](https://curl.haxx.se/libcurl/c/CURLOPT_DNS_SERVERS.html) * [CURLOPT_PROXY](https://curl.haxx.se/libcurl/c/CURLOPT_PROXY.html) * [CURLOPT_PRE_PROXY](https://curl.haxx.se/libcurl/c/CURLOPT_PRE_PROXY.html) * [CURLOPT_PROXYPORT](https://curl.haxx.se/libcurl/c/CURLOPT_PROXYPORT.html) * [CURLOPT_PROXYUSERPWD](https://curl.haxx.se/libcurl/c/CURLOPT_PROXYUSERPWD.html) * [CURLOPT_PROXYUSERNAME](https://curl.haxx.se/libcurl/c/CURLOPT_PROXYUSERNAME.html) * [CURLOPT_PROXYPASSWORD](https://curl.haxx.se/libcurl/c/CURLOPT_PROXYPASSWORD.html) * [CURLOPT_PROXY_TLSAUTH_USERNAME](https://curl.haxx.se/libcurl/c/CURLOPT_PROXY_TLSAUTH_USERNAME.html) * [CURLOPT_PROXY_TLSAUTH_PASSWORD](https://curl.haxx.se/libcurl/c/CURLOPT_PROXY_TLSAUTH_PASSWORD.html) * [CURLOPT_PROXY_TLSAUTH_TYPE](https://curl.haxx.se/libcurl/c/CURLOPT_PROXY_TLSAUTH_TYPE.html) * [CURLOPT_TLSAUTH_USERNAME](https://curl.haxx.se/libcurl/c/CURLOPT_TLSAUTH_USERNAME.html) * [CURLOPT_TLSAUTH_PASSWORD](https://curl.haxx.se/libcurl/c/CURLOPT_TLSAUTH_PASSWORD.html) * [CURLOPT_TLSAUTH_TYPE](https://curl.haxx.se/libcurl/c/CURLOPT_TLSAUTH_TYPE.html) * [CURLOPT_SSL_VERIFYHOST](https://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html) * [CURLOPT_SSL_VERIFYPEER](https://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYPEER.html) * [CURLOPT_SSLCERT](https://curl.haxx.se/libcurl/c/CURLOPT_SSLCERT.html) * [CURLOPT_SSLKEY](https://curl.haxx.se/libcurl/c/CURLOPT_SSLKEY.html) * [CURLOPT_SSLCERTTYPE](https://curl.haxx.se/libcurl/c/CURLOPT_SSLCERTTYPE.html) * [CURLOPT_CAINFO](https://curl.haxx.se/libcurl/c/CURLOPT_CAINFO.html) * [CURLOPT_TIMEOUT](https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT.html) * [CURLOPT_TIMEOUT_MS](https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT_MS.html) * [CURLOPT_TCP_KEEPALIVE](https://curl.haxx.se/libcurl/c/CURLOPT_TCP_KEEPALIVE.html) * [CURLOPT_TCP_KEEPIDLE](https://curl.haxx.se/libcurl/c/CURLOPT_TCP_KEEPIDLE.html) * [CURLOPT_CONNECTTIMEOUT](https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html) * [CURLOPT_USERAGENT](https://curl.haxx.se/libcurl/c/CURLOPT_USERAGENT.html) For example, ```sql -- Set the PROXYPORT option SELECT http_set_curlopt('CURLOPT_PROXYPORT', '12345'); -- List all currently set options SELECT * FROM http_list_curlopt(); ``` Will set the proxy port option for the lifetime of the database connection. You can reset all CURL options to their defaults using the `http_reset_curlopt()` function. Using this extension as a background automated process without supervision (e.g as a trigger) may have unintended consequences for other servers. It is considered a best practice to share contact information with your requests, so that administrators can reach you in case your HTTP calls get out of control. Certain API policies (e.g. [Wikimedia User-Agent policy](https://meta.wikimedia.org/wiki/User-Agent_policy)) may even require sharing specific contact information with each request. Others may disallow (via `robots.txt`) certain agents they don't recognize. For such cases you can set the `CURLOPT_USERAGENT` option ```sql SELECT http_set_curlopt('CURLOPT_USERAGENT', 'Examplebot/2.1 (+http://www.example.com/bot.html) Contact abuse@example.com'); SELECT status, content::json ->> 'user-agent' FROM http_get('http://httpbun.org/user-agent'); ``` ``` status | user_agent --------+----------------------------------------------------------- 200 | Examplebot/2.1 (+http://www.example.com/bot.html) Contact abuse@example.com ``` ## Keep-Alive & Timeouts *The `http_reset_curlopt()` approach described above is recommended. The global variables below will be deprecated and removed over time.* By default each request uses a fresh connection and assures that the connection is closed when the request is done. This behavior reduces the chance of consuming system resources (sockets) as the extension runs over extended periods of time. High-performance applications may wish to enable keep-alive and connection persistence to reduce latency and enhance throughput. The following GUC variable changes the behavior of the http extension to maintain connections as long as possible: http.keepalive = 'on' By default a 5 second timeout is set for the completion of a request. If a different timeout is desired the following GUC variable can be used to set it in milliseconds: http.timeout_msec = 200 ## Installation ### UNIX If you have PostgreSQL (>= 9.3) devel packages and CURL devel packages installed (>= 0.7.20), you should have `pg_config` and `curl-config` on your path, so you should be able to just run `make` (or `gmake`), then `make install`, then in your database `CREATE EXTENSION http`. If you already installed a previous version and you just want to upgrade, then `ALTER EXTENSION http UPDATE`. #### Compiling for Apt based systems, using apt postgresql packages Refer to https://wiki.postgresql.org/wiki/Apt for pulling packages from apt.postgresql.org repo ``` # replace the postgresql-server-dev-14 with your current version sudo apt install postgresql-server-dev-14 libcurl4-openssl-dev make g++ make sudo make install ``` ### Windows There is a build available at [postgresonline](http://www.postgresonline.com/journal/archives/371-http-extension.html), not maintained by me. ## Why This is a Bad Idea - "What happens if the web page takes a long time to return?" Your SQL call will just wait there until it does. Make sure your web service fails fast. Or (dangerous in a different way) run your query within [pg_background](https://github.com/vibhorkum/pg_background). - "What if the web page returns junk?" Your SQL call will have to test for junk before doing anything with the payload. - "What if the web page never returns?" Set a short timeout, or send a cancel to the request, or just wait forever. - "What if a user queries a page they shouldn't?" Restrict function access, or just don't install a footgun like this extension where users can access it. ## To Do - The new http://www.postgresql.org/docs/9.3/static/bgworker.html background worker support could be used to set up an HTTP request queue, so that pgsql-http can register a request and callback and then return immediately. - Inevitably some web server will return gzip content (Content-Encoding) without being asked for it. Handling that gracefully would be good. pgsql-http-1.6.0/ci/000077500000000000000000000000001446520663200142245ustar00rootroot00000000000000pgsql-http-1.6.0/ci/pg_hba.conf000066400000000000000000000006031446520663200163120ustar00rootroot00000000000000# TYPE DATABASE USER ADDRESS METHOD # "local" is for Unix domain socket connections only local all postgres trust # IPv4 local connections: host all postgres 127.0.0.1/32 trust # IPv6 local connections: host all postgres ::1/128 trust pgsql-http-1.6.0/expected/000077500000000000000000000000001446520663200154325ustar00rootroot00000000000000pgsql-http-1.6.0/expected/http.out000066400000000000000000000157151446520663200171530ustar00rootroot00000000000000CREATE EXTENSION http; set http.timeout_msec = 10000; SELECT http_set_curlopt('CURLOPT_TIMEOUT', '10'); http_set_curlopt ------------------ t (1 row) -- Status code SELECT status FROM http_get('https://httpbun.org/status/202'); status -------- 202 (1 row) -- Headers SELECT lower(field) AS field, value FROM ( SELECT (unnest(headers)).* FROM http_get('https://httpbun.org/response-headers?Abcde=abcde') ) a WHERE field ILIKE 'Abcde'; field | value -------+------- abcde | abcde (1 row) -- GET SELECT status, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_get('https://httpbun.org/anything?foo=bar'); status | args | url | method --------+------+----------------------------------------+-------- 200 | bar | "https://httpbun.org/anything?foo=bar" | "GET" (1 row) -- GET with data SELECT status, content::json->'args'->>'this' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_get('https://httpbun.org/anything', jsonb_build_object('this', 'that')); status | args | url | method --------+------+------------------------------------------+-------- 200 | that | "https://httpbun.org/anything?this=that" | "GET" (1 row) -- GET with data SELECT status, content::json->'args' as args, content::json->>'data' as data, content::json->'url' as url, content::json->'method' as method from http(('GET', 'https://httpbun.org/anything', NULL, 'application/json', '{"search": "toto"}')); status | args | data | url | method --------+------+--------------------+--------------------------------+-------- 200 | {} | {"search": "toto"} | "https://httpbun.org/anything" | "GET" (1 row) -- DELETE SELECT status, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_delete('https://httpbun.org/anything?foo=bar'); status | args | url | method --------+------+----------------------------------------+---------- 200 | bar | "https://httpbun.org/anything?foo=bar" | "DELETE" (1 row) -- DELETE with payload SELECT status, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method, content::json->'data' AS data FROM http_delete('https://httpbun.org/anything?foo=bar', 'payload', 'text/plain'); status | args | url | method | data --------+------+----------------------------------------+----------+----------- 200 | bar | "https://httpbun.org/anything?foo=bar" | "DELETE" | "payload" (1 row) -- PUT SELECT status, content::json->'data' AS data, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_put('https://httpbun.org/anything?foo=bar','payload','text/plain'); status | data | args | url | method --------+-----------+------+----------------------------------------+-------- 200 | "payload" | bar | "https://httpbun.org/anything?foo=bar" | "PUT" (1 row) -- PATCH SELECT status, content::json->'data' AS data, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_patch('https://httpbun.org/anything?foo=bar','{"this":"that"}','application/json'); status | data | args | url | method --------+-----------------------+------+----------------------------------------+--------- 200 | "{\"this\":\"that\"}" | bar | "https://httpbun.org/anything?foo=bar" | "PATCH" (1 row) -- POST SELECT status, content::json->'data' AS data, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_post('https://httpbun.org/anything?foo=bar','payload','text/plain'); status | data | args | url | method --------+-----------+------+----------------------------------------+-------- 200 | "payload" | bar | "https://httpbun.org/anything?foo=bar" | "POST" (1 row) -- POST with json data SELECT status, content::json->'form'->>'this' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_post('https://httpbun.org/anything', jsonb_build_object('this', 'that')); status | args | url | method --------+------+--------------------------------+-------- 200 | that | "https://httpbun.org/anything" | "POST" (1 row) -- POST with data SELECT status, content::json->'form'->>'key1' AS key1, content::json->'form'->>'key2' AS key2, content::json->'url' AS url, content::json->'method' AS method FROM http_post('https://httpbun.org/anything', 'key1=value1&key2=value2','application/x-www-form-urlencoded'); status | key1 | key2 | url | method --------+--------+--------+--------------------------------+-------- 200 | value1 | value2 | "https://httpbun.org/anything" | "POST" (1 row) -- HEAD SELECT lower(field) AS field, value FROM ( SELECT (unnest(headers)).* FROM http_head('https://httpbun.org/response-headers?Abcde=abcde') ) a WHERE field ILIKE 'Abcde'; field | value -------+------- abcde | abcde (1 row) -- Follow redirect SELECT status, content::json->'url' AS url FROM http_get('https://httpbun.org/redirect-to?url=get'); status | url --------+--------------------------- 200 | "https://httpbun.org/get" (1 row) -- Request image WITH http AS ( SELECT * FROM http_get('https://httpbingo.org/image/png') ), headers AS ( SELECT (unnest(headers)).* FROM http ) SELECT http.content_type, length(text_to_bytea(http.content)) AS length_binary FROM http, headers WHERE field ilike 'Content-Type'; content_type | length_binary --------------+--------------- image/png | 8090 (1 row) -- Alter options and and reset them and throw errors SELECT http_set_curlopt('CURLOPT_PROXY', '127.0.0.1'); http_set_curlopt ------------------ t (1 row) -- Error because proxy is not there DO $$ BEGIN SELECT status FROM http_get('https://httpbun.org/status/555'); EXCEPTION WHEN OTHERS THEN RAISE WARNING 'Failed to connect'; END; $$; WARNING: Failed to connect -- Still an error DO $$ BEGIN SELECT status FROM http_get('https://httpbun.org/status/555'); EXCEPTION WHEN OTHERS THEN RAISE WARNING 'Failed to connect'; END; $$; WARNING: Failed to connect -- Reset options SELECT http_reset_curlopt(); http_reset_curlopt -------------------- t (1 row) -- Now it should work SELECT status FROM http_get('https://httpbun.org/status/555'); status -------- 555 (1 row) -- Alter the default timeout and then run a query that is longer than -- the default (5s), but shorter than the new timeout SELECT http_set_curlopt('CURLOPT_TIMEOUT_MS', '10000'); http_set_curlopt ------------------ t (1 row) SELECT status FROM http_get('https://httpbun.org/delay/7'); status -------- 200 (1 row) pgsql-http-1.6.0/http--1.0--1.1.sql000066400000000000000000000034551446520663200162630ustar00rootroot00000000000000 -- Remove old 1.0 functions DROP FUNCTION http_get(varchar, varchar); DROP FUNCTION http_post(varchar, varchar, varchar, varchar); CREATE DOMAIN http_method AS text CHECK ( VALUE ILIKE 'get' OR VALUE ILIKE 'post' OR VALUE ILIKE 'put' OR VALUE ILIKE 'delete' ); CREATE DOMAIN content_type AS text CHECK ( VALUE ~ '^\S+\/\S+' ); CREATE TYPE http_header AS ( field VARCHAR, value VARCHAR ); CREATE TYPE http_request AS ( method http_method, uri VARCHAR, headers http_header[], content_type VARCHAR, content VARCHAR ); ALTER TYPE http_response ALTER ATTRIBUTE headers TYPE http_header[]; CREATE OR REPLACE FUNCTION http_header (field VARCHAR, value VARCHAR) RETURNS http_header AS $$ SELECT $1, $2 $$ LANGUAGE 'sql'; CREATE OR REPLACE FUNCTION http(request @extschema@.http_request) RETURNS http_response AS 'MODULE_PATHNAME', 'http_request' LANGUAGE 'c'; CREATE OR REPLACE FUNCTION http_get(uri VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('GET', $1, NULL, NULL, NULL)::http_request) $$ LANGUAGE 'sql'; CREATE OR REPLACE FUNCTION http_post(uri VARCHAR, content VARCHAR, content_type VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('POST', $1, NULL, $3, $2)::http_request) $$ LANGUAGE 'sql'; CREATE OR REPLACE FUNCTION http_put(uri VARCHAR, content VARCHAR, content_type VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('PUT', $1, NULL, $3, $2)::http_request) $$ LANGUAGE 'sql'; CREATE OR REPLACE FUNCTION http_delete(uri VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('DELETE', $1, NULL, NULL, NULL)::http_request) $$ LANGUAGE 'sql'; CREATE OR REPLACE FUNCTION urlencode(string VARCHAR) RETURNS TEXT AS 'MODULE_PATHNAME' LANGUAGE 'c' IMMUTABLE STRICT; pgsql-http-1.6.0/http--1.1--1.2.sql000066400000000000000000000013021446520663200162520ustar00rootroot00000000000000 ALTER DOMAIN http_method DROP CONSTRAINT http_method_check; ALTER DOMAIN http_method ADD CHECK ( VALUE ILIKE 'get' OR VALUE ILIKE 'post' OR VALUE ILIKE 'put' OR VALUE ILIKE 'delete' OR VALUE ILIKE 'head' ); CREATE OR REPLACE FUNCTION http_head(uri VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('HEAD', $1, NULL, NULL, NULL)::http_request) $$ LANGUAGE 'sql'; CREATE OR REPLACE FUNCTION http_set_curlopt (curlopt VARCHAR, value VARCHAR) RETURNS boolean AS 'MODULE_PATHNAME', 'http_set_curlopt' LANGUAGE 'c'; CREATE OR REPLACE FUNCTION http_reset_curlopt () RETURNS boolean AS 'MODULE_PATHNAME', 'http_reset_curlopt' LANGUAGE 'c'; pgsql-http-1.6.0/http--1.2--1.3.sql000066400000000000000000000007271446520663200162660ustar00rootroot00000000000000ALTER DOMAIN http_method DROP CONSTRAINT http_method_check; ALTER DOMAIN http_method ADD CHECK ( VALUE ILIKE 'get' OR VALUE ILIKE 'post' OR VALUE ILIKE 'put' OR VALUE ILIKE 'delete' OR VALUE ILIKE 'patch' OR VALUE ILIKE 'head' ); CREATE OR REPLACE FUNCTION http_patch(uri VARCHAR, content VARCHAR, content_type VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('PATCH', $1, NULL, $3, $2)::http_request) $$ LANGUAGE 'sql'; pgsql-http-1.6.0/http--1.3--1.4.sql000066400000000000000000000015721446520663200162670ustar00rootroot00000000000000CREATE OR REPLACE FUNCTION http_list_curlopt () RETURNS TABLE(curlopt text, value text) AS 'MODULE_PATHNAME', 'http_list_curlopt' LANGUAGE 'c'; CREATE OR REPLACE FUNCTION urlencode(string BYTEA) RETURNS TEXT AS 'MODULE_PATHNAME' LANGUAGE 'c' IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION urlencode(data JSONB) RETURNS TEXT AS 'MODULE_PATHNAME' LANGUAGE 'c' IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION http_get(uri VARCHAR, data JSONB) RETURNS http_response AS $$ SELECT @extschema@.http(('GET', $1 || '?' || @extschema@.urlencode($2), NULL, NULL, NULL)::http_request) $$ LANGUAGE 'sql'; CREATE OR REPLACE FUNCTION http_post(uri VARCHAR, data JSONB) RETURNS http_response AS $$ SELECT @extschema@.http(('POST', $1, NULL, 'application/x-www-form-urlencoded', @extschema@.urlencode($2))::http_request) $$ LANGUAGE 'sql'; pgsql-http-1.6.0/http--1.4--1.5.sql000066400000000000000000000003471446520663200162700ustar00rootroot00000000000000 CREATE OR REPLACE FUNCTION http_delete(uri VARCHAR, content VARCHAR, content_type VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('DELETE', $1, NULL, $3, $2)::@extschema@.http_request) $$ LANGUAGE 'sql'; pgsql-http-1.6.0/http--1.5--1.6.sql000066400000000000000000000005411446520663200162660ustar00rootroot00000000000000 ALTER DOMAIN http_method DROP CONSTRAINT IF EXISTS http_method_check; CREATE FUNCTION text_to_bytea(data TEXT) RETURNS BYTEA AS 'MODULE_PATHNAME', 'text_to_bytea' LANGUAGE 'c' IMMUTABLE STRICT; CREATE FUNCTION bytea_to_text(data BYTEA) RETURNS TEXT AS 'MODULE_PATHNAME', 'bytea_to_text' LANGUAGE 'c' IMMUTABLE STRICT; pgsql-http-1.6.0/http--1.6.sql000066400000000000000000000074151446520663200157170ustar00rootroot00000000000000-- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION http" to load this file. \quit CREATE DOMAIN http_method AS text; CREATE DOMAIN content_type AS text CHECK ( VALUE ~ '^\S+\/\S+' ); CREATE TYPE http_header AS ( field VARCHAR, value VARCHAR ); CREATE TYPE http_response AS ( status INTEGER, content_type VARCHAR, headers http_header[], content VARCHAR ); CREATE TYPE http_request AS ( method http_method, uri VARCHAR, headers http_header[], content_type VARCHAR, content VARCHAR ); CREATE FUNCTION http_set_curlopt (curlopt VARCHAR, value VARCHAR) RETURNS boolean AS 'MODULE_PATHNAME', 'http_set_curlopt' LANGUAGE 'c'; CREATE FUNCTION http_reset_curlopt () RETURNS boolean AS 'MODULE_PATHNAME', 'http_reset_curlopt' LANGUAGE 'c'; CREATE FUNCTION http_list_curlopt () RETURNS TABLE(curlopt text, value text) AS 'MODULE_PATHNAME', 'http_list_curlopt' LANGUAGE 'c'; CREATE FUNCTION http_header (field VARCHAR, value VARCHAR) RETURNS http_header AS $$ SELECT $1, $2 $$ LANGUAGE 'sql'; CREATE FUNCTION http(request @extschema@.http_request) RETURNS http_response AS 'MODULE_PATHNAME', 'http_request' LANGUAGE 'c'; CREATE FUNCTION http_get(uri VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('GET', $1, NULL, NULL, NULL)::@extschema@.http_request) $$ LANGUAGE 'sql'; CREATE FUNCTION http_post(uri VARCHAR, content VARCHAR, content_type VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('POST', $1, NULL, $3, $2)::@extschema@.http_request) $$ LANGUAGE 'sql'; CREATE FUNCTION http_put(uri VARCHAR, content VARCHAR, content_type VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('PUT', $1, NULL, $3, $2)::@extschema@.http_request) $$ LANGUAGE 'sql'; CREATE FUNCTION http_patch(uri VARCHAR, content VARCHAR, content_type VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('PATCH', $1, NULL, $3, $2)::@extschema@.http_request) $$ LANGUAGE 'sql'; CREATE FUNCTION http_delete(uri VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('DELETE', $1, NULL, NULL, NULL)::@extschema@.http_request) $$ LANGUAGE 'sql'; CREATE FUNCTION http_delete(uri VARCHAR, content VARCHAR, content_type VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('DELETE', $1, NULL, $3, $2)::@extschema@.http_request) $$ LANGUAGE 'sql'; CREATE FUNCTION http_head(uri VARCHAR) RETURNS http_response AS $$ SELECT @extschema@.http(('HEAD', $1, NULL, NULL, NULL)::@extschema@.http_request) $$ LANGUAGE 'sql'; CREATE FUNCTION urlencode(string VARCHAR) RETURNS TEXT AS 'MODULE_PATHNAME' LANGUAGE 'c' IMMUTABLE STRICT; CREATE FUNCTION urlencode(string BYTEA) RETURNS TEXT AS 'MODULE_PATHNAME' LANGUAGE 'c' IMMUTABLE STRICT; CREATE FUNCTION urlencode(data JSONB) RETURNS TEXT AS 'MODULE_PATHNAME', 'urlencode_jsonb' LANGUAGE 'c' IMMUTABLE STRICT; CREATE FUNCTION http_get(uri VARCHAR, data JSONB) RETURNS http_response AS $$ SELECT @extschema@.http(('GET', $1 || '?' || @extschema@.urlencode($2), NULL, NULL, NULL)::@extschema@.http_request) $$ LANGUAGE 'sql'; CREATE FUNCTION http_post(uri VARCHAR, data JSONB) RETURNS http_response AS $$ SELECT @extschema@.http(('POST', $1, NULL, 'application/x-www-form-urlencoded', @extschema@.urlencode($2))::@extschema@.http_request) $$ LANGUAGE 'sql'; CREATE FUNCTION text_to_bytea(data TEXT) RETURNS BYTEA AS 'MODULE_PATHNAME', 'text_to_bytea' LANGUAGE 'c' IMMUTABLE STRICT; CREATE FUNCTION bytea_to_text(data BYTEA) RETURNS TEXT AS 'MODULE_PATHNAME', 'bytea_to_text' LANGUAGE 'c' IMMUTABLE STRICT; pgsql-http-1.6.0/http.c000066400000000000000000001252341446520663200147630ustar00rootroot00000000000000/*********************************************************************** * * Project: PgSQL HTTP * Purpose: Main file. * *********************************************************************** * Copyright 2015 Paul Ramsey * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * ***********************************************************************/ /* Constants */ #define HTTP_VERSION "1.6.0" #define HTTP_ENCODING "gzip" #define CURL_MIN_VERSION 0x071400 /* 7.20.0 */ /* System */ #include #include #include #include /* INT_MAX */ #include /* SIGINT */ /* PostgreSQL */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if PG_VERSION_NUM >= 90300 # include #endif #if PG_VERSION_NUM >= 100000 # include #endif #if PG_VERSION_NUM >= 120000 # include #else # define table_open(rel, lock) heap_open((rel), (lock)) # define table_close(rel, lock) heap_close((rel), (lock)) #endif #if PG_VERSION_NUM < 110000 #define PG_GETARG_JSONB_P(x) DatumGetJsonb(PG_GETARG_DATUM(x)) #endif /* CURL */ #include /* Set up PgSQL */ PG_MODULE_MAGIC; /* HTTP request methods we support */ typedef enum { HTTP_GET, HTTP_POST, HTTP_DELETE, HTTP_PUT, HTTP_HEAD, HTTP_PATCH, HTTP_UNKNOWN } http_method; /* Components (and postitions) of the http_request tuple type */ enum { REQ_METHOD = 0, REQ_URI = 1, REQ_HEADERS = 2, REQ_CONTENT_TYPE = 3, REQ_CONTENT = 4 } http_request_type; /* Components (and postitions) of the http_response tuple type */ enum { RESP_STATUS = 0, RESP_CONTENT_TYPE = 1, RESP_HEADERS = 2, RESP_CONTENT = 3 } http_response_type; /* Components (and postitions) of the http_header tuple type */ enum { HEADER_FIELD = 0, HEADER_VALUE = 1 } http_header_type; typedef enum { CURLOPT_STRING, CURLOPT_LONG } http_curlopt_type; /* CURLOPT string/enum value mapping */ typedef struct { char *curlopt_str; char *curlopt_val; CURLoption curlopt; http_curlopt_type curlopt_type; bool superuser_only; } http_curlopt; /* CURLOPT values we allow user to set at run-time */ /* Be careful adding these, as they can be a security risk */ static http_curlopt settable_curlopts[] = { { "CURLOPT_CAINFO", NULL, CURLOPT_CAINFO, CURLOPT_STRING, false }, { "CURLOPT_TIMEOUT", NULL, CURLOPT_TIMEOUT, CURLOPT_LONG, false }, { "CURLOPT_TIMEOUT_MS", NULL, CURLOPT_TIMEOUT_MS, CURLOPT_LONG, false }, { "CURLOPT_CONNECTTIMEOUT", NULL, CURLOPT_CONNECTTIMEOUT, CURLOPT_LONG, false }, { "CURLOPT_USERAGENT", NULL, CURLOPT_USERAGENT, CURLOPT_STRING, false }, { "CURLOPT_USERPWD", NULL, CURLOPT_USERPWD, CURLOPT_STRING, false }, { "CURLOPT_IPRESOLVE", NULL, CURLOPT_IPRESOLVE, CURLOPT_LONG, false }, #if LIBCURL_VERSION_NUM >= 0x070903 /* 7.9.3 */ { "CURLOPT_SSLCERTTYPE", NULL, CURLOPT_SSLCERTTYPE, CURLOPT_STRING, false }, #endif #if LIBCURL_VERSION_NUM >= 0x070e01 /* 7.14.1 */ { "CURLOPT_PROXY", NULL, CURLOPT_PROXY, CURLOPT_STRING, false }, { "CURLOPT_PROXYPORT", NULL, CURLOPT_PROXYPORT, CURLOPT_LONG, false }, #endif #if LIBCURL_VERSION_NUM >= 0x071301 /* 7.19.1 */ { "CURLOPT_PROXYUSERNAME", NULL, CURLOPT_PROXYUSERNAME, CURLOPT_STRING, false }, { "CURLOPT_PROXYPASSWORD", NULL, CURLOPT_PROXYPASSWORD, CURLOPT_STRING, false }, #endif #if LIBCURL_VERSION_NUM >= 0x071504 /* 7.21.4 */ { "CURLOPT_TLSAUTH_USERNAME", NULL, CURLOPT_TLSAUTH_USERNAME, CURLOPT_STRING, false }, { "CURLOPT_TLSAUTH_PASSWORD", NULL, CURLOPT_TLSAUTH_PASSWORD, CURLOPT_STRING, false }, { "CURLOPT_TLSAUTH_TYPE", NULL, CURLOPT_TLSAUTH_TYPE, CURLOPT_STRING, false }, #endif #if LIBCURL_VERSION_NUM >= 0x071800 /* 7.24.0 */ { "CURLOPT_DNS_SERVERS", NULL, CURLOPT_DNS_SERVERS, CURLOPT_STRING, false }, #endif #if LIBCURL_VERSION_NUM >= 0x071900 /* 7.25.0 */ { "CURLOPT_TCP_KEEPALIVE", NULL, CURLOPT_TCP_KEEPALIVE, CURLOPT_LONG, false }, { "CURLOPT_TCP_KEEPIDLE", NULL, CURLOPT_TCP_KEEPIDLE, CURLOPT_LONG, false }, #endif #if LIBCURL_VERSION_NUM >= 0x072500 /* 7.37.0 */ { "CURLOPT_SSL_VERIFYHOST", NULL, CURLOPT_SSL_VERIFYHOST, CURLOPT_LONG, false }, { "CURLOPT_SSL_VERIFYPEER", NULL, CURLOPT_SSL_VERIFYPEER, CURLOPT_LONG, false }, #endif { "CURLOPT_SSLCERT", NULL, CURLOPT_SSLCERT, CURLOPT_STRING, false }, { "CURLOPT_SSLKEY", NULL, CURLOPT_SSLKEY, CURLOPT_STRING, false }, #if LIBCURL_VERSION_NUM >= 0x073400 /* 7.52.0 */ { "CURLOPT_PRE_PROXY", NULL, CURLOPT_PRE_PROXY, CURLOPT_STRING, false }, { "CURLOPT_PROXY_CAINFO", NULL, CURLOPT_PROXY_TLSAUTH_USERNAME, CURLOPT_STRING, false }, { "CURLOPT_PROXY_TLSAUTH_USERNAME", NULL, CURLOPT_PROXY_TLSAUTH_USERNAME, CURLOPT_STRING, false }, { "CURLOPT_PROXY_TLSAUTH_PASSWORD", NULL, CURLOPT_PROXY_TLSAUTH_PASSWORD, CURLOPT_STRING, false }, { "CURLOPT_PROXY_TLSAUTH_TYPE", NULL, CURLOPT_PROXY_TLSAUTH_TYPE, CURLOPT_STRING, false }, #endif { NULL, NULL, 0, 0, false } /* Array null terminator */ }; /* Function signatures */ void _PG_init(void); void _PG_fini(void); static size_t http_writeback(void *contents, size_t size, size_t nmemb, void *userp); static size_t http_readback(void *buffer, size_t size, size_t nitems, void *instream); /* Global variables */ bool g_use_keepalive; int g_timeout_msec; CURL * g_http_handle = NULL; List * g_curl_opts = NIL; /* * Interrupt support is dependent on CURLOPT_XFERINFOFUNCTION which is * only available from 7.32.0 and up */ #if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ pqsigfunc pgsql_interrupt_handler = NULL; int http_interrupt_requested = 0; /* * To support request interruption, we have libcurl run the progress meter * callback frequently, and here we watch to see if PgSQL has flipped our * global 'http_interrupt_requested' flag. If it has been flipped, * the non-zero return value will cue libcurl to abort the transfer, * leading to a CURLE_ABORTED_BY_CALLBACK return on the curl_easy_perform() */ static int http_progress_callback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { #ifdef WIN32 if (UNBLOCKED_SIGNAL_QUEUE()) { pgwin32_dispatch_queued_signals(); } #endif /* elog(DEBUG3, "http_interrupt_requested = %d", http_interrupt_requested); */ return http_interrupt_requested; } /* * We register this callback with the PgSQL signal handler to * capture SIGINT and set our local interupt flag so that * libcurl will eventually notice that a cancel is requested */ static void http_interrupt_handler(int sig) { /* Handle the signal here */ elog(DEBUG2, "http_interrupt_handler: sig=%d", sig); http_interrupt_requested = sig; pgsql_interrupt_handler(sig); return; } #endif /* 7.39.0 */ #undef HTTP_MEM_CALLBACKS #ifdef HTTP_MEM_CALLBACKS static void * http_calloc(size_t a, size_t b) { if (a>0 && b>0) return palloc0(a*b); else return NULL; } static void http_free(void *a) { if (a) pfree(a); } static void * http_realloc(void *a, size_t sz) { if (a && sz) return repalloc(a, sz); else if (sz) return palloc(sz); else return a; } static void * http_malloc(size_t sz) { return sz ? palloc(sz) : NULL; } #endif /* Startup */ void _PG_init(void) { DefineCustomBoolVariable("http.keepalive", "reuse existing connections with keepalive", NULL, &g_use_keepalive, false, PGC_USERSET, GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable("http.timeout_msec", "request completion timeout in milliseconds", NULL, &g_timeout_msec, 0, 0, INT_MAX, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_UNIT_MS, NULL, NULL, NULL); #ifdef HTTP_MEM_CALLBACKS /* * Use PgSQL memory management in Curl * Warning, https://curl.se/libcurl/c/curl_global_init_mem.html * notes "If you are using libcurl from multiple threads or libcurl * was built with the threaded resolver option then the callback * functions must be thread safe." PgSQL isn't multi-threaded, * but we have no control over whether the "threaded resolver" is * in use. We may need a semaphor to ensure our callbacks are * accessed sequentially only. */ curl_global_init_mem(CURL_GLOBAL_ALL, http_malloc, http_free, http_realloc, pstrdup, http_calloc); #else /* Set up Curl! */ curl_global_init(CURL_GLOBAL_ALL); #endif #if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ /* Register our interrupt handler (http_handle_interrupt) */ /* and store the existing one so we can call it when we're */ /* through with our work */ pgsql_interrupt_handler = pqsignal(SIGINT, http_interrupt_handler); http_interrupt_requested = 0; #endif } /* Tear-down */ void _PG_fini(void) { #if LIBCURL_VERSION_NUM >= 0x072700 /* Re-register the original signal handler */ pqsignal(SIGINT, pgsql_interrupt_handler); #endif if (g_http_handle) { curl_easy_cleanup(g_http_handle); g_http_handle = NULL; } curl_global_cleanup(); elog(NOTICE, "Goodbye from HTTP %s", HTTP_VERSION); } /** * This function is passed into CURL as the CURLOPT_WRITEFUNCTION, * this allows the return values to be held in memory, in our case in a string. */ static size_t http_writeback(void *contents, size_t size, size_t nmemb, void *userp) { size_t realsize = size * nmemb; StringInfo si = (StringInfo)userp; appendBinaryStringInfo(si, (const char*)contents, (int)realsize); return realsize; } /** * This function is passed into CURL as the CURLOPT_READFUNCTION, * this allows the PUT operation to read the data it needs. We * pass a StringInfo as our input, and per the callback contract * return the number of bytes read at each call. */ static size_t http_readback(void *buffer, size_t size, size_t nitems, void *instream) { size_t reqsize = size * nitems; StringInfo si = (StringInfo)instream; size_t remaining = si->len - si->cursor; size_t readsize = Min(reqsize, remaining); memcpy(buffer, si->data + si->cursor, readsize); si->cursor += readsize; return readsize; } static void http_error(CURLcode err, const char *error_buffer) { if ( strlen(error_buffer) > 0 ) ereport(ERROR, (errmsg("%s", error_buffer))); else ereport(ERROR, (errmsg("%s", curl_easy_strerror(err)))); } /* Utility macro to try a setopt and catch an error */ #define CURL_SETOPT(handle, opt, value) do { \ err = curl_easy_setopt((handle), (opt), (value)); \ if ( err != CURLE_OK ) \ { \ http_error(err, http_error_buffer); \ PG_RETURN_NULL(); \ } \ } while (0); /** * Convert a request type string into the appropriate enumeration value. */ static http_method request_type(const char *method) { if ( strcasecmp(method, "GET") == 0 ) return HTTP_GET; else if ( strcasecmp(method, "POST") == 0 ) return HTTP_POST; else if ( strcasecmp(method, "PUT") == 0 ) return HTTP_PUT; else if ( strcasecmp(method, "DELETE") == 0 ) return HTTP_DELETE; else if ( strcasecmp(method, "HEAD") == 0 ) return HTTP_HEAD; else if ( strcasecmp(method, "PATCH") == 0 ) return HTTP_PATCH; else return HTTP_UNKNOWN; } /** * Given a field name and value, output a http_header tuple. */ static Datum header_tuple(TupleDesc header_tuple_desc, const char *field, const char *value) { HeapTuple header_tuple; int ncolumns; Datum *header_values; bool *header_nulls; /* Prepare our return object */ ncolumns = header_tuple_desc->natts; header_values = palloc0(sizeof(Datum)*ncolumns); header_nulls = palloc0(sizeof(bool)*ncolumns); header_values[HEADER_FIELD] = CStringGetTextDatum(field); header_nulls[HEADER_FIELD] = false; header_values[HEADER_VALUE] = CStringGetTextDatum(value); header_nulls[HEADER_VALUE] = false; /* Build up a tuple from values/nulls lists */ header_tuple = heap_form_tuple(header_tuple_desc, header_values, header_nulls); return HeapTupleGetDatum(header_tuple); } /** * Our own implementation of strcasestr. */ static char * http_strcasestr(const char *s, const char *find) { char c, sc; size_t len; if ((c = *find++) != 0) { c = tolower((unsigned char)c); len = strlen(find); do { do { if ((sc = *s++) == 0) return (NULL); } while ((char)tolower((unsigned char)sc) != c); } while (strncasecmp(s, find, len) != 0); s--; } return ((char *)s); } /** * Quick and dirty, remove all \r from a StringInfo. */ static void string_info_remove_cr(StringInfo si) { int i = 0, j = 0; while ( si->data[i] ) { if ( si->data[i] != '\r' ) si->data[j++] = si->data[i++]; else i++; } si->data[j] = '\0'; si->len -= i-j; return; } /** * Add an array of http_header tuples into a Curl string list. */ static struct curl_slist * header_array_to_slist(ArrayType *array, struct curl_slist *headers) { ArrayIterator iterator; Datum value; bool isnull; #if PG_VERSION_NUM >= 90500 iterator = array_create_iterator(array, 0, NULL); #else iterator = array_create_iterator(array, 0); #endif while (array_iterate(iterator, &value, &isnull)) { HeapTupleHeader rec; HeapTupleData tuple; Oid tup_type; int32 tup_typmod, ncolumns; TupleDesc tup_desc; size_t tup_len; Datum *values; bool *nulls; /* Skip null array items */ if ( isnull ) continue; rec = DatumGetHeapTupleHeader(value); tup_type = HeapTupleHeaderGetTypeId(rec); tup_typmod = HeapTupleHeaderGetTypMod(rec); tup_len = HeapTupleHeaderGetDatumLength(rec); tup_desc = lookup_rowtype_tupdesc(tup_type, tup_typmod); ncolumns = tup_desc->natts; /* Prepare for values / nulls to hold the data */ values = (Datum *) palloc0(ncolumns * sizeof(Datum)); nulls = (bool *) palloc0(ncolumns * sizeof(bool)); /* Build a temporary HeapTuple control structure */ tuple.t_len = tup_len; ItemPointerSetInvalid(&(tuple.t_self)); tuple.t_tableOid = InvalidOid; tuple.t_data = rec; /* Break down the tuple into values/nulls lists */ heap_deform_tuple(&tuple, tup_desc, values, nulls); /* Convert the data into a header */ /* TODO: Ensure the header list is unique? Or leave that to the */ /* server to deal with. */ if ( ! nulls[HEADER_FIELD] ) { size_t total_len = 0; char *buffer = NULL; char *header_val; char *header_fld = TextDatumGetCString(values[HEADER_FIELD]); /* Don't process "content-type" in the optional headers */ if ( strlen(header_fld) <= 0 || strncasecmp(header_fld, "Content-Type", 12) == 0 ) { elog(NOTICE, "'Content-Type' is not supported as an optional header"); continue; } if ( nulls[HEADER_VALUE] ) header_val = pstrdup(""); else header_val = TextDatumGetCString(values[HEADER_VALUE]); total_len = strlen(header_val) + strlen(header_fld) + sizeof(char) + sizeof(": "); buffer = palloc(total_len); if (buffer) { snprintf(buffer, total_len, "%s: %s", header_fld, header_val); elog(DEBUG2, "pgsql-http: optional request header '%s'", buffer); headers = curl_slist_append(headers, buffer); pfree(buffer); } else { elog(ERROR, "pgsql-http: palloc(%zu) failure", total_len); } pfree(header_fld); pfree(header_val); } /* Free all the temporary structures */ ReleaseTupleDesc(tup_desc); pfree(values); pfree(nulls); } array_free_iterator(iterator); return headers; } /** * This function is now exposed in PG16 and above * so no need to redefine it for PG16 and above */ #if PG_VERSION_NUM < 160000 /** * Look up the namespace the extension is installed in */ static Oid get_extension_schema(Oid ext_oid) { Oid result; SysScanDesc scandesc; HeapTuple tuple; ScanKeyData entry[1]; #if PG_VERSION_NUM >= 120000 Oid pg_extension_oid = Anum_pg_extension_oid; #else Oid pg_extension_oid = ObjectIdAttributeNumber; #endif Relation rel = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], pg_extension_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(ext_oid)); scandesc = systable_beginscan(rel, ExtensionOidIndexId, true, NULL, 1, entry); tuple = systable_getnext(scandesc); /* We assume that there can be at most one matching tuple */ if (HeapTupleIsValid(tuple)) result = ((Form_pg_extension) GETSTRUCT(tuple))->extnamespace; else result = InvalidOid; systable_endscan(scandesc); table_close(rel, AccessShareLock); return result; } #endif /** * Look up the tuple description for a extension-defined type, * avoiding the pitfalls of using relations that are not part * of the extension, but share the same name as the relation * of interest. */ static TupleDesc typname_get_tupledesc(const char *extname, const char *typname) { Oid extoid = get_extension_oid(extname, true); Oid extschemaoid; Oid typoid; if ( ! OidIsValid(extoid) ) elog(ERROR, "could not lookup '%s' extension oid", extname); extschemaoid = get_extension_schema(extoid); #if PG_VERSION_NUM >= 120000 typoid = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid, PointerGetDatum(typname), ObjectIdGetDatum(extschemaoid)); #else typoid = GetSysCacheOid2(TYPENAMENSP, PointerGetDatum(typname), ObjectIdGetDatum(extschemaoid)); #endif if ( OidIsValid(typoid) ) { // Oid typ_oid = get_typ_typrelid(rel_oid); Oid relextoid = getExtensionOfObject(TypeRelationId, typoid); if ( relextoid == extoid ) { return TypeGetTupleDesc(typoid, NIL); } } elog(ERROR, "could not lookup '%s' tuple desc", typname); } #define RVSZ 8192 /* Max length of header element */ /** * Convert a string of headers separated by newlines/CRs into an * array of http_header tuples. */ static ArrayType * header_string_to_array(StringInfo si) { /* Array building */ size_t arr_nelems = 0; size_t arr_elems_size = 8; Datum *arr_elems = palloc0(arr_elems_size*sizeof(Datum)); Oid elem_type; int16 elem_len; bool elem_byval; char elem_align; /* Header handling */ TupleDesc header_tuple_desc = NULL; /* Regex support */ const char *regex_pattern = "^([^ \t\r\n\v\f]+): ?([^ \t\r\n\v\f]+.*)$"; regex_t regex; regmatch_t pmatch[3]; int reti; char rv1[RVSZ]; char rv2[RVSZ]; /* Compile the regular expression */ reti = regcomp(®ex, regex_pattern, REG_ICASE | REG_EXTENDED | REG_NEWLINE ); if ( reti ) elog(ERROR, "Unable to compile regex pattern '%s'", regex_pattern); /* Lookup the tuple defn */ header_tuple_desc = typname_get_tupledesc("http", "http_header"); /* Prepare array building metadata */ elem_type = header_tuple_desc->tdtypeid; get_typlenbyvalalign(elem_type, &elem_len, &elem_byval, &elem_align); /* Loop through string, matching regex pattern */ si->cursor = 0; while ( ! regexec(®ex, si->data+si->cursor, 3, pmatch, 0) ) { /* Read the regex match results */ int eo0 = pmatch[0].rm_eo; int so1 = pmatch[1].rm_so; int eo1 = pmatch[1].rm_eo; int so2 = pmatch[2].rm_so; int eo2 = pmatch[2].rm_eo; /* Copy the matched portions out of the string */ memcpy(rv1, si->data+si->cursor+so1, Min(eo1-so1, RVSZ)); rv1[eo1-so1] = '\0'; memcpy(rv2, si->data+si->cursor+so2, Min(eo2-so2, RVSZ)); rv2[eo2-so2] = '\0'; /* Move forward for next match */ si->cursor += eo0; /* Increase elements array size if necessary */ if ( arr_nelems >= arr_elems_size ) { arr_elems_size *= 2; arr_elems = repalloc(arr_elems, arr_elems_size*sizeof(Datum)); } arr_elems[arr_nelems] = header_tuple(header_tuple_desc, rv1, rv2); arr_nelems++; } regfree(®ex); ReleaseTupleDesc(header_tuple_desc); return construct_array(arr_elems, arr_nelems, elem_type, elem_len, elem_byval, elem_align); } /* Check/log version info */ static void http_check_curl_version(const curl_version_info_data *version_info) { elog(DEBUG2, "pgsql-http: curl version %s", version_info->version); elog(DEBUG2, "pgsql-http: curl version number 0x%x", version_info->version_num); elog(DEBUG2, "pgsql-http: ssl version %s", version_info->ssl_version); if ( version_info->version_num < CURL_MIN_VERSION ) { elog(ERROR, "pgsql-http requires Curl version 0.7.20 or higher"); } } static bool set_curlopt(CURL* handle, const http_curlopt *opt) { CURLcode err = CURLE_OK; char http_error_buffer[CURL_ERROR_SIZE] = "\0"; memset(http_error_buffer, 0, sizeof(http_error_buffer)); /* Argument is a string */ if (opt->curlopt_type == CURLOPT_STRING) { err = curl_easy_setopt(handle, opt->curlopt, opt->curlopt_val); elog(DEBUG2, "pgsql-http: set '%s' to value '%s', return value = %d", opt->curlopt_str, opt->curlopt_val, err); } /* Argument is a long */ else if (opt->curlopt_type == CURLOPT_LONG) { long value_long; errno = 0; value_long = strtol(opt->curlopt_val, NULL, 10); if ( errno == EINVAL || errno == ERANGE ) elog(ERROR, "invalid integer provided for '%s'", opt->curlopt_str); err = curl_easy_setopt(handle, opt->curlopt, value_long); elog(DEBUG2, "pgsql-http: set '%s' to value '%ld', return value = %d", opt->curlopt_str, value_long, err); } else { elog(ERROR, "invalid curlopt_type"); } if ( err != CURLE_OK ) { http_error(err, http_error_buffer); return false; } return true; } /* Check/create the global CURL* handle */ static CURL * http_get_handle() { http_curlopt opt; CURL *handle = g_http_handle; size_t i = 0; /* Initialize the global handle if needed */ if (!handle) { handle = curl_easy_init(); } /* Always reset because we're going to infull the user */ /* set options down below */ else { curl_easy_reset(handle); } /* Always want a default fast (1 second) connection timeout */ /* User can over-ride with http_set_curlopt() if they wish */ curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 1); curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, 5000); /* Set the user agent. If not set, use PG_VERSION as default */ curl_easy_setopt(handle, CURLOPT_USERAGENT, PG_VERSION_STR); if (!handle) ereport(ERROR, (errmsg("Unable to initialize CURL"))); /* Bring in any options the user has set this session */ while (1) { opt = settable_curlopts[i++]; if (!opt.curlopt_str) break; /* Option value is already set */ if (opt.curlopt_val) set_curlopt(handle, &opt); } g_http_handle = handle; return handle; } /** * User-defined Curl option reset. */ Datum http_reset_curlopt(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(http_reset_curlopt); Datum http_reset_curlopt(PG_FUNCTION_ARGS) { size_t i = 0; /* Set up global HTTP handle */ CURL * handle = http_get_handle(); curl_easy_reset(handle); /* Clean out the settable_curlopts global cache */ while (1) { http_curlopt *opt = settable_curlopts + i++; if (!opt->curlopt_str) break; if (opt->curlopt_val) pfree(opt->curlopt_val); opt->curlopt_val = NULL; } PG_RETURN_BOOL(true); } Datum http_list_curlopt(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(http_list_curlopt); Datum http_list_curlopt(PG_FUNCTION_ARGS) { struct list_state { size_t i; /* read position */ }; MemoryContext oldcontext, newcontext; FuncCallContext *funcctx; struct list_state *state; Datum vals[2]; bool nulls[2]; if (SRF_IS_FIRSTCALL()) { funcctx = SRF_FIRSTCALL_INIT(); newcontext = funcctx->multi_call_memory_ctx; oldcontext = MemoryContextSwitchTo(newcontext); state = palloc0(sizeof(*state)); funcctx->user_fctx = state; if(get_call_result_type(fcinfo, 0, &funcctx->tuple_desc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("composite-returning function called in context that cannot accept a composite"))); BlessTupleDesc(funcctx->tuple_desc); MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); state = funcctx->user_fctx; while (1) { Datum result; HeapTuple tuple; text *option, *value; http_curlopt *opt = settable_curlopts + state->i++; if (!opt->curlopt_str) break; if (!opt->curlopt_val) continue; option = cstring_to_text(opt->curlopt_str); value = cstring_to_text(opt->curlopt_val); vals[0] = PointerGetDatum(option); vals[1] = PointerGetDatum(value); nulls[0] = nulls[1] = 0; tuple = heap_form_tuple(funcctx->tuple_desc, vals, nulls); result = HeapTupleGetDatum(tuple); SRF_RETURN_NEXT(funcctx, result); } SRF_RETURN_DONE(funcctx); } /** * User-defined Curl option handling. */ Datum http_set_curlopt(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(http_set_curlopt); Datum http_set_curlopt(PG_FUNCTION_ARGS) { size_t i = 0; char *curlopt, *value; text *curlopt_txt, *value_txt; CURL *handle; /* Version check */ http_check_curl_version(curl_version_info(CURLVERSION_NOW)); /* We cannot handle null arguments */ if ( PG_ARGISNULL(0) || PG_ARGISNULL(1) ) PG_RETURN_BOOL(false); /* Set up global HTTP handle */ handle = http_get_handle(); /* Read arguments */ curlopt_txt = PG_GETARG_TEXT_P(0); value_txt = PG_GETARG_TEXT_P(1); curlopt = text_to_cstring(curlopt_txt); value = text_to_cstring(value_txt); while (1) { http_curlopt *opt = settable_curlopts + i++; if (!opt->curlopt_str) break; if (strcasecmp(opt->curlopt_str, curlopt) == 0) { if (opt->curlopt_val) pfree(opt->curlopt_val); opt->curlopt_val = MemoryContextStrdup(CacheMemoryContext, value); PG_RETURN_BOOL(set_curlopt(handle, opt)); } } elog(ERROR, "curl option '%s' is not available for run-time configuration", curlopt); PG_RETURN_BOOL(false); } /** * Master HTTP request function, takes in an http_request tuple and outputs * an http_response tuple. */ Datum http_request(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(http_request); Datum http_request(PG_FUNCTION_ARGS) { /* Input */ HeapTupleHeader rec; HeapTupleData tuple; Oid tup_type; int32 tup_typmod; TupleDesc tup_desc; int ncolumns; Datum *values; bool *nulls; char *uri; char *method_str; http_method method; /* Processing */ CURLcode err; char http_error_buffer[CURL_ERROR_SIZE] = "\0"; struct curl_slist *headers = NULL; StringInfoData si_data; StringInfoData si_headers; StringInfoData si_read; int http_return; long long_status; int status; char *content_type = NULL; int content_charset = -1; /* Output */ HeapTuple tuple_out; #if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ /* Set up the interrupt flag */ http_interrupt_requested = 0; #endif /* Version check */ http_check_curl_version(curl_version_info(CURLVERSION_NOW)); /* We cannot handle a null request */ if ( ! PG_ARGISNULL(0) ) rec = PG_GETARG_HEAPTUPLEHEADER(0); else { elog(ERROR, "An http_request must be provided"); PG_RETURN_NULL(); } /************************************************************************* * Build and run a curl request from the http_request argument *************************************************************************/ /* Zero out static memory */ memset(http_error_buffer, 0, sizeof(http_error_buffer)); /* Extract type info from the tuple itself */ tup_type = HeapTupleHeaderGetTypeId(rec); tup_typmod = HeapTupleHeaderGetTypMod(rec); tup_desc = lookup_rowtype_tupdesc(tup_type, tup_typmod); ncolumns = tup_desc->natts; /* Build a temporary HeapTuple control structure */ tuple.t_len = HeapTupleHeaderGetDatumLength(rec); ItemPointerSetInvalid(&(tuple.t_self)); tuple.t_tableOid = InvalidOid; tuple.t_data = rec; /* Prepare for values / nulls */ values = (Datum *) palloc0(ncolumns * sizeof(Datum)); nulls = (bool *) palloc0(ncolumns * sizeof(bool)); /* Break down the tuple into values/nulls lists */ heap_deform_tuple(&tuple, tup_desc, values, nulls); /* Read the URI */ if ( nulls[REQ_URI] ) elog(ERROR, "http_request.uri is NULL"); uri = TextDatumGetCString(values[REQ_URI]); /* Read the method */ if ( nulls[REQ_METHOD] ) elog(ERROR, "http_request.method is NULL"); method_str = TextDatumGetCString(values[REQ_METHOD]); method = request_type(method_str); elog(DEBUG2, "pgsql-http: method_str: '%s', method: %d", method_str, method); /* Set up global HTTP handle */ g_http_handle = http_get_handle(); /* Set up the error buffer */ CURL_SETOPT(g_http_handle, CURLOPT_ERRORBUFFER, http_error_buffer); /* Set the target URL */ CURL_SETOPT(g_http_handle, CURLOPT_URL, uri); /* Restrict to just http/https. Leaving unrestricted */ /* opens possibility of users requesting file:/// urls */ /* locally */ CURL_SETOPT(g_http_handle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); if ( g_use_keepalive ) { /* Keep sockets held open */ CURL_SETOPT(g_http_handle, CURLOPT_FORBID_REUSE, 0); } else { /* Keep sockets from being held open */ CURL_SETOPT(g_http_handle, CURLOPT_FORBID_REUSE, 1); } /* Set up the write-back function */ CURL_SETOPT(g_http_handle, CURLOPT_WRITEFUNCTION, http_writeback); /* Set up the write-back buffer */ initStringInfo(&si_data); initStringInfo(&si_headers); CURL_SETOPT(g_http_handle, CURLOPT_WRITEDATA, (void*)(&si_data)); CURL_SETOPT(g_http_handle, CURLOPT_WRITEHEADER, (void*)(&si_headers)); #if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ /* Connect the progress callback for interrupt support */ CURL_SETOPT(g_http_handle, CURLOPT_XFERINFOFUNCTION, http_progress_callback); CURL_SETOPT(g_http_handle, CURLOPT_NOPROGRESS, 0); #endif /* Set up the HTTP timeout */ if (g_timeout_msec > 0) CURL_SETOPT(g_http_handle, CURLOPT_TIMEOUT_MS, g_timeout_msec); /* Set the HTTP content encoding to all curl supports */ CURL_SETOPT(g_http_handle, CURLOPT_ACCEPT_ENCODING, ""); if ( method != HTTP_HEAD ) { /* Follow redirects, as many as 5 */ CURL_SETOPT(g_http_handle, CURLOPT_FOLLOWLOCATION, 1); CURL_SETOPT(g_http_handle, CURLOPT_MAXREDIRS, 5); } if ( g_use_keepalive ) { /* Add a keep alive option to the headers to reuse network sockets */ headers = curl_slist_append(headers, "Connection: Keep-Alive"); } else { /* Add a close option to the headers to avoid open network sockets */ headers = curl_slist_append(headers, "Connection: close"); } /* Let our charset preference be known */ headers = curl_slist_append(headers, "Charsets: utf-8"); /* Handle optional headers */ if ( ! nulls[REQ_HEADERS] ) { ArrayType *array = DatumGetArrayTypeP(values[REQ_HEADERS]); headers = header_array_to_slist(array, headers); } /* If we have a payload we send it, assuming we're either POST, GET, PATCH, PUT or DELETE or UNKNOWN */ if ( ! nulls[REQ_CONTENT] && values[REQ_CONTENT] ) { text *content_text; long content_size; char *cstr; char buffer[1024]; /* Read the content type */ if ( nulls[REQ_CONTENT_TYPE] || ! values[REQ_CONTENT_TYPE] ) elog(ERROR, "http_request.content_type is NULL"); cstr = TextDatumGetCString(values[REQ_CONTENT_TYPE]); /* Add content type to the headers */ snprintf(buffer, sizeof(buffer), "Content-Type: %s", cstr); headers = curl_slist_append(headers, buffer); pfree(cstr); /* Read the content */ content_text = DatumGetTextP(values[REQ_CONTENT]); content_size = VARSIZE_ANY_EXHDR(content_text); if ( method == HTTP_GET || method == HTTP_POST || method == HTTP_DELETE ) { /* Add the content to the payload */ CURL_SETOPT(g_http_handle, CURLOPT_POST, 1); if ( method == HTTP_GET ) { /* Force the verb to be GET */ CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, "GET"); } else if( method == HTTP_DELETE ) { /* Force the verb to be DELETE */ CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, "DELETE"); } CURL_SETOPT(g_http_handle, CURLOPT_POSTFIELDS, text_to_cstring(content_text)); } else if ( method == HTTP_PUT || method == HTTP_PATCH || method == HTTP_UNKNOWN ) { if ( method == HTTP_PATCH ) CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, "PATCH"); /* Assume the user knows what they are doing and pass unchanged */ if ( method == HTTP_UNKNOWN ) CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, method_str); initStringInfo(&si_read); appendBinaryStringInfo(&si_read, VARDATA(content_text), content_size); CURL_SETOPT(g_http_handle, CURLOPT_UPLOAD, 1); CURL_SETOPT(g_http_handle, CURLOPT_READFUNCTION, http_readback); CURL_SETOPT(g_http_handle, CURLOPT_READDATA, &si_read); CURL_SETOPT(g_http_handle, CURLOPT_INFILESIZE, content_size); } else { /* Never get here */ elog(ERROR, "illegal HTTP method"); } } else if ( method == HTTP_DELETE ) { CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, "DELETE"); } else if ( method == HTTP_HEAD ) { CURL_SETOPT(g_http_handle, CURLOPT_NOBODY, 1); } else if ( method == HTTP_PUT || method == HTTP_POST ) { /* If we had a content we do not reach that part */ elog(ERROR, "http_request.content is NULL"); } else if ( method == HTTP_UNKNOWN ){ /* Assume the user knows what they are doing and pass unchanged */ CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, method_str); } pfree(method_str); /* Set the headers */ CURL_SETOPT(g_http_handle, CURLOPT_HTTPHEADER, headers); /************************************************************************* * PERFORM THE REQUEST! **************************************************************************/ http_return = curl_easy_perform(g_http_handle); elog(DEBUG2, "pgsql-http: queried '%s'", uri); elog(DEBUG2, "pgsql-http: http_return '%d'", http_return); /* Clean up some input things we don't need anymore */ ReleaseTupleDesc(tup_desc); pfree(values); pfree(nulls); /************************************************************************* * Create an http_response object from the curl results *************************************************************************/ /* Write out an error on failure */ if ( http_return != CURLE_OK ) { curl_slist_free_all(headers); curl_easy_cleanup(g_http_handle); g_http_handle = NULL; #if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ /* * If the request was aborted by an interrupt request * we need to ensure that the interrupt signal * is in turn sent to the downstream interrupt handler * that we stored when we set up our own handler. */ if (http_return == CURLE_ABORTED_BY_CALLBACK && pgsql_interrupt_handler && http_interrupt_requested) { elog(DEBUG2, "calling pgsql_interrupt_handler"); (*pgsql_interrupt_handler)(http_interrupt_requested); http_interrupt_requested = 0; elog(ERROR, "HTTP request cancelled"); } #endif http_error(http_return, http_error_buffer); } /* Read the metadata from the handle directly */ if ( (CURLE_OK != curl_easy_getinfo(g_http_handle, CURLINFO_RESPONSE_CODE, &long_status)) || (CURLE_OK != curl_easy_getinfo(g_http_handle, CURLINFO_CONTENT_TYPE, &content_type)) ) { curl_slist_free_all(headers); curl_easy_cleanup(g_http_handle); g_http_handle = NULL; ereport(ERROR, (errmsg("CURL: Error in curl_easy_getinfo"))); } /* Prepare our return object */ if (get_call_result_type(fcinfo, 0, &tup_desc) != TYPEFUNC_COMPOSITE) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s called with incompatible return type", __func__))); } ncolumns = tup_desc->natts; values = palloc0(sizeof(Datum)*ncolumns); nulls = palloc0(sizeof(bool)*ncolumns); /* Status code */ status = long_status; values[RESP_STATUS] = Int32GetDatum(status); nulls[RESP_STATUS] = false; /* Content type */ if ( content_type ) { List *ctl; ListCell *lc; values[RESP_CONTENT_TYPE] = CStringGetTextDatum(content_type); nulls[RESP_CONTENT_TYPE] = false; /* Read the character set name out of the content type */ /* if there is one in there */ /* text/html; charset=iso-8859-1 */ if ( SplitIdentifierString(pstrdup(content_type), ';', &ctl) ) { foreach(lc, ctl) { /* charset=iso-8859-1 */ const char *param = (const char *) lfirst(lc); const char *paramtype = "charset="; if ( http_strcasestr(param, paramtype) ) { /* iso-8859-1 */ const char *charset = param + strlen(paramtype); content_charset = pg_char_to_encoding(charset); break; } } } } else { values[RESP_CONTENT_TYPE] = (Datum)0; nulls[RESP_CONTENT_TYPE] = true; } /* Headers array */ if ( si_headers.len ) { /* Strip the carriage-returns, because who cares? */ string_info_remove_cr(&si_headers); values[RESP_HEADERS] = PointerGetDatum(header_string_to_array(&si_headers)); nulls[RESP_HEADERS] = false; } else { values[RESP_HEADERS] = (Datum)0; nulls[RESP_HEADERS] = true; } /* Content */ if ( si_data.len ) { char *content_str; size_t content_len; elog(DEBUG2, "pgsql-http: content_charset = %d", content_charset); /* Apply character transcoding if necessary */ if ( content_charset < 0 ) { content_str = si_data.data; content_len = si_data.len; } else { content_str = pg_any_to_server(si_data.data, si_data.len, content_charset); content_len = strlen(content_str); } values[RESP_CONTENT] = PointerGetDatum(cstring_to_text_with_len(content_str, content_len)); nulls[RESP_CONTENT] = false; } else { values[RESP_CONTENT] = (Datum)0; nulls[RESP_CONTENT] = true; } /* Build up a tuple from values/nulls lists */ tuple_out = heap_form_tuple(tup_desc, values, nulls); /* Clean up */ ReleaseTupleDesc(tup_desc); if ( !g_use_keepalive ) { curl_easy_cleanup(g_http_handle); g_http_handle = NULL; } curl_slist_free_all(headers); pfree(si_headers.data); pfree(si_data.data); pfree(values); pfree(nulls); /* Return */ PG_RETURN_DATUM(HeapTupleGetDatum(tuple_out)); } /* URL Encode Escape Chars */ /* 45-46 (-.) 48-57 (0-9) 65-90 (A-Z) */ /* 95 (_) 97-122 (a-z) 126 (~) */ static int chars_to_not_encode[] = { 0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,1,1,0,1,1, 1,1,1,1,1,1,1,1,0,0, 0,0,0,0,0,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1, 1,0,0,0,0,1,0,1,1,1, 1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1, 1,1,1,0,0,0,1,0 }; /* * Take in a text pointer and output a cstring with * all encodable characters encoded. */ static char* urlencode_cstr(const char* str_in, size_t str_in_len) { char *str_out, *ptr; size_t i; int rv; if (!str_in_len) return pstrdup(""); /* Prepare the output string, encoding can fluff the ouput */ /* considerably */ str_out = palloc0(str_in_len * 4); ptr = str_out; for (i = 0; i < str_in_len; i++) { unsigned char c = str_in[i]; /* Break on NULL */ if (c == '\0') break; /* Replace ' ' with '+' */ if (c == ' ') { *ptr = '+'; ptr++; continue; } /* Pass basic characters through */ if ((c < 127) && chars_to_not_encode[(int)(str_in[i])]) { *ptr = str_in[i]; ptr++; continue; } /* Encode the remaining chars */ rv = snprintf(ptr, 4, "%%%02X", c); if ( rv < 0 ) return NULL; /* Move pointer forward */ ptr += 3; } *ptr = '\0'; return str_out; } /** * Utility function for users building URL encoded requests, applies * standard URL encoding to an input string. */ Datum urlencode(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(urlencode); Datum urlencode(PG_FUNCTION_ARGS) { /* Declare SQL function strict, so no test for NULL input */ text *txt = PG_GETARG_TEXT_P(0); char *encoded = urlencode_cstr(VARDATA(txt), VARSIZE_ANY_EXHDR(txt)); if (encoded) PG_RETURN_TEXT_P(cstring_to_text(encoded)); else PG_RETURN_NULL(); } /** * Treat the top level jsonb map as a key/value set * to be fed into urlencode and return a correctly * encoded data string. */ Datum urlencode_jsonb(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(urlencode_jsonb); Datum urlencode_jsonb(PG_FUNCTION_ARGS) { bool skipNested = false; Jsonb* jb = PG_GETARG_JSONB_P(0); JsonbIterator *it; JsonbValue v; JsonbIteratorToken r; StringInfoData si; size_t count = 0; if (!JB_ROOT_IS_OBJECT(jb)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot call %s on a non-object", __func__))); } /* Buffer to write complete output into */ initStringInfo(&si); it = JsonbIteratorInit(&jb->root); while ((r = JsonbIteratorNext(&it, &v, skipNested)) != WJB_DONE) { skipNested = true; if (r == WJB_KEY) { char *key, *key_enc, *value, *value_enc; /* Skip zero-length key */ if(!v.val.string.len) continue; /* Read and encode the key */ key = pnstrdup(v.val.string.val, v.val.string.len); key_enc = urlencode_cstr(v.val.string.val, v.val.string.len); /* Read the value for this key */ #if PG_VERSION_NUM < 130000 { JsonbValue k; k.type = jbvString; k.val.string.val = key; k.val.string.len = strlen(key); v = *findJsonbValueFromContainer(&jb->root, JB_FOBJECT, &k); } #else getKeyJsonValueFromContainer(&jb->root, key, strlen(key), &v); #endif /* Read and encode the value */ switch(v.type) { case jbvString: { value = pnstrdup(v.val.string.val, v.val.string.len); break; } case jbvNumeric: { value = numeric_normalize(v.val.numeric); break; } case jbvBool: { value = pstrdup(v.val.boolean ? "true" : "false"); break; } case jbvNull: { value = pstrdup(""); break; } default: { elog(DEBUG2, "skipping non-scalar value of '%s'", key); continue; } } /* Write the result */ value_enc = urlencode_cstr(value, strlen(value)); if (count++) appendStringInfo(&si, "&"); appendStringInfo(&si, "%s=%s", key_enc, value_enc); /* Clean up temporary strings */ if (key) pfree(key); if (value) pfree(value); if (key_enc) pfree(key_enc); if (value_enc) pfree(value_enc); } } if (si.len) PG_RETURN_TEXT_P(cstring_to_text_with_len(si.data, si.len)); else PG_RETURN_NULL(); } Datum bytea_to_text(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(bytea_to_text); Datum bytea_to_text(PG_FUNCTION_ARGS) { bytea *b = PG_GETARG_BYTEA_P(0); text *t = palloc(VARSIZE_ANY(b)); memcpy(t, b, VARSIZE(b)); PG_RETURN_TEXT_P(t); } Datum text_to_bytea(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(text_to_bytea); Datum text_to_bytea(PG_FUNCTION_ARGS) { text *t = PG_GETARG_TEXT_P(0); bytea *b = palloc(VARSIZE_ANY(t)); memcpy(b, t, VARSIZE(t)); PG_RETURN_TEXT_P(b); } // Local Variables: // mode: C++ // tab-width: 4 // c-basic-offset: 4 // indent-tabs-mode: t // End: pgsql-http-1.6.0/http.control000066400000000000000000000002201446520663200162040ustar00rootroot00000000000000default_version = '1.6' module_pathname = '$libdir/http' comment = 'HTTP client for PostgreSQL, allows web page retrieval inside the database.' pgsql-http-1.6.0/sql/000077500000000000000000000000001446520663200144305ustar00rootroot00000000000000pgsql-http-1.6.0/sql/http.sql000066400000000000000000000100751446520663200161330ustar00rootroot00000000000000CREATE EXTENSION http; set http.timeout_msec = 10000; SELECT http_set_curlopt('CURLOPT_TIMEOUT', '10'); -- Status code SELECT status FROM http_get('https://httpbun.org/status/202'); -- Headers SELECT lower(field) AS field, value FROM ( SELECT (unnest(headers)).* FROM http_get('https://httpbun.org/response-headers?Abcde=abcde') ) a WHERE field ILIKE 'Abcde'; -- GET SELECT status, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_get('https://httpbun.org/anything?foo=bar'); -- GET with data SELECT status, content::json->'args'->>'this' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_get('https://httpbun.org/anything', jsonb_build_object('this', 'that')); -- GET with data SELECT status, content::json->'args' as args, content::json->>'data' as data, content::json->'url' as url, content::json->'method' as method from http(('GET', 'https://httpbun.org/anything', NULL, 'application/json', '{"search": "toto"}')); -- DELETE SELECT status, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_delete('https://httpbun.org/anything?foo=bar'); -- DELETE with payload SELECT status, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method, content::json->'data' AS data FROM http_delete('https://httpbun.org/anything?foo=bar', 'payload', 'text/plain'); -- PUT SELECT status, content::json->'data' AS data, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_put('https://httpbun.org/anything?foo=bar','payload','text/plain'); -- PATCH SELECT status, content::json->'data' AS data, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_patch('https://httpbun.org/anything?foo=bar','{"this":"that"}','application/json'); -- POST SELECT status, content::json->'data' AS data, content::json->'args'->>'foo' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_post('https://httpbun.org/anything?foo=bar','payload','text/plain'); -- POST with json data SELECT status, content::json->'form'->>'this' AS args, content::json->'url' AS url, content::json->'method' AS method FROM http_post('https://httpbun.org/anything', jsonb_build_object('this', 'that')); -- POST with data SELECT status, content::json->'form'->>'key1' AS key1, content::json->'form'->>'key2' AS key2, content::json->'url' AS url, content::json->'method' AS method FROM http_post('https://httpbun.org/anything', 'key1=value1&key2=value2','application/x-www-form-urlencoded'); -- HEAD SELECT lower(field) AS field, value FROM ( SELECT (unnest(headers)).* FROM http_head('https://httpbun.org/response-headers?Abcde=abcde') ) a WHERE field ILIKE 'Abcde'; -- Follow redirect SELECT status, content::json->'url' AS url FROM http_get('https://httpbun.org/redirect-to?url=get'); -- Request image WITH http AS ( SELECT * FROM http_get('https://httpbingo.org/image/png') ), headers AS ( SELECT (unnest(headers)).* FROM http ) SELECT http.content_type, length(text_to_bytea(http.content)) AS length_binary FROM http, headers WHERE field ilike 'Content-Type'; -- Alter options and and reset them and throw errors SELECT http_set_curlopt('CURLOPT_PROXY', '127.0.0.1'); -- Error because proxy is not there DO $$ BEGIN SELECT status FROM http_get('https://httpbun.org/status/555'); EXCEPTION WHEN OTHERS THEN RAISE WARNING 'Failed to connect'; END; $$; -- Still an error DO $$ BEGIN SELECT status FROM http_get('https://httpbun.org/status/555'); EXCEPTION WHEN OTHERS THEN RAISE WARNING 'Failed to connect'; END; $$; -- Reset options SELECT http_reset_curlopt(); -- Now it should work SELECT status FROM http_get('https://httpbun.org/status/555'); -- Alter the default timeout and then run a query that is longer than -- the default (5s), but shorter than the new timeout SELECT http_set_curlopt('CURLOPT_TIMEOUT_MS', '10000'); SELECT status FROM http_get('https://httpbun.org/delay/7');