META.json000644000766000024 322513416043414 13351 0ustar00st21277staff000000000000Hijk-0.28{ "abstract" : "Fast & minimal low-level HTTP client", "author" : [ "Kang-min Liu ", "Ævar Arnfjörð Bjarmason ", "Borislav Nikolov ", "Damian Gryski " ], "dynamic_config" : 0, "generated_by" : "App::ModuleBuildTiny version 0.025", "license" : [ "mit" ], "meta-spec" : { "url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec", "version" : 2 }, "name" : "Hijk", "prereqs" : { "configure" : { "requires" : { "Module::Build::Tiny" : "0" } }, "develop" : { "requires" : { "App::ModuleBuildTiny" : "0.025" } }, "runtime" : { "requires" : { "Time::HiRes" : "0" } }, "test" : { "requires" : { "HTTP::Server::Simple::PSGI" : "0", "Net::Ping" : "2.41", "Net::Server::HTTP" : "0", "Plack" : "0", "Test::Exception" : "0", "Test::More" : "0" } } }, "provides" : { "Hijk" : { "file" : "lib/Hijk.pm", "version" : "0.28" } }, "release_status" : "stable", "resources" : { "bugtracker" : { "web" : "https://github.com/gugod/Hijk/issues" }, "repository" : { "type" : "git", "url" : "https://github.com/gugod/Hijk.git", "web" : "https://github.com/gugod/Hijk" } }, "version" : "0.28", "x_serialization_backend" : "JSON::PP version 2.97001", "x_spdx_expression" : "MIT", "x_static_install" : "1" } README000644000766000024 3326613416043414 12640 0ustar00st21277staff000000000000Hijk-0.28NAME Hijk - Fast & minimal low-level HTTP client SYNOPSIS A simple GET request: use Hijk (); my $res = Hijk::request({ method => "GET", host => "example.com", port => "80", path => "/flower", query_string => "color=red" }); if (exists $res->{error} and $res->{error} & Hijk::Error::TIMEOUT) { die "Oh noes we had some sort of timeout"; } die "Expecting an 'OK' response" unless $res->{status} == 200; say $res->{body}; A POST request, you have to manually set the appropriate headers, URI escape your values etc. use Hijk (); use URI::Escape qw(uri_escape); my $res = Hijk::request({ method => "POST", host => "example.com", port => "80", path => "/new", head => [ "Content-Type" => "application/x-www-form-urlencoded" ], query_string => "type=flower&bucket=the%20one%20out%20back", body => "description=" . uri_escape("Another flower, let's hope it's exciting"), }); die "Expecting an 'OK' response" unless $res->{status} == 200; DESCRIPTION Hijk is a fast & minimal low-level HTTP client intended to be used where you control both the client and the server, e.g. for talking to some internal service from a frontend user-facing web application. It is NOT a general HTTP user agent, it doesn't support redirects, proxies, SSL and any number of other advanced HTTP features like (in roughly descending order of feature completeness) LWP::UserAgent, WWW::Curl, HTTP::Tiny, HTTP::Lite or Furl. This library is basically one step above manually talking HTTP over sockets. Having said that it's lightning fast and extensively used in production at Booking.com where it's used as the go-to transport layer for talking to internal services. It uses non-blocking sockets and correctly handles all combinations of connect/read timeouts and other issues you might encounter from various combinations of parts of your system going down or becoming otherwise unavailable. FUNCTION: Hijk::request( $args :HashRef ) :HashRef Hijk::request is the only function you should use. It (or anything else in this package for that matter) is not exported, so you have to use the fully qualified name. It takes a HashRef of arguments and either dies or returns a HashRef as a response. The HashRef argument to it must contain some of the key-value pairs from the following list. The value for host and port are mandatory, but others are optional with default values listed below. protocol => "HTTP/1.1", # (or "HTTP/1.0") host => ..., port => ..., connect_timeout => undef, read_timeout => undef, read_length => 10240, method => "GET", path => "/", query_string => "", head => [], body => "", socket_cache => \%Hijk::SOCKET_CACHE, # (undef to disable, or \my %your_socket_cache) on_connect => undef, # (or sub { ... }) parse_chunked => 0, head_as_array => 0, no_default_host_header => 1, Notice how Hijk does not take a full URI string as input, you have to specify the individual parts of the URL. Users who need to parse an existing URI string to produce a request should use the URI module to do so. The value of head is an ArrayRef of key-value pairs instead of a HashRef, this way you can decide in which order the headers are sent, and you can send the same header name multiple times. For example: head => [ "Content-Type" => "application/json", "X-Requested-With" => "Hijk", ] Will produce these request headers: Content-Type: application/json X-Requested-With: Hijk In addition Hijk will provide a Host header for you by default with the host value you pass to request(). To suppress this (e.g. to send custom Host requests) pass a true value to the no_default_host_header option and provide your own Host header in the head ArrayRef (or don't, if you want to construct a Host-less request knock yourself out...). Hijk doesn't escape any values for you, it just passes them through as-is. You can easily produce invalid requests if e.g. any of these strings contain a newline, or aren't otherwise properly escaped. The value of connect_timeout or read_timeout is in floating point seconds, and is used as the time limit for connecting to the host, and reading the response back from it, respectively. The default value for both is undef, meaning no timeout limit. If you don't supply these timeouts and the host really is unreachable or slow, we'll reach the TCP timeout limit before returning some other error to you. The default protocol is HTTP/1.1, but you can also specify HTTP/1.0. The advantage of using HTTP/1.1 is support for keep-alive, which matters a lot in environments where the connection setup represents non-trivial overhead. Sometimes that overhead is negligible (e.g. on Linux talking to an nginx on the local network), and keeping open connections down and reducing complexity is more important, in those cases you can either use HTTP/1.0, or specify Connection: close in the request, but just using HTTP/1.0 is an easy way to accomplish the same thing. By default we will provide a socket_cache for you which is a global singleton that we maintain keyed on join($;, $$, $host, $port). Alternatively you can pass in socket_cache hash of your own which we'll use as the cache. To completely disable the cache pass in undef. The optional on_connect callback is intended to be used for you to figure out from production traffic what you should set the connect_timeout. I.e. you can start a timer when you call Hijk::request() that you end when on_connect is called, that's how long it took us to get a connection. If you start another timer in that callback that you end when Hijk::request() returns to you that'll give you how long it took to send/receive data after we constructed the socket, i.e. it'll help you to tweak your read_timeout. The on_connect callback is provided with no arguments, and is called in void context. We have experimental support for parsing chunked responses encoding. historically Hijk didn't support this at all and if you wanted to use it with e.g. nginx you had to add chunked_transfer_encoding off to the nginx config file. Since you may just want to do that instead of having Hijk do more work to parse this out with a more complex and experimental codepath you have to explicitly enable it with parse_chunked. Otherwise Hijk will die when it encounters chunked responses. The parse_chunked option may be turned on by default in the future. The return value is a HashRef representing a response. It contains the following key-value pairs. proto => :Str status => :StatusCode body => :Str head => :HashRef (or :ArrayRef with "head_as_array") error => :PositiveInt error_message => :Str errno_number => :Int errno_string => :Str For example, to send a request to http://example.com/flower?color=red, pass the following parameters: my $res = Hijk::request({ host => "example.com", port => "80", path => "/flower", query_string => "color=red" }); die "Response is not 'OK'" unless $res->{status} == 200; Notice that you do not need to put the leading "?" character in the query_string. You do, however, need to properly uri_escape the content of query_string. Again, Hijk doesn't escape any values for you, so these values MUST be properly escaped before being passed in, unless you want to issue invalid requests. By default the head in the response is a HashRef rather then an ArrayRef. This makes it easier to retrieve specific header fields, but it means that we'll clobber any duplicated header names with the most recently seen header value. To get the returned headers as an ArrayRef instead specify head_as_array. If you want to fiddle with the read_length value it controls how much we POSIX::read($fd, $buf, $read_length) at a time. We currently don't support servers returning a http body without an accompanying Content-Length header; bodies MUST have a Content-Length or we won't pick them up. ERROR CODES If we had a recoverable error we'll include an "error" key whose value is a bitfield that you can check against Hijk::Error::* constants. Those are: Hijk::Error::CONNECT_TIMEOUT Hijk::Error::READ_TIMEOUT Hijk::Error::TIMEOUT Hijk::Error::CANNOT_RESOLVE Hijk::Error::REQUEST_SELECT_ERROR Hijk::Error::REQUEST_WRITE_ERROR Hijk::Error::REQUEST_ERROR Hijk::Error::RESPONSE_READ_ERROR Hijk::Error::RESPONSE_BAD_READ_VALUE Hijk::Error::RESPONSE_ERROR In addition we might return error_message, errno_number and errno_string keys, see the discussion of Hijk::Error::REQUEST_* and Hijk::Error::RESPONSE_* errors below. The Hijk::Error::TIMEOUT constant is the same as Hijk::Error::CONNECT_TIMEOUT | Hijk::Error::READ_TIMEOUT. It's there for convenience so you can do: .. if exists $res->{error} and $res->{error} & Hijk::Error::TIMEOUT; Instead of the more verbose: .. if exists $res->{error} and $res->{error} & (Hijk::Error::CONNECT_TIMEOUT | Hijk::Error::READ_TIMEOUT) We'll return Hijk::Error::CANNOT_RESOLVE if we can't gethostbyname() the host you've provided. If we fail to do a select() or write() during when sending the response we'll return Hijk::Error::REQUEST_SELECT_ERROR or Hijk::Error::REQUEST_WRITE_ERROR, respectively. Similarly to Hijk::Error::TIMEOUT the Hijk::Error::REQUEST_ERROR constant is a union of these two, and any other request errors we might add in the future. When we're getting the response back we'll return Hijk::Error::RESPONSE_READ_ERROR when we can't read() the response, and Hijk::Error::RESPONSE_BAD_READ_VALUE when the value we got from read() is 0. The Hijk::Error::RESPONSE_ERROR constant is a union of these two and any other response errors we might add in the future. Some of these Hijk::Error::REQUEST_* and Hijk::Error::RESPONSE_* errors are re-thrown errors from system calls. In that case we'll also pass along error_message which is a short human readable error message about the error, as well as errno_number & errno_string, which are $!+0 and "$!" at the time we had the error. Hijk might encounter other errors during the course of the request and WILL call die if that happens, so if you don't want your program to stop when a request like that fails wrap it in eval. Having said that the point of the Hijk::Error::* interface is that all errors that happen during normal operation, i.e. making valid requests against servers where you can have issues like timeouts, network blips or the server thread on the other end being suddenly kill -9'd should be caught, categorized and returned in a structural way by Hijk. We're not currently aware of any issues that occur in such normal operations that aren't classified as a Hijk::Error::*, and if we find new issues that fit the criteria above we'll likely just make a new Hijk::Error::* for it. We're just not trying to guarantee that the library can never die, and aren't trying to catch truly exceptional issues like e.g. fcntl() failing on a valid socket. AUTHORS Kang-min Liu Ævar Arnfjörð Bjarmason Borislav Nikolov Damian Gryski COPYRIGHT Copyright (c) 2013- Kang-min Liu . LICENCE The MIT License DISCLAIMER OF WARRANTY BECAUSE THIS SOFTWARE IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE SOFTWARE, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE SOFTWARE "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE SOFTWARE IS WITH YOU. SHOULD THE SOFTWARE PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR, OR CORRECTION. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE SOFTWARE AS PERMITTED BY THE ABOVE LICENCE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE SOFTWARE (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE SOFTWARE TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. plack.t000644000766000024 307613416043414 13456 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use FindBin; use Hijk; use Test::More; use Test::Exception; my $port = 10000 + int rand(5000); my $pid = fork; die "Fail to fork then start a plack server" unless defined $pid; if ($pid == 0) { require Plack::Runner; my $runner = Plack::Runner->new; $runner->parse_options("--port", $port, "$FindBin::Bin/bin/it-takes-time.psgi"); $runner->run; exit; } sleep 10; # hopfully this is enough to launch that psgi. my %args = ( host => "localhost", port => $port, query_string => "t=5", method => "GET", ); subtest "expect connection failure (mismatching port number)" => sub { dies_ok { my $port_wrong = $port; while ($port_wrong == $port) { $port_wrong = int( 15001+rand()*3000 ); } diag "Connecting to a wrong port: $port"; my $res = Hijk::request({%args, port => $port_wrong, timeout => 10}); } 'We connect to wrong port so, as expected, the connection cannot be established.'; diag "Dying message: $@"; }; subtest "expect read timeout" => sub { lives_ok { my $res = Hijk::request({%args, timeout => 1}); ok exists $res->{error}, '$res->{error} should exist becasue a read timeout is expected.'; is $res->{error}, Hijk::Error::READ_TIMEOUT, '$res->{error} == Hijk::Error::READ_TIMEOUT'; }; }; subtest "do not expect timeout" => sub { lives_ok { my $res = Hijk::request({%args, timeout => 10}); } 'local plack send back something within 10s'; }; END { kill INT => $pid if $pid } done_testing; bench-nginx.pl000644000766000024 413113416043414 16300 0ustar00st21277staff000000000000Hijk-0.28/examples#!/usr/bin/env perl use strict; use warnings; use Benchmark ':all'; use Hijk; use HTTP::Tiny; use LWP::UserAgent; use HTTP::Request; # Rate 1k.img lwp____ 1k.img tiny___ 1k.img hijk pp 1k.img hijk xs #1k.img lwp____ 820/s -- -54% -94% -95% #1k.img tiny___ 1776/s 117% -- -86% -90% #1k.img hijk pp 12821/s 1464% 622% -- -29% #1k.img hijk xs 18182/s 2118% 924% 42% -- # Rate 10k.img lwp____ 10k.img tiny___ 10k.img hijk pp 10k.img hijk xs #10k.img lwp____ 781/s -- -54% -93% -95% #10k.img tiny___ 1692/s 117% -- -85% -89% #10k.img hijk pp 11364/s 1355% 572% -- -27% #10k.img hijk xs 15625/s 1900% 823% 37% -- # Rate 100k.img lwp____ 100k.img tiny___ 100k.img hijk pp 100k.img hijk xs #100k.img lwp____ 452/s -- -62% -93% -95% #100k.img tiny___ 1179/s 161% -- -83% -86% #100k.img hijk pp 6944/s 1436% 489% -- -16% #100k.img hijk xs 8264/s 1728% 601% 19% -- foreach my $f(qw(1k.img 10k.img 100k.img)) { my $tiny = HTTP::Tiny->new(); my $req = HTTP::Request->new('GET',"http://localhost:8080/$f"); my $lwp = LWP::UserAgent->new(); cmpthese(10_000,{ $f. ' tiny___' => sub { my $res = $tiny->get("http://localhost:8080/$f"); }, $f . ' hijk pp' => sub { my $res = Hijk::request({path => "/$f", host => 'localhost', port => 8080, method => 'GET'}); }, $f . ' lwp____' => sub { my $res = $lwp->request($req); }, }); } live-elasticsearch.t000644000766000024 237713416043414 16136 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Hijk; use Test::More; if ($ENV{TRAVIS} && $ENV{TRAVIS_OS_NAME} eq 'osx') { plan skip_all => "No elasticsearch service in this environment at the moment."; } unless ($ENV{TEST_LIVE}) { plan skip_all => "Enable live testing by setting env: TEST_LIVE=1"; } unless ($ENV{TEST_ELASTICSEARCH}) { plan skip_all => "Enable live ElasticSearch testing by setting env: TEST_ELASTICSEARCH=1"; } my %args = ( host => $ENV{TEST_HOST} || "localhost", port => "9200", method => "GET", ); my @tests = ( [ path => "/_stats" ], [ path => "/_search", body => q!{"query":{"match_all":{}}}! ], [ path => "/_search", query_string => "search_type=count", body => q!{"query":{"match_all":{}}}! ], ); for ((@tests) x (300)) { my $a = {%args, @$_ }; my $res = Hijk::request($a); if ($res->{error}) { fail "Error happened when requesting $a->{path}: $res->{error}"; } else { my $res_body = $res->{body}; my $test_name = "$a->{path}\t". substr($res_body, 0, 60)."...\n"; if (substr($res_body, 0, 1) eq '{' && substr($res_body, -1, 1) eq '}' ) { pass $test_name; } else { fail $test_name; } } } done_testing; Changes000644000766000024 2227213416043414 13246 0ustar00st21277staff000000000000Hijk-0.280.28 - Released at 2019-01-11T16:25:00+0900 - Now the distribution is made with mbtiny. - Various improvements of the test suite. - Rewrite the internally-used testing-purpose HTTP server to allow the control of 'Content-Length' request header. 0.27: # 2016-10-28T12:59:00+0100 - Unbreak with Elasticeasrch 5.0. See https://rt.cpan.org/Public/Bug/Display.html?id=118425 0.26: # 2015-11-25T12:30:00+0100 - No functional changes since 0.25, but we had some Travis-specific changes in the repo, releasing just so we have the latest code there on the CPAN. 0.25: # 2015-11-25T12:20:00+0100 - Make the t/select-timeout.t test which fails on various odd CPANtesters platforms a TODO. Maybe some OS-specific issue, maybe an issue with kill() in the CPANtesters sandboxes not behaving as we expect. 0.24: # 2015-07-05T13:40:00+0200 - Minor copyediting and formatting changes to the documentation. No code changes at all. 0.23: # 2015-07-03T17:00:00+0200 - The "Host" header can now be overriden by supplying a new `no_default_host_header` option along with a `Host` header in `head => []` to request(). Before this we'd always send "Host: $host" over, where $host was the host we were connecting to, now you can customize this. - Fixed a bug where if passed passed `head => []` to request() we'd emit a ":" header, i.e. just an empty header name with an empty value. You could have just not passed the `head => ` value if the array was empty, but no we won't screw up and emit a single line consisting of ":" if given an empty array. 0.22: # 2015-05-27T07:54:17+0200 - No feature change. Re-package due to a missing file in the tarball: https://rt.cpan.org/Ticket/Display.html?id=104624 0.21: # 2015-05-22T15:26:23+0200 - Fix "Too many CRLF" issue. Hijk has been always generating HTTP request with an extra CRLF at the end. While many HTTP servers are ignoring those, some treat it as errors. We now eliminate the extra CRLF at the end of every request. See also http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html - Handle better when select() is interrupted by signals. 0.20: # 2015-03-20-T15:10:00+0000 - Fix a regression in 0.14. When the chunked encoding support was introduced we accidentally stopped supporting "Content-Length: 0", treat those responses as the zero-body again. This broke e.g. PUT requests that would return no payload. - Add support for 204 No Content responses. According to the HTTP standard we must try to consume the body if there's no Content-Length, but not if the server returns a 204 response, then it MUST NOT include a body (see http://tools.ietf.org/html/rfc2616#page-60). This re-adds support for e.g. 204 No Content response, in practice this "worked" before 0.14, but only accidentally and other types of responses wouldn't work. - We now handle our system calls returning EINTR. 0.19: # 2015-01-10-T18:30:00+0000 - Fix a major regression in 0.16. The introduction of "head_as_array" completely broke the disconnection logic when the socket_cache was enabled (which is the default). When talking to a webserver that would disconnect us after N requests request N+1 would always fail with a 0 byte response error. This issue was reported as RT #101424 (https://rt.cpan.org/Public/Bug/Display.html?id=101424) - Fix a minor regression in 0.16: The introduction of "head_as_array" broke the "proto" part of the return value in a relatively obscure edge case where we'd read the header & had no Content-Length and couldn't read() anything. - Fix an edge case in the Trailer support. It would only kick in if we got the Transfer-Encoding header before the "Trailer" header, not the other way around. 0.18: # 2014-12-10T14:00:00+000 - We now do the right thing on "method => 'HEAD'". I.e. ignore the Content-Length parameter, previously we'd just hang trying to slurp up the body. - Fix an edge case with some of the live tests leaving around a HTTP server if they died, these don't run by default. 0.17: # 2014-08-31T18:30:00+000 - Minor documentation changes, no functional changes. - The version number for the last release was incorrect in this changelog, fixed in this release. 0.16: # 2014-08-31T00:10:00+000 - Major Change: There are several new Hijk::Error::* constants for common issues that happened during normal requests in the face of regular exceptions out of the control of this library, such as network blips. Existing code that checks $res->{error} should be forwards-compatible with this change, but anything that was doing e.g. regex checks against regular errors thrown by this library should be updated to check the new Hijk::Error::* constants instead. - It's now possible to specify "head_as_array" to get the returned headers as an array (with potential duplicated headers), rather than the default behavior of lossily returning them as a hash. - There's now a "read_length" option to control how much we POSIX::read($fd, $buf, $read_length) at a time. We don't expect this to be useful, it's mainly configurable on general principle so we don't have arbitrary unconfigurable hardcoded constants in the source. 0.15: # 2014-08-30T10:00:00+000 - The new code to support chunked transfer encoding would return a nonexisting Hijk::Error::* value of "0" when it encountered a read timeout. This meant that not only was the error reporting broken, but anything checking if there were errors via the simple idiom of "if ($res->{error}) {...}" wouldn't properly report errors. We'll now correctly report these errors as Hijk::Error::READ_TIMEOUT. - Since there may still be other bugs like that in this new parsing mode it's disabled by default, if you know you want to parse chunked responses you have to pass parse_chunked => 1 for now. Usually you probably just want to disable chunked encoding on the other end, see the note about how to do that with nginx in the docs. 0.14: # 2014-08-29T15:40:36+0900 - Start support chunked transfer encoding. 0.13: # 2014-04-27T20:00:43+0200 - Switch to use non-blocknig fd to avoid a rare deadlock situation when select() is successful and the following read() blocks forever because there are really nothing to read. 0.12: # 2014-01-31T18:20:00+0100 - Instead of dying on e.g. "Bad arg length for Socket::pack_sockaddr_in, length is 0, should be 4" when given a host we can't resolve we'll now return a $res with the error value set to Hijk::Error::CANNOT_RESOLVEif we can't gethostbyname() the provided hostname. Makes it easier to handle DNS resolution failures. 0.11: # 2014-01-06T13:20:00+0100 - Fixed broken HTTP header parsing for servers that didn't return the entire header all at the same time, but in chunks. - We now return "proto" as well as "status" etc. in the response, so you can see what the protocol the server was using to speak to us. Also we pro-actively connections to servers that claim they're speaking HTTP/1.0. - Document that what the socket_cache is keyed on, for anyone wanting to implement a tied hash or whatever. - Fix a minor bug causing redundant work under "socket_cache => undef" 0.10: # 2013-12-19T16:50:00+0100 - We can now talk HTTP/1.0 an addition to HTTP/1.1, have a way to disable the socket cache, and can specify connect and read timeouts independently. - Fix a really nasty bug with mixings up requests after encountering a timeout. See http://lists.unbit.it/pipermail/uwsgi/2013-December/006802.html for details. - Remove spurious requirenment on perl v5.14.2 - First stab at https://github.com/gugod/Hijk/issues/3 we'll now return an error key in the response with Hijk::Error::{CONNECT_TIMEOUT,READ_TIMEOUT} instead of dying. - Nuked the Hijk::HTTP::XS support from the repo, we've decided it was too complex for its own good. - Add support for an on_connect callback for seeing how long the connect/reads take. 0.09: # 2013-12-13T07:38:25+0100 - KEEP CALM AND REMOVE FETCH OPTION - Hijk::request will use XS parser only if Hijk::HTTP::XS is loaded 0.08: # 2013-12-12T20:10:00+0100 - We only checked for undefined return codes from POSIX::read(), not 0, resulting in an infinite select/read loop when a server with keep-alive enabled cut off our connection. 0.07: # 2013-12-09T12:50:00+0100 - Skip the live connect timeout test by default, it will fail making live connections on various firewalled/locked down hosts. 0.06: # 2013-12-09T12:20:00+0100 - Declare missing test dependency on Test::Exception - Declare test dependency on Net::Ping 2.41 - Various POD improvements describing more limitations in the API and providing examples. - Don't unconditionally load the yet-to-be-released Hijk::HTTP::XS module, instead provide a "fetch" option. - Shutdown and delete the cached connection in case of read error. - Handle syswrite() returning undef without spewing an uninitialized comparison error - Various work on the test suite. 0.05: # 2013-12-04T22:33:31+0100 - Properly invalidate connection cache when seeing 'Connection: close' in the response. 0.04: # 2013-12-04T00:06:16+0100 - Implement 'connect timeout' and 'read timeout' 0.02: # 2013-11-24T16:14:20+0100 - Passthrug extra HTTP header with the 'head' request arg. 0.01: # 2013-11-24T01:49:08+0100 - Initial Release, with all wanted features are implemented. bench-elasticsearch.pl000644000766000024 223013416043414 17765 0ustar00st21277staff000000000000Hijk-0.28/examples#!/usr/bin/env perl use strict; use warnings; use Benchmark ':all'; use Hijk; use HTTP::Tiny; use LWP::UserAgent; use HTTP::Request; my $tiny = HTTP::Tiny->new(); my $req = HTTP::Request->new('GET','http://localhost:9200/_search'); my $body = '{"query":{"match_all":{}}}'; $req->content($body); my $lwp = LWP::UserAgent->new(); # current results on Intel(R) Core(TM)2 Duo CPU P8400@2.26GHz with 2gb ram # and elasticsearch with one index containing ~ 500 small documents: # Rate lwp____ tiny___ hijk pp hijk xs #lwp____ 593/s -- -52% -94% -95% #tiny___ 1235/s 108% -- -88% -90% #hijk pp 10101/s 1602% 718% -- -22% #hijk xs 12987/s 2088% 952% 29% -- cmpthese(10_000,{ 'tiny___' => sub { my $res = $tiny->get('http://localhost:9200/_search',{content => $body }); }, 'hijk pp' => sub { my $res = Hijk::request({path => "/_search", body => $body, host => 'localhost', port => 9200, method => 'GET'}); }, 'lwp____' => sub { my $res = $lwp->request($req); }, }); cpanfile000644000766000024 45213416043414 13413 0ustar00st21277staff000000000000Hijk-0.28requires "Time::HiRes"; on configure => sub { requires "Module::Build::Tiny"; }; on test => sub { requires "Test::More"; requires "Test::Exception"; requires "Plack"; requires "HTTP::Server::Simple::PSGI"; requires "Net::Ping", '2.41'; requires 'Net::Server::HTTP'; }; Build.PL000644000766000024 15213416043414 13200 0ustar00st21277staff000000000000Hijk-0.28# This Build.PL for Hijk was generated by mbtiny 0.025. use 5.006; use Module::Build::Tiny 0; Build_PL(); parse-http-no-content-len-message.t000644000766000024 231013416043414 20721 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Test::More; use File::Temp (); use File::Temp qw/ :seekable /; use Hijk; my $fh = File::Temp->new(); my $fd = do { local $/ = undef; my $msg = join( "\x0d\x0a", 'HTTP/1.1 200 OK', 'Date: Sat, 23 Nov 2013 23:10:28 GMT', 'Last-Modified: Sat, 26 Oct 2013 19:41:47 GMT', 'ETag: "4b9d0211dd8a2819866bccff777af225"', 'Content-Type: text/html', 'Server: Example', 'Connection: close', '', 'a' x 100000 ); print $fh $msg; $fh->flush; $fh->seek(0, 0); fileno($fh); }; my (undef, $proto, $status, $head, $body) = Hijk::_read_http_message($fd, 10240, 0); is $status, 200; is_deeply $head, { "Date" => "Sat, 23 Nov 2013 23:10:28 GMT", "Last-Modified" => "Sat, 26 Oct 2013 19:41:47 GMT", "ETag" => '"4b9d0211dd8a2819866bccff777af225"', "Content-Type" => "text/html", "Server" => "Example", "Connection" => "close", }; is $body, 'a' x 100000; (undef, $proto, $status, $head, $body, my $error, my $error_message) = Hijk::_read_http_message($fd, 10240); is $error, Hijk::Error::RESPONSE_BAD_READ_VALUE; like $error_message, qr/0 byte/; done_testing; metamerge.json000644000766000024 50013416043414 14542 0ustar00st21277staff000000000000Hijk-0.28{ "x_static_install": true, "resources": { "bugtracker": { "web": "https://github.com/gugod/Hijk/issues" }, "repository": { "web": "https://github.com/gugod/Hijk", "url": "https://github.com/gugod/Hijk.git", "type": "git" } } } live-couchdb.t000644000766000024 415413416043414 14726 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Test::More; use Hijk; use URI; use Time::HiRes 'time'; if ($ENV{TRAVIS} && $ENV{TRAVIS_OS_NAME} eq 'osx') { plan skip_all => "No elasticsearch service in this environment at the moment."; } plan skip_all => "Enable live testing by setting env: TEST_LIVE=1" unless $ENV{TEST_LIVE}; plan skip_all => "Enable live CouchDB testing by setting env: TEST_COUCHDB=http://localhost:5984/" unless $ENV{TEST_COUCHDB}; my $uri = URI->new($ENV{TEST_COUCHDB}); plan skip_all => "Fail to parse the value of TEST_COUCHDB: $ENV{TEST_COUCHDB}" unless $uri->isa("URI::http"); subtest "get the welcome message" => sub { my $rd = { host => $uri->host, port => $uri->port }; my $res; my $t0 = time; my $count = my $total = 1000; my $ok = 0; while ($count--) { $res = Hijk::request($rd); $ok++ if $res->{status} eq '200'; } my $t1 = time; is $ok, $total, sprintf("spent %f s", $t1 - $t0); }; subtest "create database, then delete it." => sub { my $db_name = "hijk_test_$$"; my $rd = { host => $uri->host, port => $uri->port, path => "/${db_name}", method => "PUT", }; my $res = Hijk::request($rd); if ($res->{status} eq '412') { pass "db $db_name already exists (unexpected, but it is fine): $res->{body}"; } else { pass "db $db_name created"; is $res->{status}, '201', "status = 201. see http://docs.couchdb.org/en/latest/intro/api.html#databases"; my $res2 = Hijk::request($rd); if ($res2->{status} eq '412') { pass "The 2nd creation request is done with error (expected): $res->{body}"; } else { fail "The 2nd request is done without error, that is unexpected. http_status = $res2->{status}, $res2->{body}"; } } $rd->{method} = "GET"; $res = Hijk::request($rd); is $res->{status}, '200', "$db_name exists. res_body = $res->{body}"; $rd->{method} = "DELETE"; $res = Hijk::request($rd); is $res->{status}, '200', "$db_name is deleted. res_body = $res->{body}"; }; done_testing; Hijk.pm000644000766000024 7444313416043414 13753 0ustar00st21277staff000000000000Hijk-0.28/libpackage Hijk; use strict; use warnings; use Time::HiRes; use POSIX qw(:errno_h); use Socket qw(PF_INET SOCK_STREAM pack_sockaddr_in $CRLF SOL_SOCKET SO_ERROR); use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK); our $VERSION = "0.28"; sub Hijk::Error::CONNECT_TIMEOUT () { 1 << 0 } # 1 sub Hijk::Error::READ_TIMEOUT () { 1 << 1 } # 2 sub Hijk::Error::TIMEOUT () { Hijk::Error::READ_TIMEOUT | Hijk::Error::CONNECT_TIMEOUT } # 3 sub Hijk::Error::CANNOT_RESOLVE () { 1 << 2 } # 4 sub Hijk::Error::REQUEST_SELECT_ERROR () { 1 << 3 } # 8 sub Hijk::Error::REQUEST_WRITE_ERROR () { 1 << 4 } # 16 sub Hijk::Error::REQUEST_ERROR () { Hijk::Error::REQUEST_SELECT_ERROR | Hijk::Error::REQUEST_WRITE_ERROR } # 24 sub Hijk::Error::RESPONSE_READ_ERROR () { 1 << 5 } # 32 sub Hijk::Error::RESPONSE_BAD_READ_VALUE () { 1 << 6 } # 64 sub Hijk::Error::RESPONSE_ERROR () { Hijk::Error::RESPONSE_READ_ERROR | Hijk::Error::RESPONSE_BAD_READ_VALUE } # 96 sub _read_http_message { my ($fd, $read_length, $read_timeout, $parse_chunked, $head_as_array, $method) = @_; $read_timeout = undef if defined($read_timeout) && $read_timeout <= 0; my ($body,$buf,$decapitated,$nbytes,$proto); my $status_code = 0; my $header = $head_as_array ? [] : {}; my $no_content_len = 0; my $head = ""; my $method_has_no_content = do { no warnings qw(uninitialized); $method eq "HEAD" }; my $close_connection; vec(my $rin = '', $fd, 1) = 1; do { return ($close_connection,undef,0,undef,undef, Hijk::Error::READ_TIMEOUT) if ((_select($rin, undef, undef, $read_timeout) != 1) || (defined($read_timeout) && $read_timeout <= 0)); my $nbytes = POSIX::read($fd, $buf, $read_length); return ($close_connection, $proto, $status_code, $header, $body) if $no_content_len && $decapitated && (!defined($nbytes) || $nbytes == 0); if (!defined($nbytes)) { next if ($! == EWOULDBLOCK || $! == EAGAIN || $! == EINTR); return ( $close_connection, undef, 0, undef, undef, Hijk::Error::RESPONSE_READ_ERROR, "Failed to read http " . ($decapitated ? "body": "head") . " from socket", $!+0, "$!", ); } if ($nbytes == 0) { return ( $close_connection, undef, 0, undef, undef, Hijk::Error::RESPONSE_BAD_READ_VALUE, "Wasn't expecting a 0 byte response for http " . ($decapitated ? "body": "head" ) . ". This shouldn't happen", ); } if ($decapitated) { $body .= $buf; if (!$no_content_len) { $read_length -= $nbytes; } } else { $head .= $buf; my $neck_pos = index($head, "${CRLF}${CRLF}"); if ($neck_pos > 0) { $decapitated = 1; $body = substr($head, $neck_pos+4); $head = substr($head, 0, $neck_pos); $proto = substr($head, 0, 8); $status_code = substr($head, 9, 3); $method_has_no_content = 1 if $status_code == 204; # 204 NO CONTENT, see http://tools.ietf.org/html/rfc2616#page-60 substr($head, 0, index($head, $CRLF) + 2, ""); # 2 = length($CRLF) my ($doing_chunked, $content_length, $trailer_mode, $trailer_value_is_true); for (split /${CRLF}/o, $head) { my ($key, $value) = split /: /, $_, 2; my $key_lc = lc($key); # Figure this out now so we don't need to scan the # list later under $head_as_array, and just for # simplicity and to avoid duplicating code later # when !$head_as_array. if ($key_lc eq 'transfer-encoding' and $value eq 'chunked') { $doing_chunked = 1; } elsif ( ($key_lc eq 'content-length') || (lc($key) eq 'content-length') ) { $content_length = $value; } elsif ($key_lc eq 'connection' and $value eq 'close') { $close_connection = 1; } elsif ($key_lc eq 'trailer' and $value) { $trailer_value_is_true = 1; } if ($head_as_array) { push @$header => $key, $value; } else { $header->{$key} = $value; } } # We're processing the headers as a stream, and we # only want to turn on $trailer_mode if # Transfer-Encoding=chunked && Trailer=TRUE. However I # don't think there's any guarantee that # Transfer-Encoding comes before Trailer, so we're # effectively doing a second-pass here. if ($doing_chunked and $trailer_value_is_true) { $trailer_mode = 1; } if ($doing_chunked) { die "PANIC: The experimental Hijk support for chunked transfer encoding needs to be explicitly enabled with parse_chunked => 1" unless $parse_chunked; # if there is chunked encoding we have to ignore content length even if we have it return ( $close_connection, $proto, $status_code, $header, _read_chunked_body( $body, $fd, $read_length, $read_timeout, $head_as_array ? $trailer_mode : ($header->{Trailer} ? 1 : 0), ), ); } if (defined $content_length) { if ($content_length == 0) { $read_length = 0; } else { $read_length = $content_length - length($body); } } else { $read_length = 10204; $no_content_len = 1; } } } } while( !$decapitated || (!$method_has_no_content && ($read_length > 0 || $no_content_len)) ); return ($close_connection, $proto, $status_code, $header, $body); } sub _read_chunked_body { my ($buf,$fd,$read_length,$read_timeout,$true_trailer_header) = @_; my $chunk_size = 0; my $body = ""; my $trailer_mode = 0; my $wait_for_last_clrf = 0; vec(my $rin = '', $fd, 1) = 1; while(1) { # just read a 10k block and process it until it is consumed if (length($buf) < 3 || length($buf) < $chunk_size || $wait_for_last_clrf > 0) { return (undef, Hijk::Error::READ_TIMEOUT) if ((_select($rin, undef, undef, $read_timeout) != 1) || (defined($read_timeout) && $read_timeout <= 0)); my $current_buf = ""; my $nbytes = POSIX::read($fd, $current_buf, $read_length); if (!defined($nbytes)) { next if ($! == EWOULDBLOCK || $! == EAGAIN || $! == EINTR); return ( undef, Hijk::Error::RESPONSE_READ_ERROR, "Failed to chunked http body from socket", $!+0, "$!", ); } if ($nbytes == 0) { return ( undef, Hijk::Error::RESPONSE_BAD_READ_VALUE, "Wasn't expecting a 0 byte response for chunked http body. This shouldn't happen, buf:<$buf>, current_buf:<$current_buf>", ); } $buf .= $current_buf; } if ($wait_for_last_clrf > 0) { $wait_for_last_clrf -= length($buf); return $body if ($wait_for_last_clrf <= 0); } if ($trailer_mode) { # http://tools.ietf.org/html/rfc2616#section-14.40 # http://tools.ietf.org/html/rfc2616#section-3.6.1 # A server using chunked transfer-coding in a response MUST NOT use the # trailer for any header fields unless at least one of the following is # true: # a)the request included a TE header field that indicates "trailers" is # acceptable in the transfer-coding of the response, as described in # section 14.39; or, # b)the server is the origin server for the response, the trailer # fields consist entirely of optional metadata, and the recipient # could use the message (in a manner acceptable to the origin server) # without receiving this metadata. In other words, the origin server # is willing to accept the possibility that the trailer fields might # be silently discarded along the path to the client. # in case of trailer mode, we just read everything until the next CRLFCRLF my $neck_pos = index($buf, "${CRLF}${CRLF}"); if ($neck_pos > 0) { return $body; } } else { if ($chunk_size > 0 && length($buf) >= $chunk_size) { $body .= substr($buf, 0, $chunk_size - 2); # our chunk size includes the following CRLF $buf = substr($buf, $chunk_size); $chunk_size = 0; } else { my $neck_pos = index($buf, ${CRLF}); if ($neck_pos > 0) { $chunk_size = hex(substr($buf, 0, $neck_pos)); if ($chunk_size == 0) { if ($true_trailer_header) { $trailer_mode = 1; } else { $buf = substr($buf, $neck_pos + 2); # in case we are missing the ending CLRF, we have to wait for it # otherwise it is left int he socket if (length($buf) < 2) { $wait_for_last_clrf = 2 - length($buf); } else { return $body; } } } else { $chunk_size += 2; # include the following CRLF $buf = substr($buf, $neck_pos + 2); } } elsif($neck_pos == 0) { return ( undef, Hijk::Error::RESPONSE_BAD_READ_VALUE, "Wasn't expecting CLRF without chunk size. This shouldn't happen, buf:<$buf>", ); } } } } } sub _construct_socket { my ($host, $port, $connect_timeout) = @_; # If we can't find the IP address there'll be no point in even # setting up a socket. my $addr; { my $inet_aton = gethostbyname($host); return (undef, {error => Hijk::Error::CANNOT_RESOLVE}) unless defined $inet_aton; $addr = pack_sockaddr_in($port, $inet_aton); } my $tcp_proto = getprotobyname("tcp"); my $soc; socket($soc, PF_INET, SOCK_STREAM, $tcp_proto) || die "Failed to construct TCP socket: $!"; my $flags = fcntl($soc, F_GETFL, 0) or die "Failed to set fcntl F_GETFL flag: $!"; fcntl($soc, F_SETFL, $flags | O_NONBLOCK) or die "Failed to set fcntl O_NONBLOCK flag: $!"; if (!connect($soc, $addr) && $! != EINPROGRESS) { die "Failed to connect $!"; } $connect_timeout = undef if defined($connect_timeout) && $connect_timeout <= 0; vec(my $rout = '', fileno($soc), 1) = 1; if (_select(undef, $rout, undef, $connect_timeout) != 1) { if (defined($connect_timeout)) { return (undef, {error => Hijk::Error::CONNECT_TIMEOUT}); } else { return ( undef, { error => Hijk::Error::REQUEST_SELECT_ERROR, error_message => "select() error on constructing the socket", errno_number => $!+0, errno_string => "$!", }, ); } } if ($! = unpack("L", getsockopt($soc, SOL_SOCKET, SO_ERROR))) { die $!; } return $soc; } sub _build_http_message { my $args = $_[0]; my $path_and_qs = ($args->{path} || "/") . ( defined($args->{query_string}) ? ("?".$args->{query_string}) : "" ); return join( $CRLF, ($args->{method} || "GET")." $path_and_qs " . ($args->{protocol} || "HTTP/1.1"), ($args->{no_default_host_header} ? () : ("Host: $args->{host}")), defined($args->{body}) ? ("Content-Length: " . length($args->{body})) : (), ($args->{head} and @{$args->{head}}) ? ( map { $args->{head}[2*$_] . ": " . $args->{head}[2*$_+1] } 0..$#{$args->{head}}/2 ) : (), "" ) . $CRLF . (defined($args->{body}) ? $args->{body} : ""); } our $SOCKET_CACHE = {}; sub request { my $args = $_[0]; # Backwards compatibility for code that provided the old timeout # argument. $args->{connect_timeout} = $args->{read_timeout} = $args->{timeout} if exists $args->{timeout}; # Ditto for providing a default socket cache, allow for setting it # to "socket_cache => undef" to disable the cache. $args->{socket_cache} = $SOCKET_CACHE unless exists $args->{socket_cache}; # Provide a default for the read_length option $args->{read_length} = 10 * 2 ** 10 unless exists $args->{read_length}; # Use $; so we can use the $socket_cache->{$$, $host, $port} # idiom to access the cache. my $cache_key; $cache_key = join($;, $$, @$args{qw(host port)}) if defined $args->{socket_cache}; my $soc; if (defined $cache_key and exists $args->{socket_cache}->{$cache_key}) { $soc = $args->{socket_cache}->{$cache_key}; } else { ($soc, my $error) = _construct_socket(@$args{qw(host port connect_timeout)}); return $error if $error; $args->{socket_cache}->{$cache_key} = $soc if defined $cache_key; $args->{on_connect}->() if exists $args->{on_connect}; } my $r = _build_http_message($args); my $total = length($r); my $left = $total; vec(my $rout = '', fileno($soc), 1) = 1; while ($left > 0) { if (_select(undef, $rout, undef, undef) != 1) { delete $args->{socket_cache}->{$cache_key} if defined $cache_key; return { error => Hijk::Error::REQUEST_SELECT_ERROR, error_message => "Got error on select() before the write() when while writing the HTTP request the socket", errno_number => $!+0, errno_string => "$!", }; } my $rc = syswrite($soc,$r,$left, $total - $left); if (!defined($rc)) { next if ($! == EWOULDBLOCK || $! == EAGAIN || $! == EINTR); delete $args->{socket_cache}->{$cache_key} if defined $cache_key; shutdown($soc, 2); return { error => Hijk::Error::REQUEST_WRITE_ERROR, error_message => "Got error trying to write the HTTP request with write() to the socket", errno_number => $!+0, errno_string => "$!", }; } $left -= $rc; } my ($close_connection,$proto,$status,$head,$body,$error,$error_message,$errno_number,$errno_string); eval { ($close_connection,$proto,$status,$head,$body,$error,$error_message,$errno_number,$errno_string) = _read_http_message(fileno($soc), @$args{qw(read_length read_timeout parse_chunked head_as_array method)}); 1; } or do { my $err = $@ || "zombie error"; delete $args->{socket_cache}->{$cache_key} if defined $cache_key; shutdown($soc, 2); die $err; }; if ($status == 0 # We always close connections for 1.0 because some servers LIE # and say that they're 1.0 but don't close the connection on # us! An example of this. Test::HTTP::Server (used by the # ShardedKV::Storage::Rest tests) is an example of such a # server. In either case we can't cache a connection for a 1.0 # server anyway, so BEGONE! or $close_connection or (defined $proto and $proto eq 'HTTP/1.0')) { delete $args->{socket_cache}->{$cache_key} if defined $cache_key; shutdown($soc, 2); } return { proto => $proto, status => $status, head => $head, body => $body, defined($error) ? ( error => $error ) : (), defined($error_message) ? ( error_message => $error_message ) : (), defined($errno_number) ? ( errno_number => $errno_number ) : (), defined($errno_string) ? ( errno_string => $errno_string ) : (), }; } sub _select { my ($rbits, $wbits, $ebits, $timeout) = @_; while (1) { my $start = Time::HiRes::time(); my $nfound = select($rbits, $wbits, $ebits, $timeout); if ($nfound == -1 && $! == EINTR) { $timeout -= Time::HiRes::time() - $start if $timeout; next; } return $nfound; } } 1; __END__ =encoding utf8 =head1 NAME Hijk - Fast & minimal low-level HTTP client =head1 SYNOPSIS A simple GET request: use Hijk (); my $res = Hijk::request({ method => "GET", host => "example.com", port => "80", path => "/flower", query_string => "color=red" }); if (exists $res->{error} and $res->{error} & Hijk::Error::TIMEOUT) { die "Oh noes we had some sort of timeout"; } die "Expecting an 'OK' response" unless $res->{status} == 200; say $res->{body}; A POST request, you have to manually set the appropriate headers, URI escape your values etc. use Hijk (); use URI::Escape qw(uri_escape); my $res = Hijk::request({ method => "POST", host => "example.com", port => "80", path => "/new", head => [ "Content-Type" => "application/x-www-form-urlencoded" ], query_string => "type=flower&bucket=the%20one%20out%20back", body => "description=" . uri_escape("Another flower, let's hope it's exciting"), }); die "Expecting an 'OK' response" unless $res->{status} == 200; =head1 DESCRIPTION Hijk is a fast & minimal low-level HTTP client intended to be used where you control both the client and the server, e.g. for talking to some internal service from a frontend user-facing web application. It is C a general HTTP user agent, it doesn't support redirects, proxies, SSL and any number of other advanced HTTP features like (in roughly descending order of feature completeness) L, L, L, L or L. This library is basically one step above manually talking HTTP over sockets. Having said that it's lightning fast and extensively used in production at L where it's used as the go-to transport layer for talking to internal services. It uses non-blocking sockets and correctly handles all combinations of connect/read timeouts and other issues you might encounter from various combinations of parts of your system going down or becoming otherwise unavailable. =head1 FUNCTION: Hijk::request( $args :HashRef ) :HashRef C is the only function you should use. It (or anything else in this package for that matter) is not exported, so you have to use the fully qualified name. It takes a C of arguments and either dies or returns a C as a response. The C argument to it must contain some of the key-value pairs from the following list. The value for C and C are mandatory, but others are optional with default values listed below. protocol => "HTTP/1.1", # (or "HTTP/1.0") host => ..., port => ..., connect_timeout => undef, read_timeout => undef, read_length => 10240, method => "GET", path => "/", query_string => "", head => [], body => "", socket_cache => \%Hijk::SOCKET_CACHE, # (undef to disable, or \my %your_socket_cache) on_connect => undef, # (or sub { ... }) parse_chunked => 0, head_as_array => 0, no_default_host_header => 1, Notice how Hijk does not take a full URI string as input, you have to specify the individual parts of the URL. Users who need to parse an existing URI string to produce a request should use the L module to do so. The value of C is an C of key-value pairs instead of a C, this way you can decide in which order the headers are sent, and you can send the same header name multiple times. For example: head => [ "Content-Type" => "application/json", "X-Requested-With" => "Hijk", ] Will produce these request headers: Content-Type: application/json X-Requested-With: Hijk In addition Hijk will provide a C header for you by default with the C value you pass to C. To suppress this (e.g. to send custom C requests) pass a true value to the C option and provide your own C header in the C C (or don't, if you want to construct a C-less request knock yourself out...). Hijk doesn't escape any values for you, it just passes them through as-is. You can easily produce invalid requests if e.g. any of these strings contain a newline, or aren't otherwise properly escaped. The value of C or C is in floating point seconds, and is used as the time limit for connecting to the host, and reading the response back from it, respectively. The default value for both is C, meaning no timeout limit. If you don't supply these timeouts and the host really is unreachable or slow, we'll reach the TCP timeout limit before returning some other error to you. The default C is C, but you can also specify C. The advantage of using C is support for keep-alive, which matters a lot in environments where the connection setup represents non-trivial overhead. Sometimes that overhead is negligible (e.g. on Linux talking to an nginx on the local network), and keeping open connections down and reducing complexity is more important, in those cases you can either use C, or specify C in the request, but just using C is an easy way to accomplish the same thing. By default we will provide a C for you which is a global singleton that we maintain keyed on C. Alternatively you can pass in C hash of your own which we'll use as the cache. To completely disable the cache pass in C. The optional C callback is intended to be used for you to figure out from production traffic what you should set the C. I.e. you can start a timer when you call C that you end when C is called, that's how long it took us to get a connection. If you start another timer in that callback that you end when C returns to you that'll give you how long it took to send/receive data after we constructed the socket, i.e. it'll help you to tweak your C. The C callback is provided with no arguments, and is called in void context. We have experimental support for parsing chunked responses encoding. historically Hijk didn't support this at all and if you wanted to use it with e.g. nginx you had to add C to the nginx config file. Since you may just want to do that instead of having Hijk do more work to parse this out with a more complex and experimental codepath you have to explicitly enable it with C. Otherwise Hijk will die when it encounters chunked responses. The C option may be turned on by default in the future. The return value is a C representing a response. It contains the following key-value pairs. proto => :Str status => :StatusCode body => :Str head => :HashRef (or :ArrayRef with "head_as_array") error => :PositiveInt error_message => :Str errno_number => :Int errno_string => :Str For example, to send a request to C, pass the following parameters: my $res = Hijk::request({ host => "example.com", port => "80", path => "/flower", query_string => "color=red" }); die "Response is not 'OK'" unless $res->{status} == 200; Notice that you do not need to put the leading C<"?"> character in the C. You do, however, need to properly C the content of C. Again, Hijk doesn't escape any values for you, so these values B be properly escaped before being passed in, unless you want to issue invalid requests. By default the C in the response is a C rather then an C. This makes it easier to retrieve specific header fields, but it means that we'll clobber any duplicated header names with the most recently seen header value. To get the returned headers as an C instead specify C. If you want to fiddle with the C value it controls how much we C at a time. We currently don't support servers returning a http body without an accompanying C header; bodies B have a C or we won't pick them up. =head1 ERROR CODES If we had a recoverable error we'll include an "error" key whose value is a bitfield that you can check against Hijk::Error::* constants. Those are: Hijk::Error::CONNECT_TIMEOUT Hijk::Error::READ_TIMEOUT Hijk::Error::TIMEOUT Hijk::Error::CANNOT_RESOLVE Hijk::Error::REQUEST_SELECT_ERROR Hijk::Error::REQUEST_WRITE_ERROR Hijk::Error::REQUEST_ERROR Hijk::Error::RESPONSE_READ_ERROR Hijk::Error::RESPONSE_BAD_READ_VALUE Hijk::Error::RESPONSE_ERROR In addition we might return C, C and C keys, see the discussion of C and C errors below. The C constant is the same as C. It's there for convenience so you can do: .. if exists $res->{error} and $res->{error} & Hijk::Error::TIMEOUT; Instead of the more verbose: .. if exists $res->{error} and $res->{error} & (Hijk::Error::CONNECT_TIMEOUT | Hijk::Error::READ_TIMEOUT) We'll return C if we can't C the host you've provided. If we fail to do a C or C during when sending the response we'll return C or C, respectively. Similarly to C the C constant is a union of these two, and any other request errors we might add in the future. When we're getting the response back we'll return C when we can't C the response, and C when the value we got from C is C<0>. The C constant is a union of these two and any other response errors we might add in the future. Some of these C and C errors are re-thrown errors from system calls. In that case we'll also pass along C which is a short human readable error message about the error, as well as C & C, which are C<$!+0> and C<"$!"> at the time we had the error. Hijk might encounter other errors during the course of the request and B call C if that happens, so if you don't want your program to stop when a request like that fails wrap it in C. Having said that the point of the C interface is that all errors that happen during normal operation, i.e. making valid requests against servers where you can have issues like timeouts, network blips or the server thread on the other end being suddenly kill -9'd should be caught, categorized and returned in a structural way by Hijk. We're not currently aware of any issues that occur in such normal operations that aren't classified as a C, and if we find new issues that fit the criteria above we'll likely just make a new C for it. We're just not trying to guarantee that the library can never C, and aren't trying to catch truly exceptional issues like e.g. C failing on a valid socket. =head1 AUTHORS Kang-min Liu Ævar Arnfjörð Bjarmason Borislav Nikolov Damian Gryski =head1 COPYRIGHT Copyright (c) 2013- Kang-min Liu C<< >>. =head1 LICENCE The MIT License =head1 DISCLAIMER OF WARRANTY BECAUSE THIS SOFTWARE IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE SOFTWARE, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE SOFTWARE "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE SOFTWARE IS WITH YOU. SHOULD THE SOFTWARE PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR, OR CORRECTION. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE SOFTWARE AS PERMITTED BY THE ABOVE LICENCE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE SOFTWARE (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE SOFTWARE TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. =cut chunked-trailer.t000644000766000024 312613416043414 15441 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use Test::More; use File::Temp (); use File::Temp qw/ :seekable /; use Hijk; my $fh = File::Temp->new(); my $fd = do { local $/ = undef; my $data = "4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n"; my $msg = join( "\x0d\x0a", 'HTTP/1.1 200 OK', 'Last-Modified: Sat, 26 Oct 2013 19:41:47 GMT', 'ETag: "4b9d0211dd8a2819866bccff777af225"', 'Content-Type: text/html', 'Server: Example', 'Transfer-Encoding: chunked', 'Trailer: Date', 'non-sence: ' . 'a' x 20000, '', $data, 'Date: Sat, 23 Nov 2013 23:10:28 GMT', '' ); print $fh $msg; $fh->flush; $fh->seek(0, 0); fileno($fh); }; my (undef, $proto, $status, $head, $body) = Hijk::_read_http_message($fd, 10240, undef, 1); is $status, 200; is $body, "Wikipedia in\r\n\r\nchunks."; is_deeply $head, { "Last-Modified" => "Sat, 26 Oct 2013 19:41:47 GMT", "ETag" => '"4b9d0211dd8a2819866bccff777af225"', "Content-Type" => "text/html", "Server" => "Example", 'non-sence' => 'a' x 20000, "Transfer-Encoding" => "chunked", 'Trailer' => 'Date', }; # fetch again without seeking back # this will force select() to return because there are actually # 0 bytes to read - so we can simulate connection closed # from the other end of the socket (like expired keep-alive) (undef, $proto, $status, $head, $body, my $error, my $error_message) = Hijk::_read_http_message($fd, 10240); is $error, Hijk::Error::RESPONSE_BAD_READ_VALUE; like $error_message, qr/0 byte/; done_testing; LICENSE000644000766000024 242713416043414 12740 0ustar00st21277staff000000000000Hijk-0.28This software is Copyright (c) 2019 by Kang-min Liu , Ævar Arnfjörð Bjarmason , Borislav Nikolov , Damian Gryski . This is free software, licensed under: The MIT (X11) License The MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. parse-http-message.t000644000766000024 202013416043414 16061 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Test::More; use File::Temp (); use File::Temp qw/ :seekable /; use Hijk; my $fh = File::Temp->new(); my $fd = do { local $/ = undef; my $msg = join( "\x0d\x0a", 'HTTP/1.1 200 OK', 'Date: Sat, 23 Nov 2013 23:10:28 GMT', 'Last-Modified: Sat, 26 Oct 2013 19:41:47 GMT', 'ETag: "4b9d0211dd8a2819866bccff777af225"', 'Content-Type: text/html', 'Server: Example', 'Content-Length: 4', '', 'OHAI' ); print $fh $msg; $fh->flush; $fh->seek(0, 0); fileno($fh); }; my (undef, $proto, $status, $head, $body) = Hijk::_read_http_message($fd, 10240); is $proto, "HTTP/1.1"; is $status, 200; is $body, "OHAI"; is_deeply $head, { "Date" => "Sat, 23 Nov 2013 23:10:28 GMT", "Last-Modified" => "Sat, 26 Oct 2013 19:41:47 GMT", "ETag" => '"4b9d0211dd8a2819866bccff777af225"', "Content-Type" => "text/html", "Content-Length" => "4", "Server" => "Example", }; done_testing; META.yml000644000766000024 172113416043414 13200 0ustar00st21277staff000000000000Hijk-0.28--- abstract: 'Fast & minimal low-level HTTP client' author: - 'Kang-min Liu ' - 'Ævar Arnfjörð Bjarmason ' - 'Borislav Nikolov ' - 'Damian Gryski ' build_requires: HTTP::Server::Simple::PSGI: '0' Net::Ping: '2.41' Net::Server::HTTP: '0' Plack: '0' Test::Exception: '0' Test::More: '0' configure_requires: Module::Build::Tiny: '0' dynamic_config: 0 generated_by: 'App::ModuleBuildTiny version 0.025, CPAN::Meta::Converter version 2.150010' license: mit meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.4.html version: '1.4' name: Hijk provides: Hijk: file: lib/Hijk.pm version: '0.28' requires: Time::HiRes: '0' resources: bugtracker: https://github.com/gugod/Hijk/issues repository: https://github.com/gugod/Hijk.git version: '0.28' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' x_spdx_expression: MIT x_static_install: '1' build_http_message.t000644000766000024 1034413416043414 16242 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Test::More; use Hijk; my $CRLF = "\x0d\x0a"; for my $protocol ("HTTP/1.0", "HTTP/1.1") { is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com" }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "example.com" }), "GET / $protocol${CRLF}". "Host: example.com${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ method => "HEAD", protocol => $protocol, host => "example.com" }), "HEAD / $protocol${CRLF}". "Host: example.com${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", port => "8080" }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", query_string => "a=b" }), "GET /?a=b $protocol${CRLF}". "Host: www.example.com${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", path => "/flower" }), "GET /flower $protocol${CRLF}". "Host: www.example.com${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", path => "/flower", query_string => "a=b" }), "GET /flower?a=b $protocol${CRLF}". "Host: www.example.com${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", body => "morning" }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}". "Content-Length: 7${CRLF}". "${CRLF}". "morning"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", body => "0" }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}". "Content-Length: 1${CRLF}". "${CRLF}". "0"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", body => undef }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", body => "" }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}". "Content-Length: 0${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", head => ["X-Head" => "extra stuff"] }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}". "X-Head: extra stuff${CRLF}". "${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", head => ["X-Head" => "extra stuff", "X-Hat" => "ditto"] }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}". "X-Head: extra stuff${CRLF}". "X-Hat: ditto${CRLF}${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "www.example.com", head => ["X-Head" => "extra stuff"], body => "OHAI" }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}". "Content-Length: 4${CRLF}". "X-Head: extra stuff${CRLF}". "${CRLF}". "OHAI"; # Allow overriding Host header in head arrayref is Hijk::_build_http_message({ protocol => $protocol, host => "localhost", head => [ "Host" => "www.example.com" ], no_default_host_header => 1 }), "GET / $protocol${CRLF}". "Host: www.example.com${CRLF}${CRLF}"; # Also allow sending no Host header at all is Hijk::_build_http_message({ protocol => $protocol, host => "localhost", no_default_host_header => 1 }), "GET / $protocol${CRLF}${CRLF}"; is Hijk::_build_http_message({ protocol => $protocol, host => "localhost", head => [], no_default_host_header => 1 }), "GET / $protocol${CRLF}${CRLF}"; # Or even crazy multiple Host headers, whatever that means! is Hijk::_build_http_message({ protocol => $protocol, host => "localhost", head => [ Host => "foo", Host => "bar" ], no_default_host_header => 1 }), "GET / $protocol${CRLF}". "Host: foo${CRLF}". "Host: bar${CRLF}${CRLF}"; } done_testing; split-in-chunks.psgi000644000766000024 166713416043414 16667 0ustar00st21277staff000000000000Hijk-0.28/t/bin#!/usr/bin/env perl # need Starman to produce chunked response. # starman --worker 4 t/bin/split-in-chunks.psgi ## perl -E 'print "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"' | nc localhost 5000 use strict; use warnings; my $epic_graph = <([ 200, [ 'Content-Type', 'text/plain' ]]); while($epic_graph) { my $l = rand() * 30 + 1; my $chunk = substr($epic_graph, 0, $l, ''); $writer->write($chunk); } $writer->close; } } live-google.t000644000766000024 746713416043414 14605 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Hijk; use Time::HiRes (); use Test::More; use Test::Exception; unless ($ENV{TEST_LIVE}) { plan skip_all => "Enable live testing by setting env: TEST_LIVE=1"; } if($ENV{http_proxy}) { plan skip_all => "http_proxy is set. We cannot test when proxy is required to visit google.com"; } my %args = ( host => "google.com", port => "80", method => "GET", ); subtest "timeout and cache" => sub { lives_ok { my $res = Hijk::request({ host => 'google.com', port => 80, timeout => 0 }); ok !exists($res->{error}), '$res->{error} does not exist, because we do not expect connect timeout to happen'; cmp_ok scalar(keys %{$Hijk::SOCKET_CACHE}), '==', 1, "We have an entry in the global socket cache"; %{$Hijk::SOCKET_CACHE} = (); } "We could make the request"; lives_ok { my %socket_cache; my $res = Hijk::request({ host => 'google.com', port => 80, timeout => 0, socket_cache => \%socket_cache, }); ok !exists($res->{error}), '$res->{error} does not exist, because we do not expect connect timeout to happen'; cmp_ok scalar(keys %{$Hijk::SOCKET_CACHE}), '==', 0, "We have nothing in the global socket cache..."; cmp_ok scalar(keys %socket_cache), '==', 1, "...because we used our own cache"; } "We could make the request"; lives_ok { my %socket_cache; my $res = Hijk::request({ host => 'google.com', port => 80, timeout => 0, socket_cache => undef, }); ok !exists($res->{error}), '$res->{error} does not exist, because we do not expect connect timeout to happen'; cmp_ok scalar(keys %{$Hijk::SOCKET_CACHE}), '==', 0, "We have nothing in the global socket cache"; cmp_ok $res->{body}, "ne", "", "We a body with a GET requests"; } "We could make the request"; lives_ok { my %socket_cache; my $res = Hijk::request({ method => "HEAD", host => 'google.com', port => 80, timeout => 0, socket_cache => undef, }); ok !exists($res->{error}), '$res->{error} does not exist, because we do not expect connect timeout to happen'; cmp_ok scalar(keys %{$Hijk::SOCKET_CACHE}), '==', 0, "We have nothing in the global socket cache"; cmp_ok $res->{body}, "eq", "", "We have no body from HEAD requests"; } "We could make the request"; }; subtest "with 1ms timeout limit, expect an exception." => sub { lives_ok { my $res = Hijk::request({%args, timeout => 0.001}); ok exists $res->{error}; ok $res->{error} & Hijk::Error::TIMEOUT; }; }; subtest "with 10s timeout limit, do not expect an exception." => sub { lives_ok { my $res = Hijk::request({%args, timeout => 10}); diag substr($res->{body}, 0, 80); } 'google.com send back something within 10s'; }; subtest "without timeout, do not expect an exception." => sub { lives_ok { my $res = Hijk::request({%args, timeout => 0}); } 'google.com send back something without timeout'; }; subtest "Test the on_connect callback" => sub { lives_ok { my $connect_time = -Time::HiRes::time(); my $read_time; my $res = Hijk::request({ %args, timeout => 10, socket_cache => undef, on_connect => sub { $connect_time += Time::HiRes::time(); $read_time = -Time::HiRes::time(); return; }, }); $read_time += Time::HiRes::time(); ok($connect_time, "Managed to connect in $connect_time"); ok($read_time, "Managed to read in $read_time"); }; }; done_testing; dumbbench-thisurl.pl000644000766000024 175113416043414 17524 0ustar00st21277staff000000000000Hijk-0.28/examples#!/usr/bin/env perl use strict; use warnings; use Dumbbench; use Hijk; use HTTP::Tiny; use LWP::UserAgent; use HTTP::Request; my $url = shift; my $uri = URI->new($url); my $tiny = HTTP::Tiny->new(); my $req = HTTP::Request->new('GET',$url); my $lwp = LWP::UserAgent->new(); my $hijk_req_arg = { path => $uri->path, host => $uri->host, port => $uri->port || 80, method => 'GET' }; my $bench = Dumbbench->new( target_rel_precision => 0.005, initial_runs => 1_000, ); $bench->add_instances( Dumbbench::Instance::PerlSub->new( name => "hijk", code => sub { my $res = Hijk::request($hijk_req_arg); } ), Dumbbench::Instance::PerlSub->new( name => "httptiny", code => sub { my $res = $tiny->get($url); } ), Dumbbench::Instance::PerlSub->new( name => "lwpua", code => sub { my $res = $lwp->request($req); } ), ); $bench->run; $bench->report; hijkurl000644000766000024 171013416043414 15136 0ustar00st21277staff000000000000Hijk-0.28/examples#!/usr/bin/env perl use strict; use warnings; use Getopt::Long; use Hijk; use URI; my ($method, $output_file, $header, $body) = ("GET", "-", ""); my $timeout = 60; my $dump_header; GetOptions( "method|X=s" => \$method, "H=s", => \$header, 'd=s', => \$body, "output|o=s" => \$output_file, "timeout=s" => \$timeout, "D|dump-header=s" => \$dump_header, ); $method = uc($method); my $uri_string = shift(@ARGV) or die "$0 "; my $uri = URI->new($uri_string); my $res = Hijk::request { method => $method, host => $uri->host, port => $uri->port || 80, timeout => $timeout*1000, path => $uri->path, query_string => $uri->query, $header ? ( head => [split /: /, $header, 2] ) : (), $body ? ( body => $body ) : (), parse_chunked => 1, }; if ($dump_header) { for (keys %{$res->{head}}) { print "$_: $res->{head}{$_}\n"; } print "\n"; } print $res->{body}; chunked-trailer-head-as-array.t000644000766000024 313113416043414 20051 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use Test::More; use File::Temp (); use File::Temp qw/ :seekable /; use Hijk; my $fh = File::Temp->new(); my $fd = do { local $/ = undef; my $data = "4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n"; my $msg = join( "\x0d\x0a", 'HTTP/1.1 200 OK', 'Last-Modified: Sat, 26 Oct 2013 19:41:47 GMT', 'ETag: "4b9d0211dd8a2819866bccff777af225"', 'Content-Type: text/html', 'Server: Example', 'Transfer-Encoding: chunked', 'Trailer: Date', 'non-sence: ' . 'a' x 20000, '', $data, 'Date: Sat, 23 Nov 2013 23:10:28 GMT', '' ); print $fh $msg; $fh->flush; $fh->seek(0, 0); fileno($fh); }; my (undef, $proto, $status, $head, $body) = Hijk::_read_http_message($fd, 10240, undef, 1, 1); is $status, 200; is $body, "Wikipedia in\r\n\r\nchunks."; is_deeply $head, [ "Last-Modified" => "Sat, 26 Oct 2013 19:41:47 GMT", "ETag" => '"4b9d0211dd8a2819866bccff777af225"', "Content-Type" => "text/html", "Server" => "Example", "Transfer-Encoding" => "chunked", 'Trailer' => 'Date', 'non-sence' => 'a' x 20000, ]; # fetch again without seeking back # this will force select() to return because there are actually # 0 bytes to read - so we can simulate connection closed # from the other end of the socket (like expired keep-alive) (undef, $proto, $status, $head, $body, my $error, my $error_message) = Hijk::_read_http_message($fd, 10240); is $error, Hijk::Error::RESPONSE_BAD_READ_VALUE; like $error_message, qr/0 byte/; done_testing; parse-http-message-head-as-array.t000644000766000024 203113416043414 20477 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Test::More; use File::Temp (); use File::Temp qw/ :seekable /; use Hijk; my $fh = File::Temp->new(); my $fd = do { local $/ = undef; my $msg = join( "\x0d\x0a", 'HTTP/1.1 200 OK', 'Date: Sat, 23 Nov 2013 23:10:28 GMT', 'Last-Modified: Sat, 26 Oct 2013 19:41:47 GMT', 'ETag: "4b9d0211dd8a2819866bccff777af225"', 'Content-Type: text/html', 'Server: Example', 'Content-Length: 4', '', 'OHAI' ); print $fh $msg; $fh->flush; $fh->seek(0, 0); fileno($fh); }; my (undef, $proto, $status, $head, $body) = Hijk::_read_http_message($fd, 10240, 0, 0, 1); is $proto, "HTTP/1.1"; is $status, 200; is $body, "OHAI"; is_deeply $head, [ "Date" => "Sat, 23 Nov 2013 23:10:28 GMT", "Last-Modified" => "Sat, 26 Oct 2013 19:41:47 GMT", "ETag" => '"4b9d0211dd8a2819866bccff777af225"', "Content-Type" => "text/html", "Server" => "Example", "Content-Length" => "4", ]; done_testing; live-invalid-domain.t000644000766000024 67713416043414 16200 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Hijk; use Test::More; unless ($ENV{TEST_LIVE}) { plan skip_all => "Enable live testing by setting env: TEST_LIVE=1"; } my $res = Hijk::request({ method => "GET", host => "hlagh.google.com", port => "80", }); ok exists $res->{error}, "We got an error back for this invalid domain"; is $res->{error}, Hijk::Error::CANNOT_RESOLVE, "We can't resolve the domain"; done_testing; it-takes-time.psgi000644000766000024 62713416043414 16267 0ustar00st21277staff000000000000Hijk-0.28/t/bin#!/usr/bin/env perl use strict; use warnings; use Time::HiRes qw(sleep time); use Plack::Request; sub { my $env = shift; my $start_time = time; my $req = Plack::Request->new($env); my ($t) = $env->{QUERY_STRING} =~ m/\At=([0-9\.]+)\z/; $t ||= 1; sleep $t; return [200, [], [$start_time, ",", time]]; } __END__ curl 'http://localhost:5000?t=2.5' curl 'http://localhost:5000?t=17' parse-http-connection-close-message.t000644000766000024 227013416043414 21330 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Test::More; use File::Temp (); use File::Temp qw/ :seekable /; use Hijk; my $fh = File::Temp->new(); my $fd = do { local $/ = undef; my $msg = join( "\x0d\x0a", 'HTTP/1.1 200 OK', 'Date: Sat, 23 Nov 2013 23:10:28 GMT', 'Last-Modified: Sat, 26 Oct 2013 19:41:47 GMT', 'ETag: "4b9d0211dd8a2819866bccff777af225"', 'Content-Type: text/html', 'Server: Example', 'Connection: close', '', '' ); print $fh $msg; $fh->flush; $fh->seek(0, 0); fileno($fh); }; my (undef, $proto, $status, $head, $body) = Hijk::_read_http_message($fd, 10240, 0); is $status, 200; is $body, ""; is_deeply $head, { "Date" => "Sat, 23 Nov 2013 23:10:28 GMT", "Last-Modified" => "Sat, 26 Oct 2013 19:41:47 GMT", "ETag" => '"4b9d0211dd8a2819866bccff777af225"', "Content-Type" => "text/html", "Server" => "Example", "Connection" => "close", }; (undef, $proto, $status, $head, $body, my $error, my $error_message) = Hijk::_read_http_message($fd, 10240, 0); is $error, Hijk::Error::RESPONSE_BAD_READ_VALUE; like $error_message, qr/0 byte/; done_testing; .travis.yml000644000766000024 252013416043414 14036 0ustar00st21277staff000000000000Hijk-0.28os: - linux - osx language: perl perl: - "5.28" - "5.26" - "5.24" - "5.22" - "5.20" - "5.20-shrplib" - "5.18" - "5.18-shrplib" - "5.16" - "5.14" - "5.12" - "5.10" - "5.8" env: - TEST_ELASTICSEARCH=1 TEST_LIVE=1 TEST_HOST=localhost TEST_PORT=9200 TEST_COUCHDB=http://localhost:5984/ services: - elasticsearch - couchdb before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -L https://install.perlbrew.pl | bash ; source $HOME/perl5/perlbrew/etc/bashrc; perlbrew install -j4 --notest --as perl-$TRAVIS_PERL_VERSION $(perlbrew available | grep $TRAVIS_PERL_VERSION | cut -c4- | head -1); perlbrew clean; perlbrew install-cpanm; perlbrew use "$TRAVIS_PERL_VERSION"; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then git clone git://github.com/travis-perl/helpers ~/travis-perl-helpers; source ~/travis-perl-helpers/init; build-perl; fi - perl -V - cpanm --notest Devel::Cover Devel::Cover::Report::Coveralls Test::More~0.98 script: if [[ "$TRAVIS_PERL_VERSION" != "5.8" ]]; then PERL5OPT=-MDevel::Cover prove -l -v -r t && cover -report coveralls; else prove -lrv t; fi notifications: irc: channels: - "irc.freenode.org#chupei.pm" use_notice: true git: depth: 1 matrix: allow_failures: - perl: 5.8 - os: osx perl: 5.20-shrplib - os: osx perl: 5.18-shrplib live-unixis.t000644000766000024 272313416043414 14636 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Hijk; use Test::More; use Test::Exception; unless ($ENV{TEST_UNIXIS}) { plan skip_all => "Enable live testing by setting env: TEST_UNIXIS=1"; } unless ($ENV{TEST_LIVE}) { plan skip_all => "Enable live testing by setting env: TEST_LIVE=1"; } if($ENV{http_proxy}) { plan skip_all => "http_proxy is set. We cannot test when proxy is required to visit u.nix.is"; } for my $i (1..1000) { lives_ok { my $res = Hijk::request({ host => 'u.nix.is', port => 80, connect_timeout => 3, read_timeout => 3, path => "/?Hijk_test_nr=$i", head => [ "X-Request-Nr" => $i, "Referer" => "Hijk (file:" . __FILE__ . "; iteration: $i)", ], }); ok !exists($res->{error}), '$res->{error} does not exist, because we do not expect connect timeout to happen'; cmp_ok $res->{status}, '==', 200, "We got a 200 OK response"; if (exists $res->{head}->{Connection} and $res->{head}->{Connection} eq 'close') { cmp_ok scalar(keys %{$Hijk::SOCKET_CACHE}), '==', 0, "We were told to close the connection. We should have no entry in the socket cache"; } else { cmp_ok scalar(keys %{$Hijk::SOCKET_CACHE}), '==', 1, "We have an entry in the global socket cache"; } } "We could make request number $i"; } done_testing; head-request.psgi000644000766000024 101513416043414 16211 0ustar00st21277staff000000000000Hijk-0.28/t/bin#!/usr/bin/env perl use strict; use warnings; sub { my $env = shift; my ($gimme_content_length) = $env->{QUERY_STRING} =~ m/\Agimme_content_length=([01])\z/; my $hello_world = "Hello world"; my $content_length = length($hello_world); if ($env->{REQUEST_METHOD} eq 'HEAD') { $hello_world = ''; } return [ 200, [ ($gimme_content_length ? ( 'Content-Length' => length($hello_world) ) : ()), ], [$hello_world], ]; } MANIFEST000644000766000024 136613416043414 13065 0ustar00st21277staff000000000000Hijk-0.28.travis.yml Build.PL Changes LICENSE MANIFEST META.json META.yml README README.md cpanfile examples/bench-chunked-response.pl examples/bench-elasticsearch.pl examples/bench-nginx.pl examples/dumbbench-thisurl.pl examples/hijkurl lib/Hijk.pm metamerge.json t/bin/head-request.psgi t/bin/it-takes-time.psgi t/bin/nsh-head-request.pl t/bin/split-in-chunks.psgi t/build_http_message.t t/chunked-trailer-head-as-array.t t/chunked-trailer.t t/chunked.t t/live-connect-timeout.t t/live-couchdb.t t/live-elasticsearch.t t/live-google.t t/live-invalid-domain.t t/live-unixis.t t/parse-http-connection-close-message.t t/parse-http-message-head-as-array.t t/parse-http-message.t t/parse-http-no-content-len-message.t t/plack-head-request.t t/plack.t t/select-timeout.t nsh-head-request.pl000644000766000024 131713416043414 16455 0ustar00st21277staff000000000000Hijk-0.28/t/bin#!/usr/bin/env perl use strict; use warnings; BEGIN { if ($] < 5.010010) { require base; base->import('Net::Server::HTTP'); } else { require parent; parent->import('Net::Server::HTTP'); } } my $port = $ARGV[0] || '3000'; __PACKAGE__->run( port => $port ); sub process_http_request { my $self = shift; my ($gimme) = ($self->{request_info}{'query_string'} ||'') =~ m'gimme_content_length=(1?)$'; if ($gimme) { print "Content-type: text/plain\n"; print "Content-Length: 11\n\n"; } else { print "Content-type: text/plain\n\n"; } if ($self->{request_info}{request_method} ne 'HEAD') { print "Hello World\n"; } } select-timeout.t000644000766000024 202413416043414 15317 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Test::More; use Hijk; use Time::HiRes; my $parent_pid = $$; pipe(my $rh, my $wh) or die "Failed to create pipe: $!"; my $pid = fork; die "Fail to fork then start a plack server" unless defined $pid; if ($pid == 0) { Time::HiRes::sleep(0.5); for (1..10) { kill('HUP', $parent_pid); Time::HiRes::sleep(0.1); } exit; } $SIG{HUP} = sub { warn "SIGHUP received\n" }; my $timeout = 2; vec(my $rin = '', fileno($rh), 2) = 1; my $start = Time::HiRes::time; Hijk::_select($rin, undef, undef, $timeout); my $elapsed = Time::HiRes::time - $start; { my $msg = sprintf("handle signal during select, took=%.2fs, expected at least=%.2fs", $elapsed, $timeout); if ($elapsed >= $timeout) { pass($msg); } else { TODO: { local $TODO = "We don't know why, but this fails on various BSDs etc. It is known, and probably some general OS issue. Don't clutter CPANtesters with it"; fail($msg); } } } done_testing; plack-head-request.t000644000766000024 337313416043414 16043 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use FindBin; use Hijk; use Test::More; my $port = 10000 + int rand(5000); my $pid = fork; die "Fail to fork then start a plack server" unless defined $pid; if ($pid == 0) { exec($^X, "t/bin/nsh-head-request.pl", $port); } sleep 2; # hopfully this is enough to launch that daemon. my %args = ( timeout => 1, host => "localhost", port => $port, method => "HEAD", ); subtest "expect HEAD response with a Content-Length" => sub { my $res = Hijk::request({%args, query_string => "gimme_content_length=1"}); ok !exists $res->{error}, '$res->{error} should not exist because this request should have been successful'; ok defined($res->{head}->{"Content-length"}), "Got a Content-Length"; cmp_ok $res->{body}, "eq", "", "Got no body even though we had a Content-Length header"; if ($res->{head}->{"Content-length"} == 0) { pass 'Content-Length: 0, looks OK because this response has no http body'; } elsif ($res->{head}->{"Content-length"} == 11) { pass 'Content-Length: 11, looks OK because it is the length of body should this be a GET request'; } else { fail "Content-Length: " . $res->{head}->{'Content-length'} . ' does not look like a legit value.'; } }; subtest "expect HEAD response without a Content-Length" => sub { my $res = Hijk::request({%args, query_string => "gimme_content_length="}); ok !exists $res->{error}, '$res->{error} should not exist because this request should have been successful'; ok !exists $res->{head}->{"Content-length"}, "We should get no Content-Length"; cmp_ok $res->{body}, "eq", "", "Got no body wit the HEAD response, also have no Content-Length"; }; END { kill INT => $pid if $pid } done_testing; live-connect-timeout.t000644000766000024 263113416043414 16432 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use warnings; use Test::More; use Net::Ping; use Hijk; unless ($ENV{TEST_LIVE}) { plan skip_all => "Enable live testing by setting env: TEST_LIVE=1"; } # find a ip and confirm it is not reachable. my $pinger = Net::Ping->new("tcp", 2); $pinger->port_number(80); my $ip; my $iter = 10; do { $ip = join ".", 172, (int(rand()*15+16)), int(rand()*250+1), int(rand()*255+1); } while($iter-- > 0 && $pinger->ping($ip)); if ($iter == 0) { plan skip_all => "Cannot randomly generate an unreachable IP." } pass "ip generated = $ip"; my ($res, $exception); eval { $res = Hijk::request({ host => $ip, port => 80, timeout => 1 # seconds }); 1; } or do { $exception = $@ || "unknown error."; $exception =~ s/\n//g; }; if ($exception) { pass "On $^O, we have exception trying to connect to an unreachable IP: $exception"; is(scalar(keys %{$Hijk::SOCKET_CACHE}), 0, "We have nothing in the socket cache after the connect exception."); } else { ok exists $res->{error}, "On $^O, ".'$res->{error} exists because we expect error to happen.'; is $res->{error}, Hijk::Error::CONNECT_TIMEOUT, '$res->{error} contiain the value of Hijk::Error::CONNECT_TIMEOUT, indicating that it timed-out when establishing connection'; is(scalar(keys %{$Hijk::SOCKET_CACHE}), 0, "We have nothing in the socket cache after a timeout"); } done_testing; chunked.t000644000766000024 311613416043414 14000 0ustar00st21277staff000000000000Hijk-0.28/t#!/usr/bin/env perl use strict; use Test::More; use File::Temp (); use File::Temp qw/ :seekable /; use Hijk; my $fh = File::Temp->new(); my $fd = do { local $/ = undef; my $data = "4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n"; my $msg = join( "\x0d\x0a", 'HTTP/1.1 200 OK', 'Date: Sat, 23 Nov 2013 23:10:28 GMT', 'Last-Modified: Sat, 26 Oct 2013 19:41:47 GMT', 'ETag: "4b9d0211dd8a2819866bccff777af225"', 'Content-Type: text/html', 'Server: Example', 'Transfer-Encoding: chunked', 'non-sence: ' . 'a' x 20000, '', $data ); print $fh $msg; $fh->flush; $fh->seek(0, 0); fileno($fh); }; my (undef, $proto, $status, $head, $body) = Hijk::_read_http_message($fd, 10240, undef, 1); is $status, 200; is $body, "Wikipedia in\r\n\r\nchunks."; is_deeply $head, { "Date" => "Sat, 23 Nov 2013 23:10:28 GMT", "Last-Modified" => "Sat, 26 Oct 2013 19:41:47 GMT", "ETag" => '"4b9d0211dd8a2819866bccff777af225"', "Content-Type" => "text/html", "Server" => "Example", 'non-sence' => 'a' x 20000, "Transfer-Encoding" => "chunked", }; # fetch again without seeking back # this will force select() to return because there are actually # 0 bytes to read - so we can simulate connection closed # from the other end of the socket (like expired keep-alive) (undef, $proto, $status, $head, $body, my $error, my $error_message) = Hijk::_read_http_message($fd, 10240); is $error, Hijk::Error::RESPONSE_BAD_READ_VALUE; like $error_message, qr/0 byte/; done_testing; bench-chunked-response.pl000644000766000024 116013416043414 20431 0ustar00st21277staff000000000000Hijk-0.28/examples#!/usr/bin/env perl use strict; use warnings; use Benchmark ':all'; use Hijk; use HTTP::Tiny; use LWP::UserAgent; use HTTP::Request; my $req = HTTP::Request->new('GET','http://localhost:5000/'); my $tiny = HTTP::Tiny->new(); my $lwp = LWP::UserAgent->new(); cmpthese(10_000,{ 'lwp____' => sub { my $res = $lwp->request($req); }, 'tiny___' => sub { my $res = $tiny->get('http://localhost:5000/'); }, 'hijk pp' => sub { my $res = Hijk::request({ path => "/", host => 'localhost', port => 5000, method => 'GET' }); } }); README.md000644000766000024 43313416043414 13165 0ustar00st21277staff000000000000Hijk-0.28Hijk ==== Specialized HTTP Client [![Tavis-CI Build Status](https://travis-ci.org/gugod/Hijk.png?branch=master)](https://travis-ci.org/gugod/Hijk) [![Coverage Status](https://coveralls.io/repos/gugod/Hijk/badge.png?branch=master)](https://coveralls.io/r/gugod/Hijk?branch=master)