Sphinx-Search-0.29/ 0000755 0000764 0000764 00000000000 12501725115 012265 5 ustar jon jon Sphinx-Search-0.29/t/ 0000755 0000764 0000764 00000000000 12501725115 012530 5 ustar jon jon Sphinx-Search-0.29/t/boilerplate.t 0000644 0000764 0000764 00000002324 11176246374 015234 0 ustar jon jon #!perl -T
use strict;
use warnings;
use Test::More tests => 3;
sub not_in_file_ok {
my ($filename, %regex) = @_;
open my $fh, "<", $filename
or die "couldn't open $filename for reading: $!";
my %violated;
while (my $line = <$fh>) {
while (my ($desc, $regex) = each %regex) {
if ($line =~ $regex) {
push @{$violated{$desc}||=[]}, $.;
}
}
}
if (%violated) {
fail("$filename contains boilerplate text");
diag "$_ appears on lines @{$violated{$_}}" for keys %violated;
} else {
pass("$filename contains no boilerplate text");
}
}
not_in_file_ok(README =>
"The README is used..." => qr/The README is used/,
"'version information here'" => qr/to provide version information/,
);
not_in_file_ok(Changes =>
"placeholder date/time" => qr(Date/time)
);
sub module_boilerplate_ok {
my ($module) = @_;
not_in_file_ok($module =>
'the great new $MODULENAME' => qr/ - The great new /,
'boilerplate description' => qr/Quick summary of what the module/,
'stub function definition' => qr/function[12]/,
);
}
module_boilerplate_ok('lib/Sphinx/Search.pm');
Sphinx-Search-0.29/t/stringattr.t 0000644 0000764 0000764 00000000344 11357476633 015137 0 ustar jon jon #! /usr/bin/perl
use Test::More tests => 1;
use Sphinx::Search;
my $sphinx = Sphinx::Search->new();
eval {
my $r = $sphinx->SetFilter('mystring', ['4bb4afe18d9c4e550798b543']);
};
ok(! $@, "string attribute filtering");
Sphinx-Search-0.29/t/search.t 0000644 0000764 0000764 00000026310 12501470273 014166 0 ustar jon jon #! /usr/bin/perl
# Copyright 2007 Jon Schutz, all rights reserved.
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License.
# Main functional test for Sphinx::Search
# Loads data into mysql, runs indexer, starts searchd, validates results.
use strict;
use warnings;
use DBI;
use Test::More;
use File::SearchPath qw/searchpath/;
use Path::Class;
use Sphinx::Search;
use Socket;
use Data::Dumper;
use List::MoreUtils qw/all/;
use lib qw(t/testlib testlib);
use TestDB;
my $testdb = TestDB->new();
if (my $msg = $testdb->preflight) {
plan skip_all => $msg;
}
unless ($testdb->run_indexer()) {
plan skip_all => "Failed to run indexer; skipping tests.";
}
unless ($testdb->run_searchd()) {
plan skip_all => "Failed to run searchd; skipping tests.";
}
# Everything is in place; run the tests
plan tests => 117;
my $logger;
#use Log::Log4perl qw/:easy/;
#Log::Log4perl->easy_init($DEBUG);
#$logger = Log::Log4perl->get_logger();
my $sphinx = Sphinx::Search->new({ port => $testdb->searchd_port, log => $logger, debug => 1 });
ok($sphinx, "Constructor");
run_all_tests();
$sphinx->SetSortMode(SPH_SORT_RELEVANCE)
->SetRankingMode(SPH_RANK_PROXIMITY_BM25)
->SetFieldWeights({});
$sphinx->SetConnectTimeout(2);
run_all_tests();
sub run_all_tests {
# Basic test on 'a'
my $results = $sphinx->Query("a");
ok($results, "Results for 'a'");
print $sphinx->GetLastError unless $results;
ok($results->{total_found} == 4, "total_found for 'a'");
ok($results->{total} == 4, "total for 'a'");
ok(@{$results->{matches}} == 4, "matches for 'a'");
is_deeply($results->{'words'},
{
'a' => {
'hits' => 4,
'docs' => 4
}
},
"words for 'a'");
is_deeply($results->{'fields'}, [ qw/field1 field2/ ], "fields for 'a'");
is_deeply($results->{'attrs'}, { attr1 => 1, lat => 5, long => 5, stringattr => 7 }, "attributes for 'a'");
my $weight = $results->{matches}[0]{weight};
ok((all { $_->{weight} == $weight } @{$results->{matches}}), "weights for 'a'");
# Rank order test on 'bb'
$sphinx->SetSortMode(SPH_SORT_RELEVANCE);
$results = $sphinx->Query("bb");
ok($results, "Results for 'bb'");
print $sphinx->GetLastError unless $results;
ok(@{$results->{matches}} == 5, "matches for 'bb'");
my $order_ok = 1;
for (1 .. @{$results->{matches}} - 1) {
$order_ok = 0, last unless $results->{matches}->[$_ - 1]->{weight} >= $results->{matches}->[$_]->{weight};
}
ok($order_ok, 'SPH_SORT_RELEVANCE');
# Phrase on "ccc dddd"
$sphinx->SetSortMode(SPH_SORT_ATTR_ASC, "attr1");
$results = $sphinx->Query('"ccc dddd"');
ok($results, "Results for '\"ccc dddd\"'");
print $sphinx->GetLastError unless $results;
ok(@{$results->{matches}} == 3, "matches for '\"ccc dddd\"'");
$order_ok = 1;
for (1 .. @{$results->{matches}} - 1) {
$order_ok = 0, last unless $results->{matches}->[$_ - 1]->{attr1} <= $results->{matches}->[$_]->{attr1};
}
ok($order_ok, 'SPH_SORT_ATTR_ASC');
# Boolean on "bb ccc"
$sphinx->SetSortMode(SPH_SORT_ATTR_DESC, "attr1");
$results = $sphinx->Query("bb ccc");
ok($results, "Results for 'bb ccc'");
print $sphinx->GetLastError unless $results;
ok(@{$results->{matches}} == 4, "matches for 'bb ccc'");
$order_ok = 1;
for (1 .. @{$results->{matches}} - 1) {
$order_ok = 0, last unless $results->{matches}->[$_ - 1]->{attr1} >= $results->{matches}->[$_]->{attr1};
}
ok($order_ok, 'SPH_SORT_ATTR_DESC');
# Any on "bb ccc"
$sphinx->SetSortMode(SPH_SORT_EXTENDED, '@relevance DESC, attr1 ASC');
$results = $sphinx->Query("bb | ccc");
ok($results, "Results for 'bb ccc' ANY");
print $sphinx->GetLastError unless $results;
ok(@{$results->{matches}} == 5, "matches for 'bb ccc' ANY");
$order_ok = 1;
for (1 .. @{$results->{matches}} - 1) {
$order_ok = 0, last unless
($results->{matches}->[$_]->{weight} <=> $results->{matches}->[$_-1]->{weight} ||
$results->{matches}->[$_ - 1]->{attr1} <=> $results->{matches}->[$_]->{attr1}) <= 0;
}
ok($order_ok, 'SPH_SORT_EXTENDED');
$sphinx->SetSortMode(SPH_SORT_RELEVANCE)
->SetLimits(0,2);
$results = $sphinx->Query("bb");
ok($results, "Results for 'bb' with limit");
print $sphinx->GetLastError unless $results;
ok(@{$results->{matches}} == 2, "matches for 'bb'");
# Extended on "bb ccc"
$sphinx->SetLimits(0,20);
$results = $sphinx->Query('@field1 bb @field2 ccc');
ok($results, "Results for 'bb ccc' EXTENDED");
print $sphinx->GetLastError unless $results;
ok(@{$results->{matches}} == 2, "matches for 'bb ccc' EXTENDED");
ok($results->{matches}->[0]->{doc} =~ m/^(?:4|5)$/ &&
$results->{matches}->[1]->{doc} =~ m/^(?:4|5)$/, "matched docs for 'bb ccc' EXTENDED");
# SetIndexWeights
$sphinx->SetSortMode(SPH_SORT_RELEVANCE)
->SetIndexWeights({ test_jjs_index => 2});
$results = $sphinx->Query("bb | ccc");
ok($results, "Results for 'bb | ccc'");
print $sphinx->GetLastError unless $results;
$order_ok = 1;
for (1 .. @{$results->{matches}} - 1) {
$order_ok = 0, last unless $results->{matches}->[$_ - 1]->{weight} >= $results->{matches}->[$_]->{weight} && $results->{matches}->[$_]->{weight} > 1;
}
ok($order_ok, 'Weighted index');
# SetFieldWeights
$sphinx->SetSortMode(SPH_SORT_RELEVANCE)
->SetFieldWeights({ field2 => 2, field1 => 10 });
$results = $sphinx->Query("bb | ccc");
ok($results, "Results for 'bb | ccc'");
print $sphinx->GetLastError unless $results;
$order_ok = 1;
for (1 .. @{$results->{matches}} - 1) {
$order_ok = 0, last unless $results->{matches}->[$_ - 1]->{weight} >= $results->{matches}->[$_]->{weight} && $results->{matches}->[$_]->{weight} > 1;
}
ok($order_ok, 'Field-weighted relevance');
# Excerpts
$results = $sphinx->BuildExcerpts([ "bb bb ccc dddd", "bb ccc dddd" ],
"test_jjs_index",
"ccc dddd");
is_deeply($results, [ 'bb bb ccc dddd', 'bb ccc dddd' ],
"Excerpts");
# Excerpts UTF8
$results = $sphinx->BuildExcerpts([ "\x{65e5}\x{672c}\x{8a9e}" ],
"test_jjs_index",
"\x{65e5}\x{672c}\x{8a9e}");
is_deeply($results, [ "\x{65e5}\x{672c}\x{8a9e}" ], "UTF8 Excerpts");
# Keywords
$results = $sphinx->BuildKeywords("bb-dddd", "test_jjs_index", 1);
is_deeply($results, [
{
'hits' => 8,
'docs' => 5,
'tokenized' => 'bb',
'normalized' => 'bb'
},
{
'hits' => 3,
'docs' => 3,
'tokenized' => 'dddd',
'normalized' => 'dddd'
}
],
"Keywords");
# Keywords UTF8
$results = $sphinx->BuildKeywords("\x{65e5}\x{672c}\x{8a9e}", "test_jjs_index", 1);
is_deeply($results, [
{
'hits' => 1,
'docs' => 1,
'tokenized' => "\x{65e5}\x{672c}\x{8a9e}",
'normalized' => "\x{65e5}\x{672c}\x{8a9e}"
}
]);
# EscapeString
$results = $sphinx->EscapeString(q{$#abcde!@%});
is($results, '\$\#abcde\!\@\%', "EscapeString");
# Update
$sphinx->UpdateAttributes("test_jjs_index", [ qw/attr1/ ],
{
1 => [ 10 ],
2 => [ 10 ],
3 => [ 20 ],
4 => [ 20 ],
});
# Verify update with grouped search
$sphinx->SetSortMode(SPH_SORT_RELEVANCE)
->SetGroupBy("attr1", SPH_GROUPBY_ATTR);
$results = $sphinx->Query("bb");
ok($results, "Results for 'bb'");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 3, "Update attributes, grouping");
# Attribute filters
$sphinx->ResetGroupBy
->SetFilter("attr1", [ 10 ]);
$results = $sphinx->Query("bb");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 2, "Filter");
# Attribute exclude
$sphinx->ResetFilters->SetFilter("attr1", [ 10 ], 1);
$results = $sphinx->Query("bb");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 3, "Filter exclude");
# String filter
$sphinx->ResetFilters
->SetFilterString("stringattr", 'new string attribute');
$results = $sphinx->Query("bb");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 1, "String Filter");
# Range filters
$sphinx->ResetFilters->SetFilterRange("attr1", 2, 11);
$results = $sphinx->Query("bb");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 3, "Range filter");
# Range filters exclude
$sphinx->ResetFilters->SetFilterRange("attr1", 2, 11, 1);
$results = $sphinx->Query("bb");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 2, "Range filter exclude");
# Float range filters
$sphinx->ResetFilters->SetFilterFloatRange("lat", 0.2, 0.4);
$results = $sphinx->Query("a");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 3, "Float range filter");
# Float range filters exclude
$sphinx->ResetFilters->SetFilterFloatRange("lat", 0.2, 0.4, 1);
$results = $sphinx->Query("a");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 1, "Float range filter exclude");
# ID Range
$sphinx->ResetFilters->SetIDRange(2, 4);
$results = $sphinx->Query("bb");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 3, "ID range");
# Geodistance
$sphinx->SetGeoAnchor('lat', 'long', 0.4, 0.4)
->SetSortMode(SPH_SORT_EXTENDED, '@geodist desc')
->SetFilterFloatRange('@geodist', 0, 1934127);
$results = $sphinx->Query('a');
print $sphinx->GetLastError unless $results;
ok($results->{total} == 2, "SetGeoAnchor");
# UTF-8 test
$sphinx->ResetFilters->SetSortMode(SPH_SORT_RELEVANCE)->SetIDRange(0, 0xFFFFFFFF);
$results = $sphinx->Query("bb\x{2122}");
ok($results, "UTF-8");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 5, "UTF-8 results count");
$results = $sphinx->Query("\x{65e5}\x{672c}\x{8a9e}");
ok($results->{total} == 1, "UTF-8 japanese results count");
ok($results->{words}->{"\x{65e5}\x{672c}\x{8a9e}"}, "UTF-8 japanese match");
# SetQueryFlag
$sphinx->SetQueryFlag(SPH_QF_REVERSE_SCAN, 1);
$results = $sphinx->Query("");
ok($results, "SetQueryFlag"); # not sure what can be tested here
# Batch interface
$sphinx->AddQuery("ccc");
$sphinx->AddQuery("dddd");
$results = $sphinx->RunQueries;
ok(@$results == 2, "Results for batch query");
# Batch interface with error
$sphinx->AddQuery("ccc @\@dddd");
$sphinx->AddQuery("dddd");
$results = $sphinx->RunQueries;
ok(@$results == 2, "Results for batch query with error");
ok($results->[0]->{error}, "Error result");
# 64 bit ID
# Check for id64 support
SKIP: {
my $searchd = $testdb->searchd;
my $sig = `$searchd`;
skip "searchd not compiled with --enable-id64: 64 bit IDs not supported", 3 unless $sig =~ m/id64/;
$sphinx->ResetFilters
->SetIDRange(0, '18446744073709551615')
->SetSortMode(SPH_SORT_RELEVANCE);
$results = $sphinx->Query("xx");
#print Dumper($results);
#skip "64 bit IDs not supported", 3 if !$results && $sphinx->GetLastError =~ m/zero-sized/;
ok($results, "Results for 'xx'");
print $sphinx->GetLastError unless $results;
ok($results->{total} == 1, "ID 64 results count");
is($results->{matches}->[0]->{doc}, '9223372036854775807', "ID 64");
}
# Status
my $status = $sphinx->Status();
ok( $status->{connections} > 0, "Status");
ok(persistent_connection_test($sphinx), "persistent connection");
}
sub persistent_connection_test {
my $sph = shift;
unless ($sph->Open()) {
warn "Open() failed";
return 0;
}
$sph->ResetFilters
->SetSortMode(SPH_SORT_RELEVANCE);
for (1..10) {
my $results = $sphinx->Query("bb") or die "No results";
return 0 unless $results->{total} == 5;
}
unless ($sph->Close()) {
warn "Close() failed";
return 0;
}
return 1;
}
Sphinx-Search-0.29/t/64bitid.t 0000644 0000764 0000764 00000001734 11176246374 014203 0 ustar jon jon #! /usr/bin/perl
# Copyright 2007 Jon Schutz, all rights reserved.
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License.
use strict;
use warnings;
use Sphinx::Search;
use Test::More tests => 21;
my $sphinx = Sphinx::Search->new;
ok($sphinx, "Constructor");
my @tests = ( 0, 1, 0x7FFFFFFF, 0x80000000, 0xFFFFFFFF, '4294967296', '9223372036854775807', '9223372036854775808', '18446744073709551615');
for my $x (@tests) {
# print $x . " " . $sphinx->_sphUnpackU64($sphinx->_sphPackU64($x)) . "\n";
ok($sphinx->_sphUnpackU64($sphinx->_sphPackU64($x)) == $x, "64 bit unsigned transfer $x");
}
my @signed_tests = ( 0, 1, -1, 0x7FFFFFFF, 0x80000000, 0xFFFFFFFF, -0x7FFFFFFF, -0x80000000, -0xFFFFFFFF, '-4294967296', '-9223372036854775807');
for my $x (@signed_tests) {
my $packed = $sphinx->_sphPackI64($x);
ok($sphinx->_sphUnpackI64($sphinx->_sphPackI64($x)) == $x, "64 bit signed transfer $x");
}
Sphinx-Search-0.29/t/pod-coverage.t 0000644 0000764 0000764 00000000254 11176246374 015305 0 ustar jon jon #!perl -T
use Test::More;
eval "use Test::Pod::Coverage 1.04";
plan skip_all => "Test::Pod::Coverage 1.04 required for testing POD coverage" if $@;
all_pod_coverage_ok();
Sphinx-Search-0.29/t/00-load.t 0000644 0000764 0000764 00000000230 11176246374 014060 0 ustar jon jon #!perl -T
use Test::More tests => 1;
BEGIN {
use_ok( 'Sphinx::Search' );
}
diag( "Testing Sphinx::Search $Sphinx::Search::VERSION, Perl $], $^X" );
Sphinx-Search-0.29/t/pod.t 0000644 0000764 0000764 00000000214 11176246374 013510 0 ustar jon jon #!perl -T
use Test::More;
eval "use Test::Pod 1.14";
plan skip_all => "Test::Pod 1.14 required for testing POD" if $@;
all_pod_files_ok();
Sphinx-Search-0.29/t/testlib/ 0000755 0000764 0000764 00000000000 12501725115 014176 5 ustar jon jon Sphinx-Search-0.29/t/testlib/TestDB.pm 0000644 0000764 0000764 00000022354 12501461634 015672 0 ustar jon jon package TestDB;
use strict;
use warnings;
use base qw(Class::Accessor::Fast);
use DBI;
use File::SearchPath qw/searchpath/;
use Path::Class;
use Socket;
__PACKAGE__->mk_accessors(qw/searchd indexer searchd_port dbtable dsn dbuser dbpass dbname dbhost dbport dbsock testdir configfile pidfile/);
our @pids;
our @pidfiles;
sub new {
my $class = shift;
my $self = $class->SUPER::new(@_);
$self->searchd($ENV{SPHINX_SEARCHD} || searchpath('searchd')) unless $self->searchd;
$self->indexer($ENV{SPHINX_INDEXER} || searchpath('indexer')) unless $self->indexer;
$self->searchd_port($ENV{SPHINX_PORT} || int(rand(20000))) unless $self->searchd_port;;
return $self;
}
sub preflight {
my $self = shift;
my $msg;
$msg = $self->searchd_check and return $msg;
$msg = $self->indexer_check and return $msg;
$msg = $self->db_check and return $msg;
$msg = $self->files_check and return $msg;
return;
}
sub searchd_check {
my $self = shift;
my $searchd = $self->searchd;
unless ($searchd && -e $searchd) {
return "Can't find searchd; set SPHINX_SEARCHD to location of searchd binary in order to run these tests";
}
return;
}
sub indexer_check {
my $self = shift;
my $searchd = $self->searchd;
my $indexer = $self->indexer;
$indexer = Path::Class::file($searchd)->dir->file('indexer')->stringify unless $indexer;
unless ($indexer && -e $indexer) {
return "Can't find indexer; set SPHINX_INDEXER to location of indexer binary in order to run these tests";
}
$self->indexer($indexer);
return;
}
sub db_check {
my $self = shift;
$self->dbtable(my $dbtable = 'sphinx_test_jjs_092348792');
$self->dsn(my $dsn = $ENV{SPHINX_DSN} || "dbi:mysql:database=test");
$self->dbuser(my $dbuser = $ENV{SPHINX_DBUSER} || "root");
$self->dbpass(my $dbpass = $ENV{SPHINX_DBPASS} || "");
$self->dbname(( $dsn =~ m!database=([^;]+)! ) ? $1 : "test");
$self->dbhost(( $dsn =~ m!host=([^;]+)! ) ? $1 : "localhost");
$self->dbport(( $dsn =~ m!port=([^;]+)! ) ? $1 : "");
$self->dbsock(( $dsn =~ m!socket=([^;]+)! ) ? $1 : "");
my $dbi = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 0 });
unless ($dbi) {
return "Failed to connect to database; set SPHINX_DSN, SPHINX_DBUSER, SPHINX_DBPASS appropriately to run these tests";
}
unless ($self->create_db($dbi)) {
return "Failed to create database table; set SPHINX_DSN, SPHINX_DBUSER, SPHINX_DBPASS appropriately to run these tests";
}
return;
}
sub files_check {
my $self = shift;
$self->testdir(my $testdir = Path::Class::dir("data")->absolute);
eval { $testdir->mkpath };
if ($@) {
return "Failed to create 'data' directory; skipping tests. Fix permissions to run test";
}
$self->pidfile($testdir->file('searchd.pid'));
push(@pidfiles, $self->pidfile);
$self->configfile(my $configfile = $testdir->file('sphinx.conf'));
unless ($self->write_config($configfile)) {
return "Failed to write config file; skipping tests. Fix permissions to run test";
}
return;
}
sub create_db {
my ($self, $dbi) = @_;
my $dbtable = $self->dbtable;
eval {
$dbi->do(qq{DROP TABLE IF EXISTS \`$dbtable\`});
$dbi->do(qq{SET NAMES utf8});
$dbi->do(qq{CREATE TABLE \`$dbtable\` (
\`id\` BIGINT UNSIGNED NOT NULL auto_increment,
\`field1\` TEXT,
\`field2\` TEXT,
\`attr1\` INT NOT NULL,
\`lat\` FLOAT NOT NULL,
\`long\` FLOAT NOT NULL,
\`stringattr\` VARCHAR(100),
PRIMARY KEY (\`id\`)) DEFAULT CHARSET=utf8 COLLATE=utf8_bin });
$dbi->do(qq{INSERT INTO \`$dbtable\` (\`id\`,\`field1\`,\`field2\`,\`attr1\`,\`lat\`,\`long\`,\`stringattr\`) VALUES
(1, 'a', 'bb', 2, 0.35, 0.70, ''),
(2, 'a', 'bb ccc', 4, 0.70, 0.35, ''),
(3, 'a', 'bb ccc dddd', 1, 0.35, 0.70, ''),
(4, 'a bb', 'bb ccc dddd', 5, 0.35, 0.70, ''),
(5, 'bb', 'bb bb ccc dddd', 3, 1.5, 1.5, 'new string attribute'),
('9223372036854775807', 'xx', 'xx', 9000, 150, 150, ''),
(6, "\x{65e5}\x{672c}\x{8a9e}", '', 0, 0, 0, '')
});
};
if ($@) {
print STDERR "Failed to create/load database table: $@\n";
return 0;
}
return 1;
}
sub write_config {
my $self = shift;
my $configfile = $self->configfile;
my $testdir = $self->testdir;
my $pidfile = $self->pidfile;
my $dbhost = $self->dbhost;
my $dbuser = $self->dbuser;
my $dbpass = $self->dbpass;
my $dbname = $self->dbname;
my $dbport = $self->dbport;
my $dbsock = $self->dbsock;
my $dbtable = $self->dbtable;
my $searchd_port = $self->searchd_port;
eval {
my $config = <a..z, U+3041->U+30A2, U+3042->U+30A2, U+3043->U+30A4, U+3044->U+30A4, U+3045->U+30A6, U+3046->U+30A6, U+3047->U+30A8, U+3048->U+30A8, U+3049->U+30AA, U+304A->U+30AA, U+304B..U+3062->U+30AB..U+30C2, U+3063->U+30C4, U+3064..U+3082->U+30C4..U+30E2, U+3083->U+30E4, U+3084->U+30E4, U+3085->U+30E6, U+3086->U+30E6, U+3087->U+30E8, U+3088->U+30E8, U+3089..U+308D->U+30E9..U+30ED, U+308E->U+30EF, U+308F..U+3094->U+30EF..U+30F4, U+3095->U+30AB, U+3096->U+30B1, U+309F->U+30FF, U+30A1->U+30A2, U+30A2, U+30A3->U+30A4, U+30A4, U+30A5->U+30A6, U+30A6, U+30A7->U+30A8, U+30A8, U+30A9->U+30AA, U+30AA, U+30AB..U+30C2, U+30C3->U+30C4, U+30C4..U+30E2, U+30E3->U+30E4, U+30E4, U+30E5->U+30E6, U+30E6, U+30E7->U+30E8, U+30E8..U+30ED, U+30EE->U+30EF, U+30EF..U+30F4, U+30F5->U+30AB, U+30F6->U+30B1, U+30FA, U+30FF, U+31F0->U+30AF, U+31F1->U+30B7, U+31F2->U+30B9, U+31F3->U+30C8, U+31F4->U+30CC, U+31F5->U+30CF, U+31F6->U+30D2, U+31F7->U+30D5, U+31F8->U+30D8, U+31F9->U+30DB, U+31FA->U+30E0, U+31FB..U+31FF->U+30E9..U+30ED, U+3400..U+4DB5, U+4E00..U+9FC3, U+F900..U+FAD9, U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z, U+FF41..U+FF5A->a..z, U+FF66->U+30F2, U+FF67->U+30A2, U+FF68->U+30A4, U+FF69->U+30A6, U+FF6A->U+30A8, U+FF6B->U+30AA, U+FF6C->U+30E4, U+FF6D->U+30E6, U+FF6E->U+30E8, U+FF6F->U+30C4, U+FF71->U+30A2, U+FF72->U+30A4, U+FF73->U+30A6, U+FF74->U+30A8, U+FF75->U+30AA, U+FF76->U+30AD, U+FF78->U+30AF, U+FF79->U+30B1, U+FF7A->U+30B3, U+FF7B->U+30B5, U+FF7C->U+30B7, U+FF7D->U+30B9, U+FF7E->U+30BB, U+FF7F->U+30BD, U+FF80->U+30BF, U+FF81->U+30C1, U+FF82->U+30C4, U+FF83->U+30C6, U+FF84->U+30C8, U+FF85..U+FF8A->U+30CA..U+30CF, U+FF8B->U+30D2, U+FF8C->U+30D5, U+FF8D->U+30D8, U+FF8E->U+30DB, U+FF8F..U+FF93->U+30DE..U+30E2, U+FF94->U+30E4, U+FF95->U+30E6, U+FF96..U+FF9B->U+30E8..U+30ED, U+FF9C->U+30EF, U+FF9D->U+30F3, U+20000..U+2A6D6, U+2F800..U+2FA1D
}
searchd {
listen = $searchd_port
log = $testdir/searchd.log
query_log = $testdir/query.log
pid_file = $pidfile
}
EOF
$config =~ s/sql_sock.*// unless $self->dbsock;
$config =~ s/sql_port.*// unless $self->dbport;
open(CONFIG, ">$configfile");
print CONFIG $config;
close(CONFIG);
};
if ($@) {
print STDERR "While writing config: $@\n";
return 0;
}
return 1;
}
sub run_indexer {
my $self = shift;
my $configfile = $self->configfile;
my $indexer = $self->indexer;
my $res = `$indexer --config $configfile test_jjs_index`;
if ($? != 0 || $res =~ m!ERROR!) {
print STDERR "Indexer returned $?: $res";
return 0;
}
return 1;
}
sub run_searchd {
my $self = shift;
my $configfile = $self->configfile;
my $pidfile = $self->pidfile;
my $searchd = $self->searchd;
my ($pid) = _run_forks(sub {
# open STDOUT, '>&STDERR';
open STDIN, '/dev/null';
open STDOUT, '>/dev/null';
open STDERR, '>&STDOUT';
exec("$searchd --config $configfile");
});
my $fp;
unless (socket($fp, PF_INET, SOCK_STREAM, getprotobyname('tcp'))) {
print STDERR "Failed to create socket: $!";
return 0;
}
my $dest = sockaddr_in($self->searchd_port, inet_aton("localhost"));
for (0..3) {
last if -f "$pidfile";
sleep(1);
}
for (0..3) {
if (connect($fp, $dest)) {
close($fp);
return 1;
}
else {
sleep(1);
}
}
return 0;
}
sub _death_handler {
if (@pids) {
kill(15, $_) for @pids;
}
for my $pidfile (@pidfiles) {
my $pid = $pidfile->slurp;
kill(15, $pid) if $pid;
}
}
sub _run_forks {
my ($forks) = @_;
my @newpids;
if ($forks) {
$forks = [ $forks ] unless (ref($forks) eq "ARRAY");
for my $f (@{$forks}) {
my $pid = fork();
die "Fork failed: $!" unless defined $pid;
if ($pid == 0) {
@pids = (); # prevent child from killing siblings
# Child process
if (ref($f) eq "CODE") {
&$f;
}
else {
print STDERR "Don't know how to run test $f\n";
}
exit(0);
}
# Push PID for killing.
push(@pids, $pid);
push(@newpids, $pid);
}
$SIG{INT} = \&_death_handler;
$SIG{KILL} = \&_death_handler;
$SIG{TERM} = \&_death_handler;
$SIG{QUIT} = \&_death_handler;
}
return @newpids;
}
END {
_death_handler();
}
1;
Sphinx-Search-0.29/t/connect.t 0000644 0000764 0000764 00000001047 11176246374 014364 0 ustar jon jon #! /usr/bin/perl
use Test::More;
use strict;
use warnings;
use Socket;
plan tests => 4;
use_ok('Sphinx::Search');
my $sph_port = rand(12345);
my $fp;
# Create listening socket that never responds.
socket($fp, PF_INET, SOCK_STREAM, getprotobyname('tcp')) or
die("socket: $!");
bind($fp, sockaddr_in($sph_port, INADDR_ANY));
listen($fp, 1);
my $sphinx = Sphinx::Search->new({ port => $sph_port });
ok($sphinx, "Constructor");
my $t = time();
$sphinx->SetConnectTimeout(1);
ok(! $sphinx->_Connect, "connect");
ok(time < $t + 2, "Timeout");
Sphinx-Search-0.29/t/setserver.t 0000644 0000764 0000764 00000001563 11176246374 014760 0 ustar jon jon #! /usr/bin/perl
use Test::More;
use strict;
use warnings;
use Socket;
use Sphinx::Search;
use lib qw(t/testlib testlib);
use TestDB;
my $testdb = TestDB->new();
if (my $msg = $testdb->preflight) {
plan skip_all => $msg;
}
unless ($testdb->run_searchd()) {
plan skip_all => "Failed to run searchd; skipping tests.";
}
plan tests => 5;
my $sphinx = Sphinx::Search->new({ port => $testdb->searchd_port });
ok($sphinx, "Constructor");
my $e;
$sphinx->SetServer('', $testdb->searchd_port);
$e = $sphinx->Query('a');
ok(! $e, "Error on empty server");
like($sphinx->GetLastError(), qr/Failed to open connection|Bad arg length/);
$sphinx->Query('a');
$sphinx->SetServer('my.nosuchhost.exists', $testdb->searchd_port);
$e = $sphinx->Query('a');
ok(! $e, "Error on non-existent server");
like($sphinx->GetLastError(), qr/Failed to open connection|Bad arg length/);
Sphinx-Search-0.29/README 0000644 0000764 0000764 00000001543 11176246374 013164 0 ustar jon jon Sphinx-Search
Perl API client for Sphinx search engine.
INSTALLATION
To install this module, run the following commands:
perl Makefile.PL
make
make test
make install
SUPPORT AND DOCUMENTATION
After installing, you can find documentation for this module with the perldoc command.
perldoc Sphinx::Search
You can also look for information at:
Search CPAN
http://search.cpan.org/dist/Sphinx-Search
CPAN Request Tracker:
http://rt.cpan.org/NoAuth/Bugs.html?Dist=Sphinx-Search
AnnoCPAN, annotated CPAN documentation:
http://annocpan.org/dist/Sphinx-Search
CPAN Ratings:
http://cpanratings.perl.org/d/Sphinx-Search
COPYRIGHT AND LICENCE
Copyright (C) 2007 Jon Schutz
This program is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License.
Sphinx-Search-0.29/MANIFEST 0000644 0000764 0000764 00000000465 12501725115 013423 0 ustar jon jon Changes
lib/Sphinx/Search.pm
Makefile.PL
MANIFEST This list of files
META.yml
README
t/00-load.t
t/64bitid.t
t/boilerplate.t
t/connect.t
t/pod-coverage.t
t/pod.t
t/search.t
t/setserver.t
t/stringattr.t
t/testlib/TestDB.pm
META.json Module JSON meta-data (added by MakeMaker)
Sphinx-Search-0.29/lib/ 0000755 0000764 0000764 00000000000 12501725115 013033 5 ustar jon jon Sphinx-Search-0.29/lib/Sphinx/ 0000755 0000764 0000764 00000000000 12501725115 014304 5 ustar jon jon Sphinx-Search-0.29/lib/Sphinx/Search.pm 0000644 0000764 0000764 00000235466 12501724723 016073 0 ustar jon jon package Sphinx::Search;
use warnings;
use strict;
use base 'Exporter';
use Carp;
use Socket;
use Config;
use Math::BigInt;
use IO::Socket::INET;
use IO::Socket::UNIX;
use Encode qw/encode_utf8 decode_utf8/;
use List::MoreUtils qw/any/;
my $is_native64 = $Config{longsize} == 8 || defined $Config{use64bitint} || defined $Config{use64bitall};
=head1 NAME
Sphinx::Search - Sphinx search engine API Perl client
=head1 VERSION
Please note that you *MUST* install a version which is compatible with your version of Sphinx.
Use version 0.29 for Sphinx-2.2.8-release or later (or use DBI instead)
Use version 0.28 for Sphinx-2.0.8-release or later
Use version 0.27.2 for Sphinx-2.0.3-release (svn-r3043)
Use version 0.26.1 for Sphinx-2.0.1-beta (svn-r2792)
Use version 0.25_03 for Sphinx svn-r2575
Use version 0.24.1 for Sphinx-1.10-beta (svn-r2420)
Use version 0.23_02 for Sphinx svn-r2269 (experimental)
Use version 0.22 for Sphinx 0.9.9-rc2 and later (Please read the Compatibility Note under L regarding encoding changes)
Use version 0.15 for Sphinx 0.9.9-svn-r1674
Use version 0.12 for Sphinx 0.9.8
Use version 0.11 for Sphinx 0.9.8-rc1
Use version 0.10 for Sphinx 0.9.8-svn-r1112
Use version 0.09 for Sphinx 0.9.8-svn-r985
Use version 0.08 for Sphinx 0.9.8-svn-r871
Use version 0.06 for Sphinx 0.9.8-svn-r820
Use version 0.05 for Sphinx 0.9.8-cvs-20070907
Use version 0.02 for Sphinx 0.9.8-cvs-20070818
=cut
our $VERSION = '0.29';
=head1 SYNOPSIS
use Sphinx::Search;
$sph = Sphinx::Search->new();
# Standard API query
$results = $sph->SetSortMode(SPH_SORT_RELEVANCE)
->Query("search terms");
# SphinxQL query
$results = $sph->SphinxQL("SELECT * FROM myindex WHERE MATCH('search terms')");
=head1 DESCRIPTION
This is the Perl API client for the Sphinx open-source SQL full-text indexing
search engine, L.
Since 0.9.9, Sphinx supports a native MySQL-protocol client, i.e. DBI with DBD::mysql. That is, you can configure the server to have a mysql41 listening port and then simply do
my $dbh = DBI->connect('dbi:mysql:host=127.0.0.1;port=9306;mysql_enable_utf8=1') or die "Failed to connect via DBI";
my $sth = $dbh->prepare_cached("SELECT * FROM myindex WHERE MATCH('search terms')");
$sth->execute();
while (my $row = $sth->fetchrow_arrayref) {
... # Collect results
}
The DBI client turns out to be significantly (about 5x) faster than this pure-Perl API. You should probably be using that instead.
This module also supports SphinxQL queries, with the small advantage that you can use either the standard API or the SphinxQL API over the one port (i.e. the searchd server does not need to be configured with a mysql41 listening port).
Given that the DBI client has several advantages over this API, future updates of this module are unlikely.
=cut
# Constants to export.
our @EXPORT = qw(
SPH_MATCH_ALL SPH_MATCH_ANY SPH_MATCH_PHRASE SPH_MATCH_BOOLEAN SPH_MATCH_EXTENDED
SPH_MATCH_FULLSCAN SPH_MATCH_EXTENDED2
SPH_RANK_PROXIMITY_BM25 SPH_RANK_BM25 SPH_RANK_NONE SPH_RANK_WORDCOUNT
SPH_RANK_PROXIMITY SPH_RANK_MATCHANY SPH_RANK_FIELDMASK SPH_RANK_SPH04 SPH_RANK_EXPR
SPH_RANK_TOTAL
SPH_SORT_RELEVANCE SPH_SORT_ATTR_DESC SPH_SORT_ATTR_ASC SPH_SORT_TIME_SEGMENTS
SPH_SORT_EXTENDED SPH_SORT_EXPR
SPH_GROUPBY_DAY SPH_GROUPBY_WEEK SPH_GROUPBY_MONTH SPH_GROUPBY_YEAR SPH_GROUPBY_ATTR
SPH_GROUPBY_ATTRPAIR
SPH_ATTR_INTEGER SPH_ATTR_TIMESTAMP SPH_ATTR_ORDINAL SPH_ATTR_BOOL
SPH_ATTR_FLOAT SPH_ATTR_BIGINT SPH_ATTR_STRING SPH_ATTR_MULTI SPH_ATTR_MULTI64
SPH_QF_REVERSE_SCAN SPH_QF_SORT_METHOD SPH_QF_MAX_PREDICTED_TIME
SPH_QF_BOOLEAN_SIMPLIFY SPH_QF_IDF SPH_QF_GLOBAL_IDF
);
# known searchd commands
use constant SEARCHD_COMMAND_SEARCH => 0;
use constant SEARCHD_COMMAND_EXCERPT => 1;
use constant SEARCHD_COMMAND_UPDATE => 2;
use constant SEARCHD_COMMAND_KEYWORDS => 3;
use constant SEARCHD_COMMAND_PERSIST => 4;
use constant SEARCHD_COMMAND_STATUS => 5;
use constant SEARCHD_COMMAND_FLUSHATTRS => 7;
use constant SEARCHD_COMMAND_SPHINXQL => 8;
# current client-side command implementation versions
use constant VER_COMMAND_SEARCH => 0x11E;
use constant VER_COMMAND_EXCERPT => 0x104;
use constant VER_COMMAND_UPDATE => 0x103;
use constant VER_COMMAND_KEYWORDS => 0x100;
use constant VER_COMMAND_STATUS => 0x101;
use constant VER_COMMAND_FLUSHATTRS => 0x100;
use constant VER_COMMAND_SPHINXQL => 0x100;
# known searchd status codes
use constant SEARCHD_OK => 0;
use constant SEARCHD_ERROR => 1;
use constant SEARCHD_RETRY => 2;
use constant SEARCHD_WARNING => 3;
# known match modes
use constant SPH_MATCH_ALL => 0;
use constant SPH_MATCH_ANY => 1;
use constant SPH_MATCH_PHRASE => 2;
use constant SPH_MATCH_BOOLEAN => 3;
use constant SPH_MATCH_EXTENDED => 4;
use constant SPH_MATCH_FULLSCAN => 5;
use constant SPH_MATCH_EXTENDED2 => 6; # extended engine V2 (TEMPORARY, WILL BE REMOVED
# known ranking modes (ext2 only)
use constant SPH_RANK_PROXIMITY_BM25 => 0; # default mode, phrase proximity major factor and BM25 minor one
use constant SPH_RANK_BM25 => 1; # statistical mode, BM25 ranking only (faster but worse quality)
use constant SPH_RANK_NONE => 2; # no ranking, all matches get a weight of 1
use constant SPH_RANK_WORDCOUNT => 3; # simple word-count weighting, rank is a weighted sum of per-field keyword occurence counts
use constant SPH_RANK_PROXIMITY => 4;
use constant SPH_RANK_MATCHANY => 5;
use constant SPH_RANK_FIELDMASK => 6;
use constant SPH_RANK_SPH04 => 7;
use constant SPH_RANK_EXPR => 8;
use constant SPH_RANK_TOTAL => 9;
# known sort modes
use constant SPH_SORT_RELEVANCE => 0;
use constant SPH_SORT_ATTR_DESC => 1;
use constant SPH_SORT_ATTR_ASC => 2;
use constant SPH_SORT_TIME_SEGMENTS => 3;
use constant SPH_SORT_EXTENDED => 4;
use constant SPH_SORT_EXPR => 5;
# known filter types
use constant SPH_FILTER_VALUES => 0;
use constant SPH_FILTER_RANGE => 1;
use constant SPH_FILTER_FLOATRANGE => 2;
use constant SPH_FILTER_STRING => 3;
# known attribute types
use constant SPH_ATTR_INTEGER => 1;
use constant SPH_ATTR_TIMESTAMP => 2;
use constant SPH_ATTR_ORDINAL => 3;
use constant SPH_ATTR_BOOL => 4;
use constant SPH_ATTR_FLOAT => 5;
use constant SPH_ATTR_BIGINT => 6;
use constant SPH_ATTR_STRING => 7;
use constant SPH_ATTR_FACTORS => 1001;
use constant SPH_ATTR_MULTI => 0x40000001;
use constant SPH_ATTR_MULTI64 => 0x40000002;
# known grouping functions
use constant SPH_GROUPBY_DAY => 0;
use constant SPH_GROUPBY_WEEK => 1;
use constant SPH_GROUPBY_MONTH => 2;
use constant SPH_GROUPBY_YEAR => 3;
use constant SPH_GROUPBY_ATTR => 4;
use constant SPH_GROUPBY_ATTRPAIR => 5;
use constant {
SPH_QF_REVERSE_SCAN => 'reverse_scan',
SPH_QF_SORT_METHOD => 'sort_method',
SPH_QF_MAX_PREDICTED_TIME => 'max_predicted_time',
SPH_QF_BOOLEAN_SIMPLIFY => 'boolean_simplify',
SPH_QF_IDF => 'idf',
SPH_QF_GLOBAL_IDF => 'global_idf',
};
my %query_flags = ( SPH_QF_REVERSE_SCAN() => [ 0, 1 ],
SPH_QF_SORT_METHOD() => [ qw/pq kbuffer/ ],
SPH_QF_MAX_PREDICTED_TIME() => [ 0 ],
SPH_QF_BOOLEAN_SIMPLIFY() => [1, 0],
SPH_QF_IDF() => [ qw/normalized plain tfidf_normalized tfidf_unnormalized/ ],
SPH_QF_GLOBAL_IDF() => [1, 0],
);
use constant MYSQL_COL_STRING => 254;
# Floating point number matching expression
my $num_re = qr/^-?\d*\.?\d*(?:[eE][+-]?\d+)?$/;
# portably pack numeric to 64 signed bits, network order
sub _sphPackI64 {
my $self = shift;
my $v = shift;
# x64 route
my $i = $is_native64 ? int($v) : Math::BigInt->new("$v");
return pack ( "NN", $i>>32, $i & 4294967295 );
}
# portably pack numeric to 64 unsigned bits, network order
sub _sphPackU64 {
my $self = shift;
my $v = shift;
my $i = $is_native64 ? int($v) : Math::BigInt->new("$v");
return pack ( "NN", $i>>32, $i & 4294967295 );
}
sub _sphPackI64array {
my $self = shift;
my $values = shift || [];
my $s = pack("N", scalar @$values);
$s .= $self->_sphPackI64($_) for @$values;
return $s;
}
# portably unpack 64 unsigned bits, network order to numeric
sub _sphUnpackU64
{
my $self = shift;
my $v = shift;
my ($h,$l) = unpack ( "N*N*", $v );
# x64 route
return ($h<<32) + $l if $is_native64;
# x32 route, BigInt
$h = Math::BigInt->new($h);
$h->blsft(32)->badd($l);
return $h->bstr;
}
# portably unpack 64 signed bits, network order to numeric
sub _sphUnpackI64
{
my $self = shift;
my $v = shift;
my ($h,$l) = unpack ( "N*N*", $v );
my $neg = ($h & 0x80000000) ? 1 : 0;
# x64 route
if ( $is_native64 ) {
return -(~(($h<<32) + $l) + 1) if $neg;
return ($h<<32) + $l;
}
# x32 route, BigInt
if ($neg) {
$h = ~$h;
$l = ~$l;
}
my $x = Math::BigInt->new($h);
$x->blsft(32)->badd($l);
$x->binc()->bneg() if $neg;
return $x->bstr;
}
sub _sphSetBit {
my ($self, $flag, $bit, $on) = @_;
if ($on) {
$flag |= (1 << $bit);
}
else {
$flag &= ~(1 << $bit);
}
return $flag;
}
=head1 CONSTRUCTOR
=head2 new
$sph = Sphinx::Search->new;
$sph = Sphinx::Search->new(\%options);
Create a new Sphinx::Search instance.
OPTIONS
=over 4
=item log
Specify an optional logger instance. This can be any class that provides error,
warn, info, and debug methods (e.g. see L). Logging is disabled
if no logger instance is provided.
=item debug
Debug flag. If set (and a logger instance is specified), debugging messages
will be generated.
=back
=cut
# create a new client object and fill defaults
sub new {
my ($class, $options) = @_;
my $self = {
# per=client-object settings
_host => 'localhost',
_port => 9312,
_path => undef,
_socket => undef,
_persistent => undef,
_connectretries => 1,
# per-query settings
_offset => 0,
_limit => 20,
_mode => SPH_MATCH_EXTENDED2,
_weights => [],
_sort => SPH_SORT_RELEVANCE,
_sortby => "",
_min_id => 0,
_max_id => 0,
_filters => [],
_groupby => "",
_groupdistinct => "",
_groupfunc => SPH_GROUPBY_DAY,
_groupsort => '@group desc',
_maxmatches => 1000,
_cutoff => 0,
_retrycount => 0,
_retrydelay => 0,
_anchor => undef,
_indexweights => undef,
_ranker => SPH_RANK_PROXIMITY_BM25,
_rankexpr => "",
_maxquerytime => 0,
_fieldweights => {},
_overrides => {},
_select => q{*},
# per-reply fields (for single-query case)
_error => '',
_warning => '',
_connerror => '',
# request storage (for multi-query case)
_reqs => [],
_timeout => 0,
_string_encoder => \&encode_utf8,
_string_decoder => \&decode_utf8,
};
bless $self, ref($class) || $class;
$self->ResetQueryFlag;
$self->ResetOuterSelect;
# These options are supported in the constructor, but not recommended
# since there is no validation. Use the Set* methods instead.
my %legal_opts = map { $_ => 1 } qw/host port offset limit mode weights sort sortby groupby groupbyfunc maxmatches cutoff retrycount retrydelay log debug string_encoder string_decoder/;
for my $opt (keys %$options) {
$self->{'_' . $opt} = $options->{$opt} if $legal_opts{$opt};
}
# Disable debug unless we have something to log to
$self->{_debug} = 0 unless $self->{_log};
return $self;
}
=head1 METHODS
=cut
sub _Error {
my ($self, $msg) = @_;
$self->{_error} = $msg;
$self->{_log}->error($msg) if $self->{_log};
return;
}
sub _Throw {
my ($self, $msg) = @_;
die $msg;
}
=head2 GetLastError
$error = $sph->GetLastError;
Get last error message (string)
=cut
sub GetLastError {
my $self = shift;
return $self->{_error};
}
sub _Warning {
my ($self, $msg) = @_;
$self->{_warning} = $msg;
$self->{_log}->warn($msg) if $self->{_log};
return;
}
=head2 GetLastWarning
$warning = $sph->GetLastWarning;
Get last warning message (string)
=cut
sub GetLastWarning {
my $self = shift;
return $self->{_warning};
}
=head2 IsConnectError
Check connection error flag (to differentiate between network connection errors
and bad responses). Returns true value on connection error.
=cut
sub IsConnectError {
return shift->{_connerror};
}
=head2 SetEncoders
$sph->SetEncoders(\&encode_function, \&decode_function)
COMPATIBILITY NOTE: SetEncoders() was introduced in version 0.17.
Prior to that, all strings were considered to be sequences of bytes
which may have led to issues with multi-byte characters. If you were
previously encoding/decoding strings external to Sphinx::Search, you
will need to disable encoding/decoding by setting Sphinx::Search to
use raw values as explained below (or modify your code and let
Sphinx::Search do the recoding).
Set the string encoder/decoder functions for transferring strings
between perl and Sphinx. The encoder should take the perl internal
representation and convert to the bytestream that searchd expects, and
the decoder should take the bytestream returned by searchd and convert to
perl format.
The searchd format will depend on the 'charset_type' index setting in
the Sphinx configuration file.
The coders default to encode_utf8 and decode_utf8 respectively, which
are compatible with the 'utf8' charset_type.
If either the encoder or decoder functions are left undefined in the
call to SetEncoders, they return to their default values.
If you wish to send raw values (no encoding/decoding), supply a
function that simply returns its argument, e.g.
$sph->SetEncoders( sub { shift }, sub { shift });
Returns $sph.
=cut
sub SetEncoders {
my $self = shift;
my $encoder = shift;
my $decoder = shift;
$self->{_string_encoder} = $encoder ? $encoder : \&encode_utf8;
$self->{_string_decoder} = $decoder ? $decoder : \&decode_utf8;
return $self;
}
=head2 SetServer
$sph->SetServer($host, $port);
$sph->SetServer($path, $port);
In the first form, sets the host (string) and port (integer) details for the
searchd server using a network (INET) socket (default is localhost:9312).
In the second form, where $path is a local filesystem path (optionally prefixed
by 'unix://'), sets the client to access the searchd server via a local (UNIX
domain) socket at the specified path.
Returns $sph.
=cut
sub SetServer {
my $self = shift;
my $host = shift;
my $port = shift;
croak("host is not defined") unless defined($host);
if (substr($host, 0, 1) eq '/') {
$self->{_path} = $host;
return;
}
elsif (substr($host, 0, 7) eq 'unix://') {
$self->{_path} = substr($host, 7);
return;
}
$port ||= 0;
croak("port is not an number") unless $port =~ m/^\d+/o;
$port = int($port);
croak("port $port out of range 0 to 65536") if $port <0 || $port >= 65536;
$self->{_host} = $host;
$self->{_port} = $port == 0 ? 9312 : $port;
$self->{_path} = undef;
return $self;
}
=head2 SetConnectTimeout
$sph->SetConnectTimeout($timeout)
Set server connection timeout (in seconds).
Returns $sph.
=cut
sub SetConnectTimeout {
my $self = shift;
my $timeout = shift;
croak("timeout is not numeric") unless ($timeout =~ m/$num_re/);
$self->{_timeout} = $timeout;
return $self;
}
=head2 SetConnectRetries
$sph->SetConnectRetries($retries)
Set server connection retries (in case of connection fail).
Returns $sph.
=cut
sub SetConnectRetries {
my $self = shift;
my $retries = shift;
croak("connect retries is not numeric") unless ($retries =~ m/$num_re/);
$self->{connectretries} = $retries;
return $self;
}
sub _Send {
my $self = shift;
my $fp = shift;
my $data = shift;
$self->{_log}->debug("Writing to socket") if $self->{_debug};
unless ( send($fp,$data,0)){
$self->_Error("connection unexpectedly closed (timed out?): $!");
$self->{_connerror} = 1;
if ($self->{_socket}) {
close($self->{_socket});
undef $self->{_socket};
}
return 0;
}
return 1;
}
# connect to searchd server
sub _Connect {
my $self = shift;
$self->_Error(); #reset old errors in new connection
if ($self->{_socket}) {
# persistent connection, check it
return $self->{_socket} if $self->{_socket}->connected;
# force reopen
undef $self->{_socket};
}
my $debug = $self->{_debug};
my $str_dest = $self->{_path} ? 'unix://' . $self->{_path} : "$self->{_host}:$self->{_port}";
$self->{_log}->debug("Connecting to $str_dest") if $debug;
# connect socket
$self->{_connerror} = q{};
my $fp;
my %params = (); # ( Blocking => 0 );
$params{Timeout} = $self->{_timeout} if $self->{_timeout};
if ($self->{_path}) {
$fp = IO::Socket::UNIX->new( Peer => $self->{_path},
%params,
);
}
else {
$fp = IO::Socket::INET->new( PeerPort => $self->{_port},
PeerAddr => $self->{_host},
Proto => 'tcp',
%params,
);
}
if (! $fp) {
$self->_Error("Failed to open connection to $str_dest: $!");
$self->{_connerror} = 1;
return 0;
}
binmode($fp, ':bytes');
# check version
my $buf = '';
$fp->read($buf, 4) or do {
$self->_Error("Failed on initial read from $str_dest: $!");
$self->{_connerror} = 1;
return 0;
};
my $v = unpack("N*", $buf);
$v = int($v);
$self->{_log}->debug("Got version $v from searchd") if $debug;
if ($v < 1) {
close($fp);
$self->_Error("expected searchd protocol version 1+, got version '$v'");
return 0;
}
$self->{_log}->debug("Sending version") if $debug;
# All ok, send my version
unless ($self->_Send($fp, pack("N", 1))) {
$self->{_connerror} = 1;
$self->_Error("error on sending version");
return 0;
}
$self->{_log}->debug("Connection complete") if $debug;
if ($self->{_persistent}) {
my $req = pack("nnNN", SEARCHD_COMMAND_PERSIST, 0, 4, 1);
unless ($self->_Send($fp, $req)) {
$self->{_connerror} = 1;
$self->_Error("error on setting persistent connection");
return 0;
}
$self->{_socket} = $fp;
}
return $fp;
}
#-------------------------------------------------------------
# get and check response packet from searchd server
sub _GetResponse {
my $self = shift;
my $fp = shift;
my $client_ver = shift;
my $header;
my $resp = $fp->read($header, 8, 0);
if (!defined($resp) || $resp==0) {
close $self->{_socket};
undef $self->{_socket};
$self->_Error("read failed: $!");
return 0;
}
my ($status, $ver, $len ) = unpack("n2N", $header);
if ( ! defined($len) ) {
$self->_Error("read failed: $!");
return 0;
}
my $response = q{};
my $lasterror = q{};
my $lentotal = 0;
while (my $rlen = $fp->read(my $chunk, $len)) {
if ($rlen < 0) {
$lasterror = $!;
last;
}
$response .= $chunk;
$lentotal += $rlen;
last if $lentotal >= $len;
}
close($fp) unless $self->{_socket};
# check response
if ( length($response) != $len ) {
$self->_Error( $len
? "failed to read searchd response (status=$status, ver=$ver, len=$len, read=". length($response) . ", last error=$lasterror)"
: "received zero-sized searchd response");
return 0;
}
# check status
if ( $status==SEARCHD_WARNING ) {
my ($wlen) = unpack ( "N*", substr ( $response, 0, 4 ) );
$self->_Warning(substr ( $response, 4, $wlen ));
return substr ( $response, 4+$wlen );
}
if ( $status==SEARCHD_ERROR ) {
$self->_Error("searchd error: " . substr ( $response, 4 ));
return 0;
}
if ( $status==SEARCHD_RETRY ) {
$self->_Error("temporary searchd error: " . substr ( $response, 4 ));
return 0;
}
if ( $status!=SEARCHD_OK ) {
$self->_Error("unknown status code '$status'");
return 0;
}
# check version
if ( $ver<$client_ver ) {
$self->_Warning(sprintf ( "searchd command v.%d.%d older than client's v.%d.%d, some options might not work",
$ver>>8, $ver&0xff, $client_ver>>8, $client_ver&0xff ));
}
return $response;
}
#-----------------------------------------------
# connect to searchd, send request and get data
sub _ProcessRequest {
my ($self, $req, $response_command_version) = @_;
return unless $req;
my $tries = $self->{_connectretries} + 1;
while( $tries-- ) {
my $fp = $self->_Connect;
if (! $fp) {
next if $self->IsConnectError;
last;
}
$self->_Send($fp, $req) or next;
my $response = $self->_GetResponse ($fp, $response_command_version);
return $response if $response;
}
$self->_Error($self->GetLastError . "... ConnectRetries exceed...") if $self->IsConnectError;
return 0;
}
=head2 SetLimits
$sph->SetLimits($offset, $limit);
$sph->SetLimits($offset, $limit, $max);
Set match offset/limits, and optionally the max number of matches to return.
Returns $sph.
=cut
sub SetLimits {
my $self = shift;
my $offset = shift;
my $limit = shift;
my $max = shift || 0;
croak("offset should be an integer >= 0") unless ($offset =~ /^\d+$/ && $offset >= 0) ;
croak("limit should be an integer >= 0") unless ($limit =~ /^\d+$/ && $limit >= 0);
$self->{_offset} = $offset;
$self->{_limit} = $limit;
if($max > 0) {
$self->{_maxmatches} = $max;
}
return $self;
}
=head2 SetMaxQueryTime
$sph->SetMaxQueryTime($millisec);
Set maximum query time, in milliseconds, per index.
The value may not be negative; 0 means "do not limit".
Returns $sph.
=cut
sub SetMaxQueryTime {
my $self = shift;
my $max = shift;
croak("max value should be an integer >= 0") unless ($max =~ /^\d+$/ && $max >= 0) ;
$self->{_maxquerytime} = $max;
return $self;
}
=head2 SetMatchMode
** DEPRECATED **
$sph->SetMatchMode($mode);
Set match mode, which may be one of:
=over 4
=item * SPH_MATCH_ALL
Match all words
=item * SPH_MATCH_ANY
Match any words
=item * SPH_MATCH_PHRASE
Exact phrase match
=item * SPH_MATCH_BOOLEAN
Boolean match, using AND (&), OR (|), NOT (!,-) and parenthetic grouping.
=item * SPH_MATCH_EXTENDED
Extended match, which includes the Boolean syntax plus field, phrase and
proximity operators.
=back
Returns $sph.
=cut
sub SetMatchMode {
my $self = shift;
my $mode = shift;
warn "SetMatchMode is DEPRECATED. Do not call this method - use extended query syntax instead.";
croak("Match mode not defined") unless defined($mode);
croak("Unknown matchmode: $mode") unless ( $mode==SPH_MATCH_ALL
|| $mode==SPH_MATCH_ANY
|| $mode==SPH_MATCH_PHRASE
|| $mode==SPH_MATCH_BOOLEAN
|| $mode==SPH_MATCH_EXTENDED
|| $mode==SPH_MATCH_FULLSCAN
|| $mode==SPH_MATCH_EXTENDED2 );
$self->{_mode} = $mode;
return $self;
}
=head2 SetRankingMode
$sph->SetRankingMode(SPH_RANK_BM25, $rank_exp);
Set ranking mode, which may be one of:
=over 4
=item * SPH_RANK_PROXIMITY_BM25
Default mode, phrase proximity major factor and BM25 minor one
=item * SPH_RANK_BM25
Statistical mode, BM25 ranking only (faster but worse quality)
=item * SPH_RANK_NONE
No ranking, all matches get a weight of 1
=item * SPH_RANK_WORDCOUNT
Simple word-count weighting, rank is a weighted sum of per-field keyword
occurence counts
=item * SPH_RANK_MATCHANY
Returns rank as it was computed in SPH_MATCH_ANY mode earlier, and is internally used to emulate SPH_MATCH_ANY queries.
=item * SPH_RANK_FIELDMASK
Returns a 32-bit mask with N-th bit corresponding to N-th fulltext field, numbering from 0. The bit will only be set when the respective field has any keyword occurences satisfiying the query.
=item * SPH_RANK_SPH04
SPH_RANK_SPH04 is generally based on the default SPH_RANK_PROXIMITY_BM25 ranker, but additionally boosts the matches when they occur in the very beginning or the very end of a text field.
=item * SPH_RANK_EXPR
Allows the ranking formula to be specified at run time. It exposes a number of internal text factors and lets you define how the final weight should be computed from those factors. $rank_exp should be set to the ranking expression string, e.g. to emulate SPH_RANK_PROXIMITY_BM25, use "sum(lcs*user_weight)*1000+bm25".
=back
Returns $sph.
=cut
sub SetRankingMode {
my $self = shift;
my $ranker = shift;
my $rankexp = shift;
croak("Unknown ranking mode: $ranker") unless ( $ranker == 0
|| ( $ranker >= 1
&& $ranker < SPH_RANK_TOTAL ));
$self->{_ranker} = $ranker;
$self->{_rankexpr} = $rankexp || "";
return $self;
}
=head2 SetSortMode
$sph->SetSortMode(SPH_SORT_RELEVANCE);
$sph->SetSortMode($mode, $sortby);
Set sort mode, which may be any of:
=over 4
=item SPH_SORT_RELEVANCE - sort by relevance
=item SPH_SORT_ATTR_DESC, SPH_SORT_ATTR_ASC
Sort by attribute descending/ascending. $sortby specifies the sorting attribute.
=item SPH_SORT_TIME_SEGMENTS
Sort by time segments (last hour/day/week/month) in descending order, and then
by relevance in descending order. $sortby specifies the time attribute.
=item SPH_SORT_EXTENDED
Sort by SQL-like syntax. $sortby is the sorting specification.
=item SPH_SORT_EXPR
=back
Returns $sph.
=cut
sub SetSortMode {
my $self = shift;
my $mode = shift;
my $sortby = shift || "";
croak("Sort mode not defined") unless defined($mode);
croak("Unknown sort mode: $mode") unless ( $mode == SPH_SORT_RELEVANCE
|| $mode == SPH_SORT_ATTR_DESC
|| $mode == SPH_SORT_ATTR_ASC
|| $mode == SPH_SORT_TIME_SEGMENTS
|| $mode == SPH_SORT_EXTENDED
|| $mode == SPH_SORT_EXPR
);
croak("Sortby must be defined") unless ($mode==SPH_SORT_RELEVANCE || length($sortby));
$self->{_sort} = $mode;
$self->{_sortby} = $sortby;
return $self;
}
=head2 SetWeights
** DEPRECATED **
$sph->SetWeights([ 1, 2, 3, 4]);
This method is deprecated. Use L instead.
Set per-field (integer) weights. The ordering of the weights correspond to the
ordering of fields as indexed.
Returns $sph.
=cut
sub SetWeights {
my $self = shift;
my $weights = shift;
warn "SetWeights is DEPRECATED, Do not call this method; use SetFieldWeights instead";
croak("Weights is not an array reference") unless (ref($weights) eq 'ARRAY');
foreach my $weight (@$weights) {
croak("Weight: $weight is not an integer") unless ($weight =~ /^\d+$/);
}
$self->{_weights} = $weights;
return $self;
}
=head2 SetFieldWeights
$sph->SetFieldWeights(\%weights);
Set per-field (integer) weights by field name. The weights hash provides field
name to weight mappings.
Takes precedence over L.
Unknown names will be silently ignored. Missing fields will be given a weight of 1.
Returns $sph.
=cut
sub SetFieldWeights {
my $self = shift;
my $weights = shift;
croak("Weights is not a hash reference") unless (ref($weights) eq 'HASH');
foreach my $field (keys %$weights) {
croak("Weight: $weights->{$field} is not an integer >= 0") unless ($weights->{$field} =~ /^\d+$/);
}
$self->{_fieldweights} = $weights;
return $self;
}
=head2 SetIndexWeights
$sph->SetIndexWeights(\%weights);
Set per-index (integer) weights. The weights hash is a mapping of index name to integer weight.
Returns $sph.
=cut
sub SetIndexWeights {
my $self = shift;
my $weights = shift;
croak("Weights is not a hash reference") unless (ref($weights) eq 'HASH');
foreach (keys %$weights) {
croak("IndexWeight $_: $weights->{$_} is not an integer") unless ($weights->{$_} =~ /^\d+$/);
}
$self->{_indexweights} = $weights;
return $self;
}
=head2 SetIDRange
$sph->SetIDRange($min, $max);
Set IDs range only match those records where document ID
is between $min and $max (including $min and $max)
Returns $sph.
=cut
sub SetIDRange {
my $self = shift;
my $min = shift;
my $max = shift;
croak("min_id is not numeric") unless ($min =~ m/$num_re/);
croak("max_id is not numeric") unless ($max =~ m/$num_re/);
croak("min_id is larger than or equal to max_id") unless ($min < $max);
$self->{_min_id} = $min;
$self->{_max_id} = $max;
return $self;
}
=head2 SetFilter
$sph->SetFilter($attr, \@values);
$sph->SetFilter($attr, \@values, $exclude);
Sets the results to be filtered on the given attribute. Only results which have
attributes matching the given values will be returned. (Attribute values must be integers).
This may be called multiple times with different attributes to select on
multiple attributes.
If 'exclude' is set, excludes results that match the filter.
Returns $sph.
=cut
sub SetFilter {
my ($self, $attribute, $values, $exclude) = @_;
croak("attribute is not defined") unless (defined $attribute);
croak("values is not an array reference") unless (ref($values) eq 'ARRAY');
croak("values reference is empty") unless (scalar(@$values));
push(@{$self->{_filters}}, {
type => SPH_FILTER_VALUES,
attr => $attribute,
values => $values,
exclude => $exclude ? 1 : 0,
});
return $self;
}
=head2 SetFilterString
$sph->SetFilterString($attr, $value)
$sph->SetFilterString($attr, $value, $exclude)
Adds new string value filter. Only those documents where $attr column value matches the string value from $value will be matched (or rejected, if $exclude is true).
=cut
sub SetFilterString {
my ($self, $attribute, $value, $exclude) = @_;
croak("attribute is not defined") unless (defined $attribute);
croak("value is not a string") unless ($value && ! ref($value));
push(@{$self->{_filters}}, {
type => SPH_FILTER_STRING,
attr => $attribute,
value => $value,
exclude => $exclude ? 1 : 0,
});
return $self;
}
=head2 SetFilterRange
$sph->SetFilterRange($attr, $min, $max);
$sph->SetFilterRange($attr, $min, $max, $exclude);
Sets the results to be filtered on a range of values for the given
attribute. Only those records where $attr column value is between $min and $max
(including $min and $max) will be returned.
If 'exclude' is set, excludes results that fall within the given range.
Returns $sph.
=cut
sub SetFilterRange {
my ($self, $attribute, $min, $max, $exclude) = @_;
croak("attribute is not defined") unless (defined $attribute);
croak("min: $min is not an integer") unless ($min =~ m/$num_re/);
croak("max: $max is not an integer") unless ($max =~ m/$num_re/);
croak("min value should be <= max") unless ($min <= $max);
push(@{$self->{_filters}}, {
type => SPH_FILTER_RANGE,
attr => $attribute,
min => $min,
max => $max,
exclude => $exclude ? 1 : 0,
});
return $self;
}
=head2 SetFilterFloatRange
$sph->SetFilterFloatRange($attr, $min, $max, $exclude);
Same as L, but allows floating point values.
Returns $sph.
=cut
sub SetFilterFloatRange {
my ($self, $attribute, $min, $max, $exclude) = @_;
croak("attribute is not defined") unless (defined $attribute);
croak("min: $min is not numeric") unless ($min =~ m/$num_re/);
croak("max: $max is not numeric") unless ($max =~ m/$num_re/);
croak("min value should be <= max") unless ($min <= $max);
push(@{$self->{_filters}}, {
type => SPH_FILTER_FLOATRANGE,
attr => $attribute,
min => $min,
max => $max,
exclude => $exclude ? 1 : 0,
});
return $self;
}
=head2 SetGeoAnchor
$sph->SetGeoAnchor($attrlat, $attrlong, $lat, $long);
Setup anchor point for using geosphere distance calculations in filters and sorting.
Distance will be computed with respect to this point
=over 4
=item $attrlat is the name of latitude attribute
=item $attrlong is the name of longitude attribute
=item $lat is anchor point latitude, in radians
=item $long is anchor point longitude, in radians
=back
Returns $sph.
=cut
sub SetGeoAnchor {
my ($self, $attrlat, $attrlong, $lat, $long) = @_;
croak("attrlat is not defined") unless defined $attrlat;
croak("attrlong is not defined") unless defined $attrlong;
croak("lat: $lat is not numeric") unless ($lat =~ m/$num_re/);
croak("long: $long is not numeric") unless ($long =~ m/$num_re/);
$self->{_anchor} = {
attrlat => $attrlat,
attrlong => $attrlong,
lat => $lat,
long => $long,
};
return $self;
}
=head2 SetGroupBy
$sph->SetGroupBy($attr, $func);
$sph->SetGroupBy($attr, $func, $groupsort);
Sets attribute and function of results grouping.
In grouping mode, all matches are assigned to different groups based on grouping
function value. Each group keeps track of the total match count, and the best
match (in this group) according to current sorting function. The final result
set contains one best match per group, with grouping function value and matches
count attached.
$attr is any valid attribute. Use L to disable grouping.
$func is one of:
=over 4
=item * SPH_GROUPBY_DAY
Group by day (assumes timestamp type attribute of form YYYYMMDD)
=item * SPH_GROUPBY_WEEK
Group by week (assumes timestamp type attribute of form YYYYNNN)
=item * SPH_GROUPBY_MONTH
Group by month (assumes timestamp type attribute of form YYYYMM)
=item * SPH_GROUPBY_YEAR
Group by year (assumes timestamp type attribute of form YYYY)
=item * SPH_GROUPBY_ATTR
Group by attribute value
=item * SPH_GROUPBY_ATTRPAIR
Group by two attributes, being the given attribute and the attribute that
immediately follows it in the sequence of indexed attributes. The specified
attribute may therefore not be the last of the indexed attributes.
=back
Groups in the set of results can be sorted by any SQL-like sorting clause,
including both document attributes and the following special internal Sphinx
attributes:
=over 4
=item @id - document ID;
=item @weight, @rank, @relevance - match weight;
=item @group - group by function value;
=item @count - number of matches in group.
=back
The default mode is to sort by groupby value in descending order,
ie. by "@group desc".
In the results set, "total_found" contains the total amount of matching groups
over the whole index.
WARNING: grouping is done in fixed memory and thus its results
are only approximate; so there might be more groups reported
in total_found than actually present. @count might also
be underestimated.
For example, if sorting by relevance and grouping by a "published"
attribute with SPH_GROUPBY_DAY function, then the result set will
contain only the most relevant match for each day when there were any
matches published, with day number and per-day match count attached,
and sorted by day number in descending order (ie. recent days first).
=cut
sub SetGroupBy {
my $self = shift;
my $attribute = shift;
my $func = shift;
my $groupsort = shift || '@group desc';
croak("attribute is not defined") unless (defined $attribute);
croak("Unknown grouping function: $func") unless ($func==SPH_GROUPBY_DAY
|| $func==SPH_GROUPBY_WEEK
|| $func==SPH_GROUPBY_MONTH
|| $func==SPH_GROUPBY_YEAR
|| $func==SPH_GROUPBY_ATTR
|| $func==SPH_GROUPBY_ATTRPAIR
);
$self->{_groupby} = $attribute;
$self->{_groupfunc} = $func;
$self->{_groupsort} = $groupsort;
return $self;
}
=head2 SetGroupDistinct
$sph->SetGroupDistinct($attr);
Set count-distinct attribute for group-by queries
=cut
sub SetGroupDistinct {
my $self = shift;
my $attribute = shift;
croak("attribute is not defined") unless (defined $attribute);
$self->{_groupdistinct} = $attribute;
return $self;
}
=head2 SetRetries
$sph->SetRetries($count, $delay);
Set distributed retries count and delay
=cut
sub SetRetries {
my $self = shift;
my $count = shift;
my $delay = shift || 0;
croak("count: $count is not an integer >= 0") unless ($count =~ /^\d+$/o && $count >= 0);
croak("delay: $delay is not an integer >= 0") unless ($delay =~ /^\d+$/o && $delay >= 0);
$self->{_retrycount} = $count;
$self->{_retrydelay} = $delay;
return $self;
}
=head2 SetOverride
** DEPRECATED **
$sph->SetOverride($attrname, $attrtype, $values);
Set attribute values override. There can be only one override per attribute.
$values must be a hash that maps document IDs to attribute values
=cut
sub SetOverride {
my $self = shift;
my $attrname = shift;
my $attrtype = shift;
my $values = shift;
die "SetOverride is DEPRECATED. Do not call this method.";
croak("attribute name is not defined") unless defined $attrname;
croak("Uknown attribute type: $attrtype") unless ($attrtype == SPH_ATTR_INTEGER
|| $attrtype == SPH_ATTR_TIMESTAMP
|| $attrtype == SPH_ATTR_BOOL
|| $attrtype == SPH_ATTR_FLOAT
|| $attrtype == SPH_ATTR_BIGINT);
$self->{_overrides}->{$attrname} = { attr => $attrname,
type => $attrtype,
values => $values,
};
return $self;
}
=head2 SetSelect
$sph->SetSelect($select)
Set select list (attributes or expressions). SQL-like syntax.
=cut
sub SetSelect {
my $self = shift;
$self->{_select} = shift;
return $self;
}
=head2 SetQueryFlag
$sph->SetQueryFlag($flag_name, $flag_value);
=cut
sub SetQueryFlag {
my ($self, $flag_name, $flag_value) = @_;
croak("Unknown flag $flag_name") unless exists $query_flags{$flag_name};
croak("Unknown or illegal flag value ($flag_value) for '$flag_name'") unless
(any { $_ eq $flag_value } @{$query_flags{$flag_name}})
|| ($flag_name eq 'max_predicted_time' && $flag_value =~ m/^\d+$/);
if ($flag_name eq SPH_QF_REVERSE_SCAN) {
$self->{_query_flags} = $self->_sphSetBit( $self->{_query_flags}, 0, $flag_value == 1);
}
elsif ($flag_name eq SPH_QF_SORT_METHOD) {
$self->{_query_flags} = $self->_sphSetBit( $self->{_query_flags}, 1, $flag_value == "kbuffer");
}
elsif ($flag_name eq SPH_QF_MAX_PREDICTED_TIME) {
$self->{_query_flags} = $self->_sphSetBit( $self->{_query_flags}, 2, $flag_value > 0);
$self->{_predictedtime} = $flag_value;
}
elsif ($flag_name eq SPH_QF_BOOLEAN_SIMPLIFY) {
$self->{_query_flags} = $self->_sphSetBit( $self->{_query_flags}, 3, $flag_value);
}
elsif ($flag_name eq SPH_QF_IDF) {
if ($flag_value eq 'normalized' || $flag_value eq 'plain') {
$self->{_query_flags} = $self->_sphSetBit( $self->{_query_flags}, 4, $flag_value eq 'normalized');
}
else { # must be tfidf_normalized or tfidf_unnormalized
$self->{_query_flags} = $self->_sphSetBit( $self->{_query_flags}, 6, $flag_value eq 'tfidf_normalized');
}
}
elsif ($flag_name eq SPH_QF_GLOBAL_IDF) {
$self->{_query_flags} = $self->_sphSetBit( $self->{_query_flags}, 5, $flag_value);
}
return $self;
}
=head2 SetOuterSelect
$sph->SetOuterSelect($orderby, $offset, $limit)
=cut
sub SetOuterSelect {
my ($self, $orderby, $offset, $limit) = @_;
croak("orderby must be a string") unless $orderby && ! ref($orderby);
croak("offset and limit must be integers > 0") unless $offset =~ m/^\d+$/ && $limit =~ m/^\d+$/;
$self->{_outerorderby} = $orderby;
$self->{_outeroffsetlimit} = $offset;
$self->{_outerlimit} = $limit;
$self->{_hasouter} = 1;
return $self;
}
=head2 ResetFilters
$sph->ResetFilters;
Clear all filters.
=cut
sub ResetFilters {
my $self = shift;
$self->{_filters} = [];
$self->{_anchor} = undef;
return $self;
}
=head2 ResetGroupBy
$sph->ResetGroupBy;
Clear all group-by settings (for multi-queries)
=cut
sub ResetGroupBy {
my $self = shift;
$self->{_groupby} = "";
$self->{_groupfunc} = SPH_GROUPBY_DAY;
$self->{_groupsort} = '@group desc';
$self->{_groupdistinct} = "";
return $self;
}
=head2 ResetOverrides
Clear all attribute value overrides (for multi-queries)
=cut
sub ResetOverrides {
my $self = shift;
$self->{_select} = undef;
return $self;
}
=head2 ResetQueryFlag
Clear all query flags.
=cut
sub ResetQueryFlag {
my $self = shift;
$self->{_query_flags} = $self->_sphSetBit(0, 6, 1);
$self->{_predictedtime} = 0;
return $self;
}
=head2 ResetOuterSelect
Clear all outer select settings.
=cut
sub ResetOuterSelect {
my $self = shift;
$self->{_outerorderby} = '';
$self->{_outeroffset} = 0;
$self->{_outerlimit} = 0;
$self->{_hasouter} = 0;
return $self;
}
=head2 Query
$results = $sph->Query($query, $index);
Connect to searchd server and run given search query.
=over 4
=item query is query string
=item index is index name to query, default is "*" which means to query all indexes. Use a space or comma separated list to search multiple indexes.
=back
Returns undef on failure
Returns hash which has the following keys on success:
=over 4
=item matches
Array containing hashes with found documents ( "doc", "weight", "group", "stamp" )
=item total
Total amount of matches retrieved (upto SPH_MAX_MATCHES, see sphinx.h)
=item total_found
Total amount of matching documents in index
=item time
Search time
=item words
Hash which maps query terms (stemmed!) to ( "docs", "hits" ) hash
=back
Returns the results array on success, undef on error.
=cut
sub Query {
my $self = shift;
my $query = shift;
my $index = shift || '*';
my $comment = shift || '';
croak("_reqs is not empty") unless @{$self->{_reqs}} == 0;
$self->AddQuery($query, $index, $comment);
my $results = $self->RunQueries or return;
$self->_Error($results->[0]->{error}) if $results->[0]->{error};
$self->_Warning($results->[0]->{warning}) if $results->[0]->{warning};
return if $results->[0]->{status} && $results->[0]->{status} == SEARCHD_ERROR;
return $results->[0];
}
# helper to pack floats in network byte order
sub _PackFloat {
my $f = shift;
my $t1 = pack ( "f", $f ); # machine order
my $t2 = unpack ( "L*", $t1 ); # int in machine order
return pack ( "N", $t2 );
}
=head2 AddQuery
$sph->AddQuery($query, $index);
Add a query to a batch request.
Batch queries enable searchd to perform internal optimizations,
if possible; and reduce network connection overheads in all cases.
For instance, running exactly the same query with different
groupby settings will enable searched to perform expensive
full-text search and ranking operation only once, but compute
multiple groupby results from its output.
Parameters are exactly the same as in Query() call.
Returns corresponding index to the results array returned by RunQueries() call.
=cut
sub AddQuery {
my $self = shift;
my $query = shift;
my $index = shift || '*';
my $comment = shift || '';
##################
# build request
##################
my $req;
$req = pack ( "NNNNN", $self->{_query_flags}, $self->{_offset}, $self->{_limit}, $self->{_mode}, $self->{_ranker}); # mode and limits
if ($self->{_ranker} == SPH_RANK_EXPR) {
$req .= pack ( "N/a*", $self->{_rankexpr});
}
$req .= pack ( "N", $self->{_sort} ); # (deprecated) sort mode
$req .= pack ( "N/a*", $self->{_sortby});
$req .= pack ( "N/a*", $self->{_string_encoder}->($query) ); # query itself
$req .= pack ( "N*", scalar(@{$self->{_weights}}), @{$self->{_weights}});
$req .= pack ( "N/a*", $index); # indexes
$req .= pack ( "N", 1)
. $self->_sphPackU64($self->{_min_id})
. $self->_sphPackU64($self->{_max_id}); # id64 range
# filters
$req .= pack ( "N", scalar @{$self->{_filters}} );
foreach my $filter (@{$self->{_filters}}) {
$req .= pack ( "N/a*", $filter->{attr});
$req .= pack ( "N", $filter->{type});
my $t = $filter->{type};
if ($t == SPH_FILTER_VALUES) {
$req .= $self->_sphPackI64array($filter->{values});
}
elsif ($t == SPH_FILTER_RANGE) {
$req .= $self->_sphPackI64($filter->{min}) . $self->_sphPackI64($filter->{max});
}
elsif ($t == SPH_FILTER_FLOATRANGE) {
$req .= _PackFloat ( $filter->{"min"} ) . _PackFloat ( $filter->{"max"} );
}
elsif ($t == SPH_FILTER_STRING) {
$req .= pack ( "N/a*", $filter->{value});
}
else {
croak("Unhandled filter type $t");
}
$req .= pack ( "N", $filter->{exclude});
}
# group-by clause, max-matches count, group-sort clause, cutoff count
$req .= pack ( "NN/a*", $self->{_groupfunc}, $self->{_groupby} );
$req .= pack ( "N", $self->{_maxmatches} );
$req .= pack ( "N/a*", $self->{_groupsort});
$req .= pack ( "NNN", $self->{_cutoff}, $self->{_retrycount}, $self->{_retrydelay} );
$req .= pack ( "N/a*", $self->{_groupdistinct});
if (!defined $self->{_anchor}) {
$req .= pack ( "N", 0);
}
else {
my $a = $self->{_anchor};
$req .= pack ( "N", 1);
$req .= pack ( "N/a*", $a->{attrlat});
$req .= pack ( "N/a*", $a->{attrlong});
$req .= _PackFloat($a->{lat}) . _PackFloat($a->{long});
}
# per-index weights
$req .= pack( "N", scalar keys %{$self->{_indexweights}});
$req .= pack ( "N/a*N", $_, $self->{_indexweights}->{$_} ) for keys %{$self->{_indexweights}};
# max query time
$req .= pack ( "N", $self->{_maxquerytime} );
# per-field weights
$req .= pack ( "N", scalar keys %{$self->{_fieldweights}} );
$req .= pack ( "N/a*N", $_, $self->{_fieldweights}->{$_}) for keys %{$self->{_fieldweights}};
# comment
$req .= pack ( "N/a*", $comment);
# attribute overrides
$req .= pack ( "N", scalar keys %{$self->{_overrides}} );
for my $entry (values %{$self->{_overrides}}) {
$req .= pack ("N/a*", $entry->{attr})
. pack ("NN", $entry->{type}, scalar keys %{$entry->{values}});
for my $id (keys %{$entry->{values}}) {
croak "Attribute value key is not numeric" unless $id =~ m/$num_re/;
my $v = $entry->{values}->{$id};
croak "Attribute value key is not numeric" unless $v =~ m/$num_re/;
$req .= $self->_sphPackU64($id);
if ($entry->{type} == SPH_ATTR_FLOAT) {
$req .= $self->_packfloat($v);
}
elsif ($entry->{type} == SPH_ATTR_BIGINT) {
$req .= $self->_sphPackI64($v);
}
else {
$req .= pack("N", $v);
}
}
}
# select list
$req .= pack("N/a*", $self->{_select} || '');
# max_predicted_time
if ($self->{_predictedtime} > 0) {
$req .= pack ( "N", $self->{_predictedtime} );
}
$req .= pack ( "N/a*", $self->{_outerorderby});
$req .= pack ( "NN", $self->{_outeroffset}, $self->{_outerlimit} );
$req .= pack ("N", $self->{_hasouter} ? 1 : 0 );
push(@{$self->{_reqs}}, $req);
return scalar $#{$self->{_reqs}};
}
=head2 RunQueries
$sph->RunQueries
Run batch of queries, as added by AddQuery.
Returns undef on network IO failure.
Returns an array of result sets on success.
Each result set in the returned array is a hash which contains
the same keys as the hash returned by L, plus:
=over 4
=item * error
Errors, if any, for this query.
=item * warning
Any warnings associated with the query.
=back
=cut
sub RunQueries {
my $self = shift;
unless (@{$self->{_reqs}}) {
$self->_Error("no queries defined, issue AddQuery() first");
return;
}
##################
# send query, get response
##################
my $nreqs = @{$self->{_reqs}};
my $req = pack("NNa*", 0, $nreqs, join("", @{$self->{_reqs}}));
$req = pack ( "nnN/a*", SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, $req); # add header
my $response = $self->_ProcessRequest($req, VER_COMMAND_SEARCH);
$self->{_reqs} = [];
return unless $response;
##################
# parse response
##################
my $p = 0;
my $max = length($response); # Protection from broken response
my @results;
for (my $ires = 0; $ires < $nreqs; $ires++) {
my $result = {}; # Empty hash ref
push(@results, $result);
$result->{matches} = []; # Empty array ref
$result->{error} = "";
$result->{warning} = "";
# extract status
my $status = unpack("N", substr ( $response, $p, 4 ) ); $p += 4;
if ($status != SEARCHD_OK) {
my $len = unpack("N", substr ( $response, $p, 4 ) ); $p += 4;
my $message = substr ( $response, $p, $len ); $p += $len;
if ($status == SEARCHD_WARNING) {
$result->{warning} = $message;
}
else {
$result->{error} = $message;
next;
}
}
# read schema
my @fields;
my (%attrs, @attr_list);
my $nfields = unpack ( "N", substr ( $response, $p, 4 ) ); $p += 4;
while ( $nfields-->0 && $p<$max ) {
my $len = unpack ( "N", substr ( $response, $p, 4 ) ); $p += 4;
push(@fields, substr ( $response, $p, $len )); $p += $len;
}
$result->{"fields"} = \@fields;
my $nattrs = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4;
while ( $nattrs-->0 && $p<$max ) {
my $len = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4;
my $attr = substr ( $response, $p, $len ); $p += $len;
my $type = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4;
$attrs{$attr} = $type;
push(@attr_list, $attr);
}
$result->{"attrs"} = \%attrs;
# read match count
my $count = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4;
my $id64 = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4;
# read matches
while ( $count-->0 && $p<$max ) {
my $data = {};
if ($id64) {
$data->{doc} = $self->_sphUnpackU64(substr($response, $p, 8)); $p += 8;
$data->{weight} = unpack("N*", substr($response, $p, 4)); $p += 4;
}
else {
( $data->{doc}, $data->{weight} ) = unpack("N*N*", substr($response,$p,8));
$p += 8;
}
foreach my $attr (@attr_list) {
if ($attrs{$attr} == SPH_ATTR_BIGINT) {
$data->{$attr} = $self->_sphUnpackI64(substr($response, $p, 8)); $p += 8;
next;
}
if ($attrs{$attr} == SPH_ATTR_FLOAT) {
my $uval = unpack( "N*", substr ( $response, $p, 4 ) ); $p += 4;
$data->{$attr} = [ unpack("f*", pack("L", $uval)) ];
next;
}
my $val = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4;
if ($attrs{$attr} == SPH_ATTR_MULTI) {
my $nvalues = $val;
$data->{$attr} = [];
while ($nvalues-- > 0 && $p < $max) {
$val = unpack( "N*", substr ( $response, $p, 4 ) ); $p += 4;
push(@{$data->{$attr}}, $val);
}
}
elsif ($attrs{$attr} == SPH_ATTR_MULTI64) {
my $nvalues = $val;
$data->{$attr} = [];
while ($nvalues > 0 && $p < $max) {
$val = unpack( "q*", substr ( $response, $p, 8 ) ); $p += 8;
push(@{$data->{$attr}}, $val);
$nvalues -= 2;
}
}
elsif ($attrs{$attr} == SPH_ATTR_STRING) {
$data->{$attr} = $self->{_string_decoder}->(substr ($response, $p, $val));
$p += $val;
}
elsif ($attrs{$attr} == SPH_ATTR_FACTORS) {
$data->{$attr} = $self->{_string_decoder}->(substr ($response, $p, $val - 4));
$p += $val - 4;
}
else {
$data->{$attr} = $val;
}
}
push(@{$result->{matches}}, $data);
}
my $words;
($result->{total}, $result->{total_found}, $result->{time}, $words) = unpack("N*N*N*N*", substr($response, $p, 16));
$result->{time} = sprintf ( "%.3f", $result->{"time"}/1000 );
$p += 16;
while ( $words-->0 && $p < $max) {
my $len = unpack ( "N*", substr ( $response, $p, 4 ) );
$p += 4;
my $word = $self->{_string_decoder}->( substr ( $response, $p, $len ) );
$p += $len;
my ($docs, $hits) = unpack ("N*N*", substr($response, $p, 8));
$p += 8;
$result->{words}{$word} = {
"docs" => $docs,
"hits" => $hits
};
}
}
return \@results;
}
=head2 SphinxQL
my $results = $sph->SphinxQL($sphinxql_query);
This is an alternative implementation of the SphinxQL API to the DBI option. Frankly, it was an experiment, and the DBI driver proved to have much better performance. Whilst this may be useful to some, in general if you are considering using this method then you should probably look at connecting directly via DBI instead.
Results are return in a hash containing an array of 'columns' and 'rows' and possibly a warning count. If a server-side error occurs, the hash contains the 'error' field. If a communication error occurs, the return value will be undefined. In either error case, GetLastError will return the error.
=cut
sub SphinxQL {
my ($self, $ql) = @_;
my $req = pack ("N/a*", $self->{_string_encoder}->($ql));
$req = pack ( "nnN/a*", SEARCHD_COMMAND_SPHINXQL, VER_COMMAND_SPHINXQL, $req); # add header
my $response = $self->_ProcessRequest($req, VER_COMMAND_SPHINXQL);
return if ! $response;
my ($result) = $self->_mysql_unpack_table($response, 0, length($response));
return $result;
}
sub _mysql_unpack_header {
my ($self, $response, $p) = @_;
my $max = length($response);
return (undef, undef, $p, 1, undef, undef) if $p == $max;
$self->_Throw("Decode pointer ($p) beyond end of buffer ($max)") if $p > $max;
my $header = unpack ( "L<", substr( $response, $p, 4 ) ); $p += 4; # packet ID << 24 | packet length
my $packet_id = $header >> 24;
my $packet_len = $header & 0x00FFFFFF;
return ($packet_id, $packet_len, $p, 1, undef, undef) if $packet_len == 0;
# Peek for EOF or error
my ($is_eof, $warns, $more_results);
my $peek_byte = unpack("C", substr($response, $p, 1));
if ($packet_len == 5 && $peek_byte == 0xfe) { # EOF
$is_eof = 1;
$p += 1; # 0xFE
my $warns = unpack ( "L<", substr( $response, $p, 4 ) ); $p += 4;
my $more_results = (($warns >> 16) & 8) != 0; # SPH_MYSQL_FLAG_MORE_RESULTS = 8
$warns &= 0xFFFF;
}
elsif ($peek_byte == 0xff) {
$p += 1; # 0xFF
my $error_code = unpack ( "S<", substr( $response, $p, 2 ) ); $p += 2;
my $error_code2 = unpack("a*", substr($response, $p, 6)); $p += 6;
my $len = $packet_len - 9;
my $err_msg = $self->{_string_decoder}->(unpack("a*", substr($response, $p, $len))); $p += $len;
$self->_Throw("Error Code $error_code: $error_code2 $err_msg");
}
return ($packet_id, $packet_len, $p, $is_eof, $warns, $more_results);
}
sub _mysql_unpack_table {
my ($self, $response, $p, $max) = @_;
my ($columns, $warns, $more_results, $rows);
eval {
($columns, $warns, $more_results, $p) = $self->_mysql_unpack_table_header($response, $p);
($rows, $p, $warns, $more_results) = $self->_mysql_unpack_table_rows($response, $p, $max, $columns);
};
if (my $e = $@) {
$self->_Error($e);
return ({ error => $e }, $p);
}
my %result = (columns => $columns,
rows => $rows,
warnings => $warns,
);
return (\%result, $p);
}
sub _mysql_unpack_table_header {
my ($self, $response, $p) = @_;
my @columns;
# table header begin
my ($packet_id, $packet_len, $is_eof, $warns, $more_results);
($packet_id, $packet_len, $p, $is_eof, $warns, $more_results) = $self->_mysql_unpack_header($response, $p);
if (! $is_eof) {
my $ncols;
($ncols, $p) = $self->_mysql_unpack_varint($response, $p);
# column info
for my $i (0 .. $ncols - 1) {
($columns[$i], $p) = $self->_mysql_unpack_field_packet($response, $p);
}
# table header end
($packet_id, $packet_len, $p, $is_eof, $warns, $more_results) = $self->_mysql_unpack_header($response, $p);
}
return (\@columns, $warns, $more_results, $p);
}
sub _mysql_unpack_table_rows {
my ($self, $response, $p, $max, $columns) = @_;
my @rows;
my ($warns, $more_results);
while ($p < $max) {
my $row;
($row, $p, $warns, $more_results) = $self->_mysql_unpack_table_row($response, $p, $columns);
if ($row) {
push(@rows, $row);
}
else {
last;
}
}
return (\@rows, $p, $warns, $more_results);
}
sub _mysql_unpack_table_row {
my ($self, $response, $p, $columns) = @_;
my ($packet_id, $packet_len, $is_eof, $warns, $more_results);
($packet_id, $packet_len, $p, $is_eof, $warns, $more_results) = $self->_mysql_unpack_header($response, $p);
if ($is_eof) {
return(undef, $p, $warns, $more_results);
}
my @row;
for my $col (@$columns) {
my $val;
if ($col->{column_type} == MYSQL_COL_STRING) {
($val, $p) = $self->_mysql_unpack_string($response, $p, 1);
}
else {
($val, $p) = $self->_mysql_unpack_string($response, $p);
}
push(@row, $val);
}
return (\@row, $p);
}
sub _mysql_unpack_varint {
my ($self, $response, $p) = @_;
my $prefix = unpack("C", substr($response, $p, 1)); $p += 1;
if ($prefix < 251) {
return ($prefix, $p);
}
if ($prefix == 0xFC) {
$prefix = unpack("S<", substr($response, $p, 2)); $p += 2;
return ($prefix, $p);
}
if ($prefix == 0xFC) {
$prefix = unpack("S<", substr($response, $p, 2)); $p += 2;
$prefix += unpack("C", substr($response, $p, 1)) << 16; $p += 1;
return ($prefix, $p);
}
if ($prefix == 0xFE) {
$prefix = unpack("L<", substr($response, $p, 4)); $p += 4;
}
$p += 4; # discard 4 null bytes
return ($prefix, $p);
}
sub _mysql_unpack_string {
my ($self, $response, $p, $decode) = @_;
my $len;
($len, $p) = $self->_mysql_unpack_varint($response, $p);
my $s = substr($response, $p, $len); $p += $len;
$s = $self->{_string_decoder}->($s) if $decode;
return ($s, $p);
}
sub _mysql_unpack_field_packet {
my ($self, $response, $p) = @_;
my ($packet_id, $packet_len, $is_eof, $warns, $more_results);
($packet_id, $packet_len, $p, $is_eof, $warns, $more_results) = $self->_mysql_unpack_header($response, $p);
my %field;
($field{catalog}, $p) = $self->_mysql_unpack_string($response, $p);
($field{db}, $p) = $self->_mysql_unpack_string($response, $p);
($field{table}, $p) = $self->_mysql_unpack_string($response, $p);
($field{org_table}, $p) = $self->_mysql_unpack_string($response, $p);
($field{name}, $p) = $self->_mysql_unpack_string($response, $p);
($field{org_name}, $p) = $self->_mysql_unpack_string($response, $p);
$p += 3; # filler=12, charset_nr=0x21 (utf8)
$field{column_length} = unpack ( "L<", substr( $response, $p, 4 ) ); $p += 4;
$field{column_type} = unpack("C", substr($response, $p, 1)); $p += 1;
$p += 5; # flags, decimals, filler
return (\%field, $p);
}
=head2 BuildExcerpts
$excerpts = $sph->BuildExcerpts($docs, $index, $words, $opts)
Generate document excerpts for the specified documents.
=over 4
=item docs
An array reference of strings which represent the document
contents
=item index
A string specifiying the index whose settings will be used
for stemming, lexing and case folding
=item words
A string which contains the words to highlight
=item opts
A hash which contains additional optional highlighting parameters:
=over 4
=item before_match - a string to insert before a set of matching words, default is ""
=item after_match - a string to insert after a set of matching words, default is ""
=item chunk_separator - a string to insert between excerpts chunks, default is " ... "
=item limit - max excerpt size in symbols (codepoints), default is 256
=item limit_passages - Limits the maximum number of passages that can be included into the snippet. Integer, default is 0 (no limit).
=item limit_words - Limits the maximum number of keywords that can be included into the snippet. Integer, default is 0 (no limit).
=item around - how many words to highlight around each match, default is 5
=item exact_phrase - whether to highlight exact phrase matches only, default is false
=item single_passage - whether to extract single best passage only, default is false
=item use_boundaries
=item weight_order - Whether to sort the extracted passages in order of relevance (decreasing weight), or in order of appearance in the document (increasing position). Boolean, default is false.
=item query_mode - Whether to handle $words as a query in extended syntax, or as a bag of words (default behavior). For instance, in query mode ("one two" | "three four") will only highlight and include those occurrences "one two" or "three four" when the two words from each pair are adjacent to each other. In default mode, any single occurrence of "one", "two", "three", or "four" would be highlighted. Boolean, default is false.
=item force_all_words - Ignores the snippet length limit until it includes all the keywords. Boolean, default is false.
=item start_passage_id - Specifies the starting value of %PASSAGE_ID% macro (that gets detected and expanded in before_match, after_match strings). Integer, default is 1.
=item load_files - Whether to handle $docs as data to extract snippets from (default behavior), or to treat it as file names, and load data from specified files on the server side. Boolean, default is false.
=item html_strip_mode - HTML stripping mode setting. Defaults to "index", which means that index settings will be used. The other values are "none" and "strip", that forcibly skip or apply stripping irregardless of index settings; and "retain", that retains HTML markup and protects it from highlighting. The "retain" mode can only be used when highlighting full documents and thus requires that no snippet size limits are set. String, allowed values are "none", "strip", "index", and "retain".
=item allow_empty - Allows empty string to be returned as highlighting result when a snippet could not be generated (no keywords match, or no passages fit the limit). By default, the beginning of original text would be returned instead of an empty string. Boolean, default is false.
=item passage_boundary
=item emit_zones
=item load_files_scattered
=back
=back
Returns undef on failure.
Returns an array ref of string excerpts on success.
=cut
sub BuildExcerpts {
my ($self, $docs, $index, $words, $opts) = @_;
$opts ||= {};
croak("BuildExcepts() called with incorrect parameters")
unless (ref($docs) eq 'ARRAY'
&& defined($index)
&& defined($words)
&& ref($opts) eq 'HASH');
##################
# fixup options
##################
$opts->{"before_match"} ||= "";
$opts->{"after_match"} ||= "";
$opts->{"chunk_separator"} ||= " ... ";
$opts->{"limit"} ||= 256;
$opts->{"limit_passages"} ||= 0;
$opts->{"limit_words"} ||= 0;
$opts->{"around"} ||= 5;
$opts->{"exact_phrase"} ||= 0;
$opts->{"single_passage"} ||= 0;
$opts->{"use_boundaries"} ||= 0;
$opts->{"weight_order"} ||= 0;
$opts->{"query_mode"} ||= 0;
$opts->{"force_all_words"} ||= 0;
$opts->{"start_passage_id"} ||= 1;
$opts->{"load_files"} ||= 0;
$opts->{"html_strip_mode"} ||= "index";
$opts->{"allow_empty"} ||= 0;
$opts->{"passage_boundary"} ||= "none";
$opts->{"emit_zones"} ||= 0;
$opts->{"load_files_scattered"} ||= 0;
##################
# build request
##################
# v.1.2 req
my $req;
my $flags = 1; # remove spaces
$flags |= 2 if ( $opts->{"exact_phrase"} );
$flags |= 4 if ( $opts->{"single_passage"} );
$flags |= 8 if ( $opts->{"use_boundaries"} );
$flags |= 16 if ( $opts->{"weight_order"} );
$flags |= 32 if ( $opts->{"query_mode"} );
$flags |= 64 if ( $opts->{"force_all_words"} );
$flags |= 128 if ( $opts->{"load_files"} );
$flags |= 256 if ( $opts->{"allow_empty"} );
$flags |= 512 if ( $opts->{"emit_zones"} );
$flags |= 1024 if ( $opts->{"load_files_scattered"} );
$req = pack ( "NN", 0, $flags ); # mode=0, flags=$flags
$req .= pack ( "N/a*", $index ); # req index
$req .= pack ( "N/a*", $self->{_string_encoder}->($words)); # req words
# options
$req .= pack ( "N/a*", $opts->{"before_match"});
$req .= pack ( "N/a*", $opts->{"after_match"});
$req .= pack ( "N/a*", $opts->{"chunk_separator"});
$req .= pack ( "NN", int($opts->{"limit"}), int($opts->{"around"}) );
$req .= pack ( "NNN", int($opts->{"limit_passages"}),
int($opts->{"limit_words"}),
int($opts->{"start_passage_id"}) ); # v1.2
$req .= pack ( "N/a*", $opts->{"html_strip_mode"});
$req .= pack ( "N/a*", $opts->{"passage_boundary"});
# documents
$req .= pack ( "N", scalar(@$docs) );
foreach my $doc (@$docs) {
croak('BuildExcerpts: Found empty document in $docs') unless ($doc);
$req .= pack("N/a*", $self->{_string_encoder}->($doc));
}
##########################
# send query, get response
##########################
$req = pack ( "nnN/a*", SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, $req); # add header
my $response = $self->_ProcessRequest($req, VER_COMMAND_EXCERPT);
return unless $response;
my ($pos, $i) = 0;
my $res = []; # Empty hash ref
my $rlen = length($response);
for ( $i=0; $i< scalar(@$docs); $i++ ) {
my $len = unpack ( "N*", substr ( $response, $pos, 4 ) );
$pos += 4;
if ( $pos+$len > $rlen ) {
$self->_Error("incomplete reply");
return;
}
push(@$res, $self->{_string_decoder}->( substr ( $response, $pos, $len ) ));
$pos += $len;
}
return $res;
}
=head2 BuildKeywords
$results = $sph->BuildKeywords($query, $index, $hits)
Generate keyword list for a given query
Returns undef on failure,
Returns an array of hashes, where each hash describes a word in the query with the following keys:
=over 4
=item * tokenized
Tokenised term from query
=item * normalized
Normalised term from query
=item * docs
Number of docs in which word was found (if $hits is true)
=item * hits
Number of occurrences of word (if $hits is true)
=back
=cut
sub BuildKeywords {
my ( $self, $query, $index, $hits ) = @_;
# v.1.0 req
my $req = pack("N/a*", $self->{_string_encoder}->($query) );
$req .= pack("N/a*", $index);
$req .= pack("N", $self->{_string_encoder}->($hits) );
##################
# send query, get response
##################
$req = pack ( "nnN/a*", SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, $req);
my $response = $self->_ProcessRequest($req, VER_COMMAND_KEYWORDS);
return unless $response;
##################
# parse response
##################
my $p = 0;
my @res;
my $rlen = length($response);
my $nwords = unpack("N", substr ( $response, $p, 4 ) ); $p += 4;
for (my $i=0; $i < $nwords; $i++ ) {
my $len = unpack("N", substr ( $response, $p, 4 ) ); $p += 4;
my $tokenized = $len ? $self->{_string_decoder}->( substr ( $response, $p, $len ) ) : ""; $p += $len;
$len = unpack("N", substr ( $response, $p, 4 ) ); $p += 4;
my $normalized = $len ? $self->{_string_decoder}->( substr ( $response, $p, $len ) ) : ""; $p += $len;
my %data = ( tokenized => $tokenized, normalized => $normalized );
if ($hits) {
( $data{docs}, $data{hits} ) = unpack("N*N*", substr($response,$p,8));
$p += 8;
}
push(@res, \%data);
}
if ( $p > $rlen ) {
$self->_Error("incomplete reply");
return;
}
return \@res;
}
=head2 EscapeString
$escaped = $sph->EscapeString('abcde!@#$%')
Inserts backslash before all non-word characters in the given string.
=cut
sub EscapeString {
my $self = shift;
return quotemeta(shift);
}
=head2 UpdateAttributes
$sph->UpdateAttributes($index, \@attrs, \%values);
$sph->UpdateAttributes($index, \@attrs, \%values, $mva);
$sph->UpdateAttributes($index, \@attrs, \%values, $mva, $ignorenonexistent);
Update specified attributes on specified documents
=over 4
=item index
Name of the index to be updated
=item attrs
Array of attribute name strings
=item values
A hash with key as document id, value as an array of new attribute values
=item mva
If set, indicates that there is update of MVA attributes
=item ignorenonexistent
If set, the update will silently ignore any warnings about trying to update a column which is not exists in current index schema.
=back
Returns number of actually updated documents (0 or more) on success
Returns undef on failure
Usage example:
$sph->UpdateAttributes("test1", [ qw/group_id/ ], { 1 => [ 456] }) );
=cut
sub UpdateAttributes {
my ($self, $index, $attrs, $values, $mva, $ignorenonexistent ) = @_;
croak("index is not defined") unless (defined $index);
croak("attrs must be an array") unless ref($attrs) eq "ARRAY";
for my $attr (@$attrs) {
croak("attribute is not defined") unless (defined $attr);
}
croak("values must be a hashref") unless ref($values) eq "HASH";
for my $id (keys %$values) {
my $entry = $values->{$id};
croak("value id $id is not numeric") unless ($id =~ /$num_re/);
croak("value entry must be an array") unless ref($entry) eq "ARRAY";
croak("size of values must match size of attrs") unless @$entry == @$attrs;
for my $v (@$entry) {
if ($mva) {
croak("multi-valued entry $v is not an array") unless ref($v) eq 'ARRAY';
for my $vv (@$v) {
croak("array entry value $vv is not an integer") unless ($vv =~ /^(\d+)$/o);
}
}
else {
croak("entry value $v is not an integer") unless ($v =~ /^(\d+)$/o);
}
}
}
## build request
my $req = pack ( "N/a*", $index);
$req .= pack ( "N", scalar @$attrs );
$req .= pack ( "N", $ignorenonexistent ? 1 : 0 );
for my $attr (@$attrs) {
$req .= pack ( "N/a*", $attr)
. pack("N", $mva ? 1 : 0);
}
$req .= pack ( "N", scalar keys %$values );
foreach my $id (keys %$values) {
my $entry = $values->{$id};
$req .= $self->_sphPackU64($id);
if ($mva) {
for my $v ( @$entry ) {
$req .= pack ( "N", @$v );
for my $vv (@$v) {
$req .= pack ("N", $vv);
}
}
}
else {
for my $v ( @$entry ) {
$req .= pack ( "N", $v );
}
}
}
## connect, send query, get response
$req = pack ( "nnN/a*", SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, $req); ## add header
my $response = $self->_ProcessRequest($req, VER_COMMAND_UPDATE);
return unless $response;
## parse response
my ($updated) = unpack ( "N*", substr ( $response, 0, 4 ) );
return $updated;
}
=head2 Open
$sph->Open()
Opens a persistent connection for subsequent queries.
To reduce the network connection overhead of making Sphinx queries, you can call
$sph->Open(), then run any number of queries, and call $sph->Close() when
finished.
Returns 1 on success, 0 on failure.
=cut
sub Open {
my $self = shift;
$self->{_persistent} = 1;
if ($self->{_socket}) {
$self->_Error("already connected");
return 0;
}
my $fp = $self->_Connect() or return 0;
return 1;
}
=head2 Close
$sph->Close()
Closes a persistent connection.
Returns 1 on success, 0 on failure.
=cut
sub Close {
my $self = shift;
$self->{_persistent} = 0;
if (! $self->{_socket}) {
$self->_Error("not connected");
return 0;
}
close($self->{_socket});
$self->{_socket} = undef;
return 1;
}
=head2 Status
$status = $sph->Status()
$status = $sph->Status($session)
Queries searchd status, and returns a hash of status variable name and value pairs.
Returns undef on failure.
=cut
sub Status {
my ($self, $session) = @_;
my $req = pack("nnNN", SEARCHD_COMMAND_STATUS, VER_COMMAND_STATUS, 4, $session ? 0 : 1 ); # len=4, body=1
my $response = $self->_ProcessRequest($req, VER_COMMAND_STATUS);
return unless $response;
my $p = 0;
my ($rows, $cols) = unpack("N*N*", substr ( $response, $p, 8 ) ); $p += 8;
return {} unless $rows && $cols;
my %res;
for (1 .. $rows ) {
my @entry;
for ( 1 .. $cols) {
my $len = unpack("N*", substr ( $response, $p, 4 ) ); $p += 4;
push(@entry, $len ? substr ( $response, $p, $len ) : ""); $p += $len;
}
if ($cols <= 2) {
$res{$entry[0]} = $entry[1];
}
else {
my $name = shift @entry;
$res{$name} = \@entry;
}
}
return \%res;
}
=head2 FlushAttributes
=cut
sub FlushAttributes {
my $self = shift;
my $req = pack("nnN", SEARCHD_COMMAND_FLUSHATTRS, VER_COMMAND_FLUSHATTRS, 0 ); # len=0
my $response = $self->_ProcessRequest($req, VER_COMMAND_FLUSHATTRS);
return unless $response;
my $tag = -1;
if (length($response) == 4) {
$tag = unpack ( "N*", substr ( $response, 0, 4 ) );
}
else {
$self->_Error("unexpected response length");
}
return $tag;
}
=head1 SEE ALSO
L
=head1 NOTES
There is (or was) a bundled Sphinx.pm in the contrib area of the Sphinx source
distribution, which was used as the starting point of Sphinx::Search.
Maintenance of that version appears to have lapsed at sphinx-0.9.7, so many of
the newer API calls are not available there. Sphinx::Search is mostly
compatible with the old Sphinx.pm except:
=over 4
=item On failure, Sphinx::Search returns undef rather than 0 or -1.
=item Sphinx::Search 'Set' functions are cascadable, e.g. you can do
Sphinx::Search->new
->SetMatchMode(SPH_MATCH_ALL)
->SetSortMode(SPH_SORT_RELEVANCE)
->Query("search terms")
=back
Sphinx::Search also provides documentation and unit tests, which were the main
motivations for branching from the earlier work.
=head1 AUTHOR
Jon Schutz
L
=head1 BUGS
Please report any bugs or feature requests to
C, or through the web interface at
L.
I will be notified, and then you'll automatically be notified of progress on
your bug as I make changes.
=head1 SUPPORT
You can find documentation for this module with the perldoc command.
perldoc Sphinx::Search
You can also look for information at:
=over 4
=item * AnnoCPAN: Annotated CPAN documentation
L
=item * CPAN Ratings
L
=item * RT: CPAN's request tracker
L
=item * Search CPAN
L
=back
=head1 ACKNOWLEDGEMENTS
This module is based on Sphinx.pm (not deployed to CPAN) for Sphinx version
0.9.7-rc1, by Len Kranendonk, which was in turn based on the Sphinx PHP API.
Thanks to Alexey Kholodkov for contributing a significant patch for handling persistent connections.
=head1 COPYRIGHT & LICENSE
Copyright 2015 Jon Schutz, all rights reserved.
This program is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License.
=cut
1;
Sphinx-Search-0.29/Makefile.PL 0000644 0000764 0000764 00000001356 11720302750 014242 0 ustar jon jon use strict;
use warnings;
use ExtUtils::MakeMaker;
WriteMakefile(
NAME => 'Sphinx::Search',
AUTHOR => 'Jon Schutz ',
VERSION_FROM => 'lib/Sphinx/Search.pm',
ABSTRACT_FROM => 'lib/Sphinx/Search.pm',
PL_FILES => {},
PREREQ_PM => {
'Test::More' => 0,
'File::SearchPath' => 0,
'Path::Class' => 0,
'Carp' => 0,
'Socket' => 0,
'DBI' => 0,
'Math::BigInt' => 0,
'Config' => 0,
'Errno' => 0,
'Fcntl' => 0,
'Class::Accessor::Fast' => 0,
'Data::Dumper' => 0,
'Encode' => 0,
'List::MoreUtils' => 0,
},
dist => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', },
clean => { FILES => 'Sphinx-Search-*' },
);
Sphinx-Search-0.29/Changes 0000644 0000764 0000764 00000006371 12501723326 013571 0 ustar jon jon Revision history for Sphinx-Search
0.01 2007-09-07
Initial release for sphinx-0.9.8-cvs-20070818
0.02 2007-09-08
Update to support UTF-8
0.03 2007-09-09
Upgraded API with AddQuery/RunQueries for sphinx-0.9.8-cvs-20070907
0.04 2007-09-10
Added support for tests to use non-standard mysql socket
0.05 2007-09-16
Fixed bug [rt.cpan.org #29383]
Added support for filter excludes
0.06 2007-09-24
Upgraded API with SetGeoAnchor and SetFilterFloatRange for
sphinx-0.9.8-svn-r820
0.07 2007-10-15
Upgraded API with SetIndexWeights for sphinx-0.9.8-svn-r871
0.08 2007-11-24
Fixed handling of error conditions on recv and connect (Igor Gerdler,
rt.cpan.org #30934, #30935)
0.09 2007-12-12
Upgraded API for sphinx-0.9.8-svn-r985
0.10 2008-01-30
Upgraded API for sphinx-0.9.8-svn-r1112
0.11 2008-03-11
Upgraded API for sphinx-0.9.8-rc1
0.12 2008-07-25
Upgraded API for sphinx-0.9.8
0.13_01 2009-01-29
Upgraded API for sphinx-0.9.9-rc1, development release. Open/Close
for persistent connections are known to not work; also 64 bit IDs fail.
0.14 2009-02-06
Fixed 64 bit ID problem (testing issue) and persistent connections.
0.15 2009-02-25
Fixed return values in BuildExcerpts (some error conditions returned 0,
should have been undef) (rt.cpan.org #43583)
0.16 2009-04-08
Upgraded API for sphinx-0.9.9-rc2
0.17 2009-04-09
Added support for inserting encoder/decoder for translating charsets between perl and searchd.
Additional UTF-8 tests.
0.18 2009-04-09
Included missing test files in CPAN package.
0.19 2009-04-10
Removed request for GMP implementation of Math::BigInt due to apparent
bug in Math::BigInt on 32 bit architectures (Math::BigInt::Calc and
Math::BigInt::GMP give different answers for 32 bit shift of long
integer)
Fixed incorrect setting for _max_id that was causing results with 64 bit IDs to be
ignored unless SetIDRange() had been called.
0.20 2009-05-01
Fixed dependency list to prevent failing tests on some systems.
0.21 2009-05-04
Fixed another missing dependency (Encode)
Fixed 64 bit ID signed transfer on 32 bit systems compiled with -Duse64bitint.
0.22 2009-05-07
Moved use of Config variables out of new so Config is only accessed at startup, to avoid delays in
new(). (rt.cpan.org #45789)
0.23_02 2010-04-04
Updated for compatibility with svn-r2269.
rt.cpan.org #54698 - doc fix
rt.cpan.org #56406 - string attributes
0.23_03 2010-05-06
rt.cpan.org #57171 - fixed occasional warnings due to uninitialised
values
0.24 2010-12-07
Updated for compatibility with 1.10-beta (svn-r2420)
0.25_01 2010-12-07
Updated for compatibility with svn-r2575
0.25_02 2010-12-16
rt.cpan.org #63945 - fixed missing constants
0.25_03 2011-03-23
Fixed variable "warnings" in results hash, should be "warning"
0.26.1 2011-07-27
Updated for compatibility with 2.0.1-beta (svn-2792)
0.27.1 2012-02-19
Updated for compatibility with 2.0.3-release
0.27.2 2012-03-18
Incorporated persistent connections patch from Alexey Kholodkov (rt.cpan.org #70760)
Minor changes to fix perlcritic warnings.
0.28 2013-06-20
rt.cpan.org #84830 - documentation update
Updated for compatibility with 2.0.8-release
0.29 2015-03-17
Updated for 2.2.8. Added experimental SphinxQL support.
Sphinx-Search-0.29/META.yml 0000664 0000764 0000764 00000001334 12501725115 013541 0 ustar jon jon ---
abstract: 'Sphinx search engine API Perl client'
author:
- 'Jon Schutz '
build_requires:
ExtUtils::MakeMaker: '0'
configure_requires:
ExtUtils::MakeMaker: '0'
dynamic_config: 1
generated_by: 'ExtUtils::MakeMaker version 6.72, CPAN::Meta::Converter version 2.142060'
license: unknown
meta-spec:
url: http://module-build.sourceforge.net/META-spec-v1.4.html
version: '1.4'
name: Sphinx-Search
no_index:
directory:
- t
- inc
requires:
Carp: '0'
Class::Accessor::Fast: '0'
Config: '0'
DBI: '0'
Data::Dumper: '0'
Encode: '0'
Errno: '0'
Fcntl: '0'
File::SearchPath: '0'
List::MoreUtils: '0'
Math::BigInt: '0'
Path::Class: '0'
Socket: '0'
Test::More: '0'
version: '0.29'
Sphinx-Search-0.29/META.json 0000664 0000764 0000764 00000002412 12501725115 013707 0 ustar jon jon {
"abstract" : "Sphinx search engine API Perl client",
"author" : [
"Jon Schutz "
],
"dynamic_config" : 1,
"generated_by" : "ExtUtils::MakeMaker version 6.72, CPAN::Meta::Converter version 2.142060",
"license" : [
"unknown"
],
"meta-spec" : {
"url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec",
"version" : "2"
},
"name" : "Sphinx-Search",
"no_index" : {
"directory" : [
"t",
"inc"
]
},
"prereqs" : {
"build" : {
"requires" : {
"ExtUtils::MakeMaker" : "0"
}
},
"configure" : {
"requires" : {
"ExtUtils::MakeMaker" : "0"
}
},
"runtime" : {
"requires" : {
"Carp" : "0",
"Class::Accessor::Fast" : "0",
"Config" : "0",
"DBI" : "0",
"Data::Dumper" : "0",
"Encode" : "0",
"Errno" : "0",
"Fcntl" : "0",
"File::SearchPath" : "0",
"List::MoreUtils" : "0",
"Math::BigInt" : "0",
"Path::Class" : "0",
"Socket" : "0",
"Test::More" : "0"
}
}
},
"release_status" : "stable",
"version" : "0.29"
}