Bug 12748: (QA followup) make new tests pass

* t/Koha/ItemType.pm:
    * Rename to .t
    * Fix method names
  * t/db_dependent/Koha/ItemTypes.t: Fix method names and Koha::Database usage

Signed-off-by: Jesse Weaver <jweaver@bywatersolutions.com>
Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>

Signed-off-by: Kyle M Hall <kyle@bywatersolutions.com>

C4/AuthoritiesMarc.pm

@@ -29,6 +29,9 @@ use C4::Log;
 use Koha::MetadataRecord::Authority;
 use Koha::Authorities;
 use Koha::Authority::Types;
+use Koha::Authority;
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
 
 use vars qw(@ISA @EXPORT);
 
@@ -349,7 +352,10 @@ sub CountUsage {
         ### ZOOM search here
         my $query;
         $query= "an:".$authid;
-  		my ($err,$res,$result) = C4::Search::SimpleSearch($query,0,10);
+        # Should really be replaced with a real count call, this is a
+        # bad way.
+        my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
+        my ($err,$res,$result) = $searcher->simple_search_compat($query,0,1);
         if ($err) {
             warn "Error: $err from search $query";
             $result = 0;
@@ -822,7 +828,8 @@ sub FindDuplicateAuthority {
             $_->[1]=~s/$filtervalues/ /g; $query.= " $op he:\"".$_->[1]."\"" if ($_->[0]=~/[A-z]/);
         }
     }
-    my ($error, $results, $total_hits) = C4::Search::SimpleSearch( $query, 0, 1, [ "authorityserver" ] );
+    my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
+    my ($error, $results, $total_hits) = $searcher->simple_search_compat( $query, 0, 1 );
     # there is at least 1 result => return the 1st one
     if (!defined $error && @{$results} ) {
         my $marcrecord = C4::Search::new_record_from_zebra(

C4/Biblio.pm

@@ -37,12 +37,15 @@ use C4::ClassSource;
 use C4::Charset;
 use C4::Linker;
 use C4::OAI::Sets;
+use C4::Debug;
 
 use Koha::Cache;
 use Koha::Authority::Types;
 use Koha::Acquisition::Currencies;
+use Koha::SearchEngine;
 
 use vars qw(@ISA @EXPORT);
+use vars qw($debug $cgi_debug);
 
 BEGIN {
 
@@ -2888,31 +2891,60 @@ sub TransformMarcToKohaOneField {
 
 =head2 ModZebra
 
-  ModZebra( $biblionumber, $op, $server );
+  ModZebra( $biblionumber, $op, $server, $record );
 
 $biblionumber is the biblionumber we want to index
 
-$op is specialUpdate or delete, and is used to know what we want to do
+$op is specialUpdate or recordDelete, and is used to know what we want to do
 
 $server is the server that we want to update
 
+$record is the update MARC record if it's available. If it's not supplied
+and is needed, it'll be loaded from the database.
+
 =cut
 
 sub ModZebra {
 ###Accepts a $server variable thus we can use it for biblios authorities or other zebra dbs
-    my ( $biblionumber, $op, $server ) = @_;
+    my ( $biblionumber, $op, $server, $record ) = @_;
+    $debug && warn "ModZebra: update requested for: $biblionumber $op $server\n";
+    if ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' ) {
+
+        # TODO abstract to a standard API that'll work for whatever
+        require Koha::ElasticSearch::Indexer;
+        my $indexer = Koha::ElasticSearch::Indexer->new(
+            {
+                index => $server eq 'biblioserver'
+                ? $Koha::SearchEngine::BIBLIOS_INDEX
+                : $Koha::SearchEngine::AUTHORITIES_INDEX
+            }
+        );
+        if ( $op eq 'specialUpdate' ) {
+            unless ($record) {
+                $record = GetMarcBiblio($biblionumber, 1);
+            }
+            my $records = [$record];
+            $indexer->update_index_background( [$biblionumber], [$record] );
+        }
+        elsif ( $op eq 'recordDelete' ) {
+            $indexer->delete_index_background( [$biblionumber] );
+        }
+        else {
+            croak "ModZebra called with unknown operation: $op";
+        }
+    }
+
     my $dbh = C4::Context->dbh;
 
     # true ModZebra commented until indexdata fixes zebraDB crashes (it seems they occur on multiple updates
     # at the same time
     # replaced by a zebraqueue table, that is filled with ModZebra to run.
     # the table is emptied by rebuild_zebra.pl script (using the -z switch)
-
     my $check_sql = "SELECT COUNT(*) FROM zebraqueue
-                     WHERE server = ?
-                     AND   biblio_auth_number = ?
-                     AND   operation = ?
-                     AND   done = 0";
+    WHERE server = ?
+        AND   biblio_auth_number = ?
+        AND   operation = ?
+        AND   done = 0";
     my $check_sth = $dbh->prepare_cached($check_sql);
     $check_sth->execute( $server, $biblionumber, $op );
     my ($count) = $check_sth->fetchrow_array;
@@ -3468,7 +3500,7 @@ sub ModBiblioMarc {
     $sth = $dbh->prepare("UPDATE biblioitems SET marc=?,marcxml=? WHERE biblionumber=?");
     $sth->execute( $record->as_usmarc(), $record->as_xml_record($encoding), $biblionumber );
     $sth->finish;
-    ModZebra( $biblionumber, "specialUpdate", "biblioserver" );
+    ModZebra( $biblionumber, "specialUpdate", "biblioserver", $record );
     return $biblionumber;
 }
 

C4/Items.pm

@@ -38,6 +38,8 @@ use Koha::DateUtils qw/dt_from_string/;
 use Koha::Database;
 
 use Koha::Database;
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
 
 use vars qw(@ISA @EXPORT);
 
@@ -2601,12 +2603,12 @@ counts Usage of itemnumber in Analytical bibliorecords.
 
 sub GetAnalyticsCount {
     my ($itemnumber) = @_;
-    require C4::Search;
 
     ### ZOOM search here
     my $query;
     $query= "hi=".$itemnumber;
-            my ($err,$res,$result) = C4::Search::SimpleSearch($query,0,10);
+    my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
+    my ($err,$res,$result) = $searcher->simple_search_compat($query,0,10);
     return ($result);
 }
 

C4/Matcher.pm

@@ -23,7 +23,8 @@ use warnings;
 use C4::Context;
 use MARC::Record;
 
-
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
 
 =head1 NAME
 
@@ -655,10 +656,9 @@ sub get_matches {
                     #NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it.
             }
 
-            require C4::Search;
-
+            my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
             ( $error, $searchresults, $total_hits ) =
-              C4::Search::SimpleSearch( $query, 0, $max_matches );
+              $searcher->simple_search_compat( $query, 0, $max_matches );
         }
         elsif ( $self->{'record_type'} eq 'authority' ) {
             my $authresults;

C4/Search.pm

@@ -623,6 +623,15 @@ sub getRecords {
                 }
             }
         );
+
+    # This sorts the facets into alphabetical order
+    if (@facets_loop) {
+        foreach my $f (@facets_loop) {
+            $f->{facets} = [ sort { uc($a->{facet_label_value}) cmp uc($b->{facet_label_value}) } @{ $f->{facets} } ];
+        }
+        @facets_loop = sort {$a->{expand} cmp $b->{expand}} @facets_loop;
+    }
+
     return ( undef, $results_hashref, \@facets_loop );
 }
 
@@ -1321,6 +1330,7 @@ Shim function to ease the transition from buildQuery to a new QueryParser.
 This function is called at the beginning of buildQuery, and modifies
 buildQuery's input. If it can handle the input, it returns a query that
 buildQuery will not try to parse.
+
 =cut
 
 sub parseQuery {
@@ -2538,7 +2548,7 @@ sub _ZOOM_event_loop {
 
 =head2 new_record_from_zebra
 
-Given raw data from a Zebra result set, return a MARC::Record object
+Given raw data from a searchengine result set, return a MARC::Record object
 
 This helper function is needed to take into account all the involved
 system preferences and configuration variables to properly create the
@@ -2547,6 +2557,9 @@ MARC::Record object.
 If we are using GRS-1, then the raw data we get from Zebra should be USMARC
 data. If we are using DOM, then it has to be MARCXML.
 
+If we are using elasticsearch, it'll already be a MARC::Record and this
+function needs a new name.
+
 =cut
 
 sub new_record_from_zebra {
@@ -2554,6 +2567,10 @@ sub new_record_from_zebra {
     my $server   = shift;
     my $raw_data = shift;
     # Set the default indexing modes
+    my $search_engine = C4::Context->preference("SearchEngine");
+    if ($search_engine eq 'Elasticsearch') {
+        return $raw_data;
+    }
     my $index_mode = ( $server eq 'biblioserver' )
                         ? C4::Context->config('zebra_bib_index_mode') // 'dom'
                         : C4::Context->config('zebra_auth_index_mode') // 'dom';

C4/XISBN.pm

@@ -26,6 +26,9 @@ use C4::External::Syndetics qw(get_syndetics_editions);
 use LWP::UserAgent;
 use HTTP::Request::Common;
 
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
+
 use strict;
 #use warnings; FIXME - Bug 2505
 use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
@@ -63,7 +66,8 @@ sub _get_biblio_from_xisbn {
     my $xisbn = shift;
     my $dbh = C4::Context->dbh;
 
-    my ( $errors, $results, $total_hits ) = C4::Search::SimpleSearch( "nb=$xisbn", 0, 1 );
+    my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
+    my ( $errors, $results, $total_hits ) = $searcher->simple_search_compat( "nb=$xisbn", 0, 1 );
     return unless ( !$errors && scalar @$results );
 
     my $record = C4::Search::new_record_from_zebra( 'biblioserver', $results->[0] );

Koha/AuthorisedValue.pm

@@ -27,7 +27,7 @@ use base qw(Koha::Object);
 
 =head1 NAME
 
-Koha::AuthorisedValue - Koha Authorised Value Object class
+Koha::AuthorisedValue - Koha Authorised value Object class
 
 =head1 API
 

Koha/AuthorisedValues.pm

@@ -29,7 +29,7 @@ use base qw(Koha::Objects);
 
 =head1 NAME
 
-Koha::AuthorisedValues - Koha AuthorisedValue Object set class
+Koha::AuthorisedValues - Koha Authorised value Object set class
 
 =head1 API
 

Koha/Authority.pm

@@ -22,6 +22,8 @@ use Modern::Perl;
 use Carp;
 
 use Koha::Database;
+use C4::Context;
+use MARC::Record;
 
 use base qw(Koha::Object);
 
@@ -43,4 +45,54 @@ sub _type {
     return 'AuthHeader';
 }
 
+=head2 get_all_authorities_iterator
+
+    my $it = Koha::Authority->get_all_authorities_iterator();
+
+This will provide an iterator object that will, one by one, provide the
+Koha::Authority of each authority.
+
+The iterator is a Koha::MetadataIterator object.
+
+=cut
+
+sub get_all_authorities_iterator {
+    my $database = Koha::Database->new();
+    my $schema   = $database->schema();
+    my $rs =
+      $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },
+        { columns => [qw/ authid authtypecode marcxml /] } );
+    my $next_func = sub {
+        my $row = $rs->next();
+        return undef if !$row;
+        my $authid       = $row->authid;
+        my $authtypecode = $row->authtypecode;
+        my $marcxml      = $row->marcxml;
+
+        my $record = eval {
+            MARC::Record->new_from_xml(
+                StripNonXmlChars($marcxml),
+                'UTF-8',
+                (
+                    C4::Context->preference("marcflavour") eq "UNIMARC"
+                    ? "UNIMARCAUTH"
+                    : C4::Context->preference("marcflavour")
+                )
+            );
+        };
+        confess $@ if ($@);
+        $record->encoding('UTF-8');
+
+        # I'm not sure why we don't use the authtypecode from the database,
+        # but this is how the original code does it.
+        require C4::AuthoritiesMarc;
+        $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record);
+
+        my $auth = __PACKAGE__->new( $record, $authid, $authtypecode );
+
+        return $auth;
+      };
+      return Koha::MetadataIterator->new($next_func);
+}
+
 1;

Koha/BiblioUtils.pm

@@ -0,0 +1,164 @@
+package Koha::BiblioUtils;
+
+# This contains functions to do with managing biblio records.
+
+# Copyright 2014 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+=head1 NAME
+
+Koha::BiblioUtils - contains fundamental biblio-related functions
+
+=head1 DESCRIPTION
+
+This contains functions for normal operations on biblio records.
+
+Note: really, C4::Biblio does the main functions, but the Koha namespace is
+the new thing that should be used.
+
+=cut
+
+use C4::Biblio; # EmbedItemsInMarcBiblio
+use Koha::MetadataIterator;
+use Koha::Database;
+use Modern::Perl;
+
+use Data::Dumper; # TODO remove
+
+use base qw(Koha::MetadataRecord);
+
+__PACKAGE__->mk_accessors(qw( record schema id datatype ));
+
+=head1 FUNCTIONS
+
+=head2 new
+
+    my $biblio = Koha::BiblioUtils->new($marc_record, [$biblionumber]);
+
+Creates an instance of C<Koha::BiblioUtils> based on the marc record. If known,
+the biblionumber can be provided too.
+
+=cut
+
+sub new {
+    my $class = shift;
+    my $record = shift;
+    my $biblionumber = shift;
+
+    my $self = $class->SUPER::new(
+        {
+            'record'   => $record,
+            'schema'   => lc C4::Context->preference("marcflavour"),
+            'id'       => $biblionumber,
+            'datatype' => 'biblio',
+        }
+    );
+    bless $self, $class;
+    return $self;
+}
+
+=head2 get_from_biblionumber
+
+    my $biblio = Koha::BiblioUtils->get_from_biblionumber($biblionumber, %options);
+
+This will give you an instance of L<Koha::BiblioUtils> that is the biblio that
+you requested.
+
+Options are:
+
+=over 4
+
+=item C<$item_data>
+
+If true, then the item data will be merged into the record when it's loaded.
+
+=back
+
+It will return C<undef> if the biblio doesn't exist.
+
+=cut
+
+sub get_from_biblionumber {
+    my ($class, $bibnum, %options) = @_;
+
+    my $marc = $class->get_marc_biblio($bibnum, %options);
+    return $class->new($marc, $bibnum);
+}
+
+=head2 get_all_biblios_iterator
+
+    my $it = Koha::BiblioUtils->get_all_biblios_iterator();
+
+This will provide an iterator object that will, one by one, provide the
+Koha::BiblioUtils of each biblio. This will include the item data.
+
+The iterator is a Koha::MetadataIterator object.
+
+=cut
+
+sub get_all_biblios_iterator {
+    my $database = Koha::Database->new();
+    my $schema   = $database->schema();
+    my $rs =
+      $schema->resultset('Biblio')->search( {},
+        { columns => [qw/ biblionumber /] } );
+    my $next_func = sub {
+        # Warn and skip bad records, otherwise we break the loop
+        while (1) {
+            my $row = $rs->next();
+            return if !$row;
+            my $marc = C4::Biblio::GetMarcBiblio( $row->biblionumber, 1 );
+            my $next = eval {
+                __PACKAGE__->new($marc, $row->biblionumber);
+            };
+            if ($@) {
+                warn "Something went wrong reading record for biblio $row->biblionumber: $@\n";
+                next;
+            }
+            return $next;
+        }
+    };
+    return Koha::MetadataIterator->new($next_func);
+}
+
+=head2 get_marc_biblio
+
+    my $marc = Koha::BiblioUtils->get_marc_biblio($bibnum, %options);
+
+This non-class function fetches the MARC::Record for the given biblio number.
+Nothing is returned if the biblionumber couldn't be found (or it somehow has no
+MARC data.)
+
+Options are:
+
+=over 4
+
+=item item_data
+
+If set to true, item data is embedded in the record. Default is to not do this.
+
+=back
+
+=cut
+
+sub get_marc_biblio {
+    my ($class, $bibnum, %options) = @_;
+
+    return C4::Biblio::GetMarcBiblio( $bibnum, ($options{item_data} ? 1 : 0 ) );
+}
+
+1;

Koha/BiblioUtils/Iterator.pm

@@ -0,0 +1,126 @@
+package Koha::BiblioUtils::Iterator;
+
+# This contains an iterator over biblio records
+
+# Copyright 2014 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+=head1 NAME
+
+Koha::BiblioUtils::Iterator - iterates over biblios provided by a DBIx::Class::ResultSet
+
+=head1 DESCRIPTION
+
+This provides an iterator that gives the MARC::Record of each biblio that's
+returned by a L<DBIx::Class::ResultSet> that provides a C<biblionumber>, and
+C<marc> or C<marcxml> column from the biblioitems table.
+
+=head1 SYNOPSIS
+
+  use Koha::BiblioUtils::Iterator;
+  my $rs = $schema->resultset('biblioitems');
+  my $iterator = Koha::BiblioUtils::Iterator->new($rs);
+  while (my $record = $iterator->next()) {
+      // do something with $record
+  }
+
+=head1 METHODS
+
+=cut
+
+use C4::Biblio;    # :( - for EmbedItemsInMarcBiblio
+
+use Carp;
+use MARC::Record;
+use MARC::File::XML;
+use Modern::Perl;
+
+=head2 new
+
+    my $it = new($sth, option => $value, ...);
+
+Takes a ResultSet to iterate over, and gives you an iterator on it. Optional
+options may be specified.
+
+=head3 Options
+
+=over 4
+
+=item items
+
+Set to true to include item data in the resulting MARC record.
+
+=back
+
+=cut
+
+sub new {
+    my ( $class, $rs, %options ) = @_;
+
+    bless {
+        rs => $rs,
+        %options,
+    }, $class;
+}
+
+=head2 next()
+
+In a scalar context, provides the next MARC::Record from the ResultSet, or
+C<undef> if there are no more.
+
+In a list context it will provide ($biblionumber, $record).
+
+=cut
+
+sub next {
+    my ($self) = @_;
+
+    my $marc;
+    my $row = $self->{rs}->next();
+    return if !$row;
+    if ( $row->marc ) {
+        $marc = MARC::Record->new_from_usmarc( $row->marc );
+    }
+    elsif ( $row->marcxml ) {
+        $marc = MARC::Record->new_from_xml( $row->marcxml );
+    }
+    else {
+        confess "No marc or marcxml column returned in the request.";
+    }
+
+    my $bibnum;
+    if ( $self->{items} ) {
+        $bibnum = $row->get_column('biblionumber');
+        confess "No biblionumber column returned in the request."
+          if ( !defined($bibnum) );
+
+        # TODO this should really be in Koha::BiblioUtils or something similar.
+        C4::Biblio::EmbedItemsInMarcBiblio( $marc, $bibnum );
+    }
+
+    if (wantarray) {
+        $bibnum //= $row->get_column('biblionumber');
+        confess "No biblionumber column returned in the request."
+          if ( !defined($bibnum) );
+        return ( $bibnum, $marc );
+    }
+    else {
+        return $marc;
+    }
+}
+
+1;

Koha/Database.pm

@@ -41,6 +41,8 @@ use vars qw($database);
 
 __PACKAGE__->mk_accessors(qw( ));
 
+our $schema; # the schema is a singleton
+
 # _new_schema
 # Internal helper function (not a method!). This creates a new
 # database connection from the data given in the current context, and
@@ -110,8 +112,7 @@ creates one, and connects to the database.
 
 This database handle is cached for future use: if you call
 C<$database-E<gt>schema> twice, you will get the same handle both
-times. If you need a second database handle, use C<&new_schema> and
-possibly C<&set_schema>.
+times. If you need a second database handle, use C<&new_schema>.
 
 =cut
 

Koha/ElasticSearch.pm

@@ -0,0 +1,450 @@
+package Koha::ElasticSearch;
+
+# Copyright 2015 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use base qw(Class::Accessor);
+
+use C4::Context;
+
+use Koha::Database;
+
+use Carp;
+use JSON;
+use Modern::Perl;
+use Readonly;
+
+use Data::Dumper;    # TODO remove
+
+__PACKAGE__->mk_ro_accessors(qw( index ));
+__PACKAGE__->mk_accessors(qw( sort_fields ));
+
+# Constants to refer to the standard index names
+Readonly our $BIBLIOS_INDEX     => 'biblios';
+Readonly our $AUTHORITIES_INDEX => 'authorities';
+
+=head1 NAME
+
+Koha::ElasticSearch - Base module for things using elasticsearch
+
+=head1 ACCESSORS
+
+=over 4
+
+=item index
+
+The name of the index to use, generally 'biblios' or 'authorities'.
+
+=back
+
+=head1 FUNCTIONS
+
+=cut
+
+sub new {
+    my $class = shift @_;
+    my $self = $class->SUPER::new(@_);
+    # Check for a valid index
+    croak('No index name provided') unless $self->index;
+    return $self;
+}
+
+=head2 get_elasticsearch_params
+
+    my $params = $self->get_elasticsearch_params();
+
+This provides a hashref that contains the parameters for connecting to the
+ElasicSearch servers, in the form:
+
+    {
+        'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
+        'index_name' => 'koha_instance_index',
+    }
+
+This is configured by the following in the C<config> block in koha-conf.xml:
+
+    <elasticsearch>
+        <server>127.0.0.1:9200</server>
+        <server>anotherserver:9200</server>
+        <index_name>koha_instance</index_name>
+    </elasticsearch>
+
+=cut
+
+sub get_elasticsearch_params {
+    my ($self) = @_;
+
+    # Copy the hash so that we're not modifying the original
+    my $conf = C4::Context->config('elasticsearch');
+    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
+    my $es = { %{ $conf } };
+
+    # Helpfully, the multiple server lines end up in an array for us anyway
+    # if there are multiple ones, but not if there's only one.
+    my $server = $es->{server};
+    delete $es->{server};
+    if ( ref($server) eq 'ARRAY' ) {
+
+        # store it called 'nodes' (which is used by newer Search::Elasticsearch)
+        $es->{nodes} = $server;
+    }
+    elsif ($server) {
+        $es->{nodes} = [$server];
+    }
+    else {
+        die "No elasticsearch servers were specified in koha-conf.xml.\n";
+    }
+    die "No elasticserver index_name was specified in koha-conf.xml.\n"
+      if ( !$es->{index_name} );
+    # Append the name of this particular index to our namespace
+    $es->{index_name} .= '_' . $self->index;
+    return $es;
+}
+
+=head2 get_elasticsearch_settings
+
+    my $settings = $self->get_elasticsearch_settings();
+
+This provides the settings provided to elasticsearch when an index is created.
+These can do things like define tokenisation methods.
+
+A hashref containing the settings is returned.
+
+=cut
+
+sub get_elasticsearch_settings {
+    my ($self) = @_;
+
+    # Ultimately this should come from a file or something, and not be
+    # hardcoded.
+    my $settings = {
+        index => {
+            analysis => {
+                analyzer => {
+                    analyser_phrase => {
+                        tokenizer => 'keyword',
+                        filter    => ['lowercase'],
+                    },
+                    analyser_standard => {
+                        tokenizer => 'standard',
+                        filter    => ['lowercase'],
+                    }
+                },
+            }
+        }
+    };
+    return $settings;
+}
+
+=head2 get_elasticsearch_mappings
+
+    my $mappings = $self->get_elasticsearch_mappings();
+
+This provides the mappings that get passed to elasticsearch when an index is
+created.
+
+=cut
+
+sub get_elasticsearch_mappings {
+    my ($self) = @_;
+
+    # TODO cache in the object?
+    my $mappings = {
+        data => {
+            properties => {
+                record => {
+                    store          => "yes",
+                    include_in_all => JSON::false,
+                    type           => "string",
+                },
+                '_all.phrase' => {
+                    search_analyzer => "analyser_phrase",
+                    index_analyzer  => "analyser_phrase",
+                    type            => "string",
+                },
+            }
+        }
+    };
+    my %sort_fields;
+    my $marcflavour = lc C4::Context->preference('marcflavour');
+    $self->_foreach_mapping(
+        sub {
+            my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
+            return if $marc_type ne $marcflavour;
+            # TODO if this gets any sort of complexity to it, it should
+            # be broken out into its own function.
+
+            # TODO be aware of date formats, but this requires pre-parsing
+            # as ES will simply reject anything with an invalid date.
+            my $es_type =
+              $type eq 'boolean'
+              ? 'boolean'
+              : 'string';
+            $mappings->{data}{properties}{$name} = {
+                search_analyzer => "analyser_standard",
+                index_analyzer  => "analyser_standard",
+                type            => $es_type,
+                fields          => {
+                    phrase => {
+                        search_analyzer => "analyser_phrase",
+                        index_analyzer  => "analyser_phrase",
+                        type            => "string",
+                        copy_to         => "_all.phrase",
+                    },
+                    raw => {
+                        "type" => "string",
+                        "index" => "not_analyzed",
+                    }
+                },
+            };
+            $mappings->{data}{properties}{$name}{null_value} = 0
+              if $type eq 'boolean';
+            if ($facet) {
+                $mappings->{data}{properties}{ $name . '__facet' } = {
+                    type  => "string",
+                    index => "not_analyzed",
+                };
+            }
+            if ($suggestible) {
+                $mappings->{data}{properties}{ $name . '__suggestion' } = {
+                    type => 'completion',
+                    index_analyzer => 'simple',
+                    search_analyzer => 'simple',
+                };
+            }
+            # Sort may be true, false, or undef. Here we care if it's
+            # anything other than undef.
+            if (defined $sort) {
+                $mappings->{data}{properties}{ $name . '__sort' } = {
+                    search_analyzer => "analyser_phrase",
+                    index_analyzer  => "analyser_phrase",
+                    type            => "string",
+                    include_in_all  => JSON::false,
+                    fields          => {
+                        phrase => {
+                            search_analyzer => "analyser_phrase",
+                            index_analyzer  => "analyser_phrase",
+                            type            => "string",
+                        },
+                    },
+                };
+                $sort_fields{$name} = 1;
+            }
+        }
+    );
+    $self->sort_fields(\%sort_fields);
+    return $mappings;
+}
+
+# This overrides the accessor provided by Class::Accessor so that if
+# sort_fields isn't set, then it'll generate it.
+sub sort_fields {
+    my $self = shift;
+
+    if (@_) {
+        $self->_sort_fields_accessor(@_);
+        return;
+    }
+    my $val = $self->_sort_fields_accessor();
+    return $val if $val;
+
+    # This will populate the accessor as a side effect
+    $self->get_elasticsearch_mappings();
+    return $self->_sort_fields_accessor();
+}
+
+# Provides the rules for data conversion.
+sub get_fixer_rules {
+    my ($self) = @_;
+
+    my $marcflavour = lc C4::Context->preference('marcflavour');
+    my @rules;
+    $self->_foreach_mapping(
+        sub {
+            my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
+            return if $marc_type ne $marcflavour;
+            my $options = '';
+
+            # There's a bug when using 'split' with something that
+            # selects a range
+            # The split makes everything into nested arrays, but that's not
+            # really a big deal, ES doesn't mind.
+            $options = '-split => 1' unless $marc_field =~ m|_/| || $type eq 'sum';
+            push @rules, "marc_map('$marc_field','${name}', $options)";
+            if ($facet) {
+                push @rules, "marc_map('$marc_field','${name}__facet', $options)";
+            }
+            if ($suggestible) {
+                push @rules,
+"marc_map('$marc_field','${name}__suggestion.input.\$append', $options)";
+            }
+            if ( $type eq 'boolean' ) {
+
+                # boolean gets special handling, basically if it doesn't exist,
+                # it's added and set to false. Otherwise we can't query it.
+                push @rules,
+                  "unless exists('$name') add_field('$name', 0) end";
+            }
+            if ($type eq 'sum' ) {
+                push @rules, "sum('$name')";
+            }
+            # Sort is a bit special as it can be true, false, undef. For
+            # fixer rules, we care about "true", or "undef" if there is
+            # special handling of this field from other one. "undef" means
+            # to do the default thing, which is make it sortable.
+            if ($self->sort_fields()->{$name}) {
+                if ($sort || !defined $sort) {
+                    push @rules, "marc_map('$marc_field','${name}__sort', $options)";
+                }
+            }
+        }
+    );
+    return \@rules;
+}
+
+=head2 _foreach_mapping
+
+    $self->_foreach_mapping(
+        sub {
+            my ( $name, $type, $facet, $suggestible, $sort, $marc_type,
+                $marc_field )
+              = @_;
+            return unless $marc_type eq 'marc21';
+            print "Data comes from: " . $marc_field . "\n";
+        }
+    );
+
+This allows you to apply a function to each entry in the elasticsearch mappings
+table, in order to build the mappings for whatever is needed.
+
+In the provided function, the files are:
+
+=over 4
+
+=item C<$name>
+
+The field name for elasticsearch (corresponds to the 'mapping' column in the
+database.
+
+=item C<$type>
+
+The type for this value, e.g. 'string'.
+
+=item C<$facet>
+
+True if this value should be facetised. This only really makes sense if the
+field is understood by the facet processing code anyway.
+
+=item C<$sort>
+
+True if this is a field that a) needs special sort handling, and b) if it
+should be sorted on. False if a) but not b). Undef if not a). This allows,
+for example, author to be sorted on but not everything marked with "author"
+to be included in that sort.
+
+=item C<$marc_type>
+
+A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
+'unimarc', 'normarc'.
+
+=item C<$marc_field>
+
+A string that describes the MARC field that contains the data to extract.
+These are of a form suited to Catmandu's MARC fixers.
+
+=back
+
+=cut
+
+sub _foreach_mapping {
+    my ( $self, $sub ) = @_;
+
+    # TODO use a caching framework here
+    my $search_fields = Koha::Database->schema->resultset('SearchField')->search(
+        {
+            'search_marc_map.index_name' => $self->index,
+        },
+        {   join => { search_marc_to_fields => 'search_marc_map' },
+            '+select' => [
+                'search_marc_to_fields.facet',
+                'search_marc_to_fields.suggestible',
+                'search_marc_to_fields.sort',
+                'search_marc_map.marc_type',
+                'search_marc_map.marc_field',
+            ],
+            '+as'     => [
+                'facet',
+                'suggestible',
+                'sort',
+                'marc_type',
+                'marc_field',
+            ],
+        }
+    );
+
+    while ( my $search_field = $search_fields->next ) {
+        $sub->(
+            $search_field->name,
+            $search_field->type,
+            $search_field->get_column('facet'),
+            $search_field->get_column('suggestible'),
+            $search_field->get_column('sort'),
+            $search_field->get_column('marc_type'),
+            $search_field->get_column('marc_field'),
+        );
+    }
+}
+
+=head2 process_error
+
+    die process_error($@);
+
+This parses an Elasticsearch error message and produces a human-readable
+result from it. This result is probably missing all the useful information
+that you might want in diagnosing an issue, so the warning is also logged.
+
+Note that currently the resulting message is not internationalised. This
+will happen eventually by some method or other.
+
+=cut
+
+sub process_error {
+    my ($self, $msg) = @_;
+
+    warn $msg; # simple logging
+
+    # This is super-primitive
+    return "Unable to understand your search query, please rephrase and try again.\n" if $msg =~ /ParseException/;
+
+    return "Unable to perform your search. Please try again.\n";
+}
+
+1;
+
+__END__
+
+=head1 AUTHOR
+
+=over 4
+
+=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
+
+=item Robin Sheat C<< <robin@catalyst.net.nz> >>
+
+=back
+
+=cut

Koha/ElasticSearch/Indexer.pm

@@ -0,0 +1,207 @@
+package Koha::ElasticSearch::Indexer;
+
+# Copyright 2013 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use Carp;
+use Modern::Perl;
+use base qw(Koha::ElasticSearch);
+use Data::Dumper;
+
+# For now just marc, but we can do anything here really
+use Catmandu::Importer::MARC;
+use Catmandu::Store::ElasticSearch;
+
+Koha::ElasticSearch::Indexer->mk_accessors(qw( store ));
+
+=head1 NAME
+
+Koha::ElasticSearch::Indexer - handles adding new records to the index
+
+=head1 SYNOPSIS
+
+    my $indexer = Koha::ElasticSearch::Indexer->new(
+        { index => Koha::SearchEngine::BIBLIOS_INDEX } );
+    $indexer->drop_index();
+    $indexer->update_index(\@biblionumbers, \@records);
+
+=head1 FUNCTIONS
+
+=head2 $indexer->update_index($biblionums, $records);
+
+C<$biblionums> is an arrayref containing the biblionumbers for the records.
+
+C<$records> is an arrayref containing the L<MARC::Record>s themselves.
+
+The values in the arrays must match up, and the 999$c value in the MARC record
+will be rewritten using the values in C<$biblionums> to ensure they are correct.
+If C<$biblionums> is C<undef>, this won't happen, but you should be sure that
+999$c is correct on your own then.
+
+Note that this will modify the original record if C<$biblionums> is supplied.
+If that's a problem, clone them first.
+
+=cut
+
+sub update_index {
+    my ($self, $biblionums, $records) = @_;
+
+    # TODO should have a separate path for dealing with a large number
+    # of records at once where we use the bulk update functions in ES.
+    if ($biblionums) {
+        $self->_sanitise_records($biblionums, $records);
+    }
+
+    my $from    = $self->_convert_marc_to_json($records);
+    if ( !$self->store ) {
+        my $params  = $self->get_elasticsearch_params();
+        $self->store(
+            Catmandu::Store::ElasticSearch->new(
+                %$params,
+                index_settings => $self->get_elasticsearch_settings(),
+                index_mappings => $self->get_elasticsearch_mappings(),
+            )
+        );
+    }
+    $self->store->bag->add_many($from);
+    $self->store->bag->commit;
+    return 1;
+}
+
+=head2 $indexer->update_index_background($biblionums, $records)
+
+This has exactly the same API as C<update_index_background> however it'll
+return immediately. It'll start a background process that does the adding.
+
+If it fails to add to Elasticsearch then it'll add to a queue that will cause
+it to be updated by a regular index cron job in the future.
+
+# TODO implement in the future - I don't know the best way of doing this yet.
+# If fork: make sure process group is changed so apache doesn't wait for us.
+
+=cut
+
+sub update_index_background {
+    my $self = shift;
+    $self->update_index(@_);
+}
+
+=head2 $indexer->delete_index($biblionums)
+
+C<$biblionums> is an arrayref of biblionumbers to delete from the index.
+
+=cut
+
+sub delete_index {
+    my ($self, $biblionums) = @_;
+
+    if ( !$self->store ) {
+        my $params  = $self->get_elasticsearch_params();
+        $self->store(
+            Catmandu::Store::ElasticSearch->new(
+                %$params,
+                index_settings => $self->get_elasticsearch_settings(),
+                index_mappings => $self->get_elasticsearch_mappings(),
+            )
+        );
+    }
+    $self->store->bag->delete($_) foreach @$biblionums;
+    $self->store->bag->commit;
+}
+
+=head2 $indexer->delete_index_background($biblionums)
+
+Identical to L<delete_index>, this will return immediately and start a
+background process to do the actual deleting.
+
+=cut
+
+# TODO implement in the future
+
+sub delete_index_background {
+    my $self = shift;
+    $self->delete_index(@_);
+}
+
+=head2 $indexer->drop_index();
+
+Drops the index from the elasticsearch server. Calling C<update_index>
+after this will recreate it again.
+
+=cut
+
+sub drop_index {
+    my ($self) = @_;
+
+    if (!$self->store) {
+        # If this index doesn't exist, this will create it. Then it'll be
+        # deleted. That's not the end of the world however.
+        my $params  = $self->get_elasticsearch_params();
+        $self->store(
+            Catmandu::Store::ElasticSearch->new(
+                %$params,
+                index_settings => $self->get_elasticsearch_settings(),
+                index_mappings => $self->get_elasticsearch_mappings(),
+            )
+        );
+    }
+    $self->store->drop();
+    $self->store(undef);
+}
+
+sub _sanitise_records {
+    my ($self, $biblionums, $records) = @_;
+
+    confess "Unequal number of values in \$biblionums and \$records." if (@$biblionums != @$records);
+
+    my $c = @$biblionums;
+    for (my $i=0; $i<$c; $i++) {
+        my $bibnum = $biblionums->[$i];
+        my $rec = $records->[$i];
+        # I've seen things you people wouldn't believe. Attack ships on fire
+        # off the shoulder of Orion. I watched C-beams glitter in the dark near
+        # the Tannhauser gate. MARC records where 999$c doesn't match the
+        # biblionumber column. All those moments will be lost in time... like
+        # tears in rain...
+        $rec->delete_fields($rec->field('999'));
+        $rec->append_fields(MARC::Field->new('999','','','c' => $bibnum, 'd' => $bibnum));
+    }
+}
+
+sub _convert_marc_to_json {
+    my $self    = shift;
+    my $records = shift;
+    my $importer =
+      Catmandu::Importer::MARC->new( records => $records, id => '999c' );
+    my $fixer = Catmandu::Fix->new( fixes => $self->get_fixer_rules() );
+    $importer = $fixer->fix($importer);
+    return $importer;
+}
+
+1;
+
+__END__
+
+=head1 AUTHOR
+
+=over 4
+
+=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
+
+=item Robin Sheat C<< <robin@catalyst.net.nz> >>
+
+=back

Koha/ItemType.pm

@@ -89,4 +89,8 @@ sub _type {
     return 'Itemtype';
 }
 
+sub type {
+    return 'Itemtype';
+}
+
 1;

Koha/ItemTypes.pm

@@ -43,6 +43,10 @@ sub _type {
     return 'Itemtype';
 }
 
+sub type {
+    return 'Itemtype';
+}
+
 sub object_class {
     return 'Koha::ItemType';
 }

Koha/MetadataIterator.pm

@@ -0,0 +1,74 @@
+package Koha::MetadataIterator;
+
+# This contains an iterator over biblio and authority records
+
+# Copyright 2014 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+=head1 NAME
+
+Koha::MetadataIterator - iterates over records
+
+=head1 DESCRIPTION
+
+This provides a fairly generic iterator that will return records provided
+by a function.
+
+=head1 SYNOPSIS
+
+    use Koha::MetadataIterator;
+    my $next_func = sub {
+        # something that'll return each record
+    };
+    my $iterator = Koha::MetadataIterator->new($next_func);
+    while ( my $record = $iterator->next() ) {
+        # do something with $record
+    }
+
+=head1 METHODS
+
+=cut
+
+use Modern::Perl;
+
+=head2 new
+
+    my $it = new($next_func);
+
+Takes a function that will provide the next bit of data.
+
+=cut
+
+sub new {
+    my ( $class, $next_func ) = @_;
+
+    bless { next_func => $next_func, }, $class;
+}
+
+=head2 next()
+
+Provides the next record.
+
+=cut
+
+sub next {
+    my ($self) = @_;
+
+    return $self->{next_func}->();
+}
+
+1;

Koha/Objects.pm

@@ -89,6 +89,24 @@ sub find {
     return $object;
 }
 
+=head3 Koha::Objects->find_or_create();
+
+my $object = Koha::Objects->find_or_create( $attrs );
+
+=cut
+
+sub find_or_create {
+    my ( $self, $params ) = @_;
+
+    my $result = $self->_resultset->find_or_create($params);
+
+    return unless $result;
+
+    my $object = $self->object_class->_new_from_dbic($result);
+
+    return $object;
+}
+
 =head3 Koha::Objects->search();
 
 my @objects = Koha::Objects->search($params);

Koha/Schema/Result/SearchField.pm

@@ -0,0 +1,122 @@
+use utf8;
+package Koha::Schema::Result::SearchField;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+=head1 NAME
+
+Koha::Schema::Result::SearchField
+
+=cut
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+=head1 TABLE: C<search_field>
+
+=cut
+
+__PACKAGE__->table("search_field");
+
+=head1 ACCESSORS
+
+=head2 id
+
+  data_type: 'integer'
+  is_auto_increment: 1
+  is_nullable: 0
+
+=head2 name
+
+  data_type: 'varchar'
+  is_nullable: 0
+  size: 255
+
+the name of the field as it will be stored in the search engine
+
+=head2 label
+
+  data_type: 'varchar'
+  is_nullable: 0
+  size: 255
+
+the human readable name of the field, for display
+
+=head2 type
+
+  data_type: 'enum'
+  extra: {list => ["string","date","number","boolean","sum"]}
+  is_nullable: 0
+
+what type of data this holds, relevant when storing it in the search engine
+
+=cut
+
+__PACKAGE__->add_columns(
+  "id",
+  { data_type => "integer", is_auto_increment => 1, is_nullable => 0 },
+  "name",
+  { data_type => "varchar", is_nullable => 0, size => 255 },
+  "label",
+  { data_type => "varchar", is_nullable => 0, size => 255 },
+  "type",
+  {
+    data_type => "enum",
+    extra => { list => ["string", "date", "number", "boolean", "sum"] },
+    is_nullable => 0,
+  },
+);
+
+=head1 PRIMARY KEY
+
+=over 4
+
+=item * L</id>
+
+=back
+
+=cut
+
+__PACKAGE__->set_primary_key("id");
+
+=head1 UNIQUE CONSTRAINTS
+
+=head2 C<name>
+
+=over 4
+
+=item * L</name>
+
+=back
+
+=cut
+
+__PACKAGE__->add_unique_constraint("name", ["name"]);
+
+=head1 RELATIONS
+
+=head2 search_marc_to_fields
+
+Type: has_many
+
+Related object: L<Koha::Schema::Result::SearchMarcToField>
+
+=cut
+
+__PACKAGE__->has_many(
+  "search_marc_to_fields",
+  "Koha::Schema::Result::SearchMarcToField",
+  { "foreign.search_field_id" => "self.id" },
+  { cascade_copy => 0, cascade_delete => 0 },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.07042 @ 2015-10-12 16:41:47
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:4MJO9216VF37w7PlWCcBKg
+
+__PACKAGE__->many_to_many("search_marc_maps", "search_marc_to_fields", "search_marc_map");
+
+1;

Koha/Schema/Result/SearchMarcMap.pm

@@ -0,0 +1,130 @@
+use utf8;
+package Koha::Schema::Result::SearchMarcMap;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+=head1 NAME
+
+Koha::Schema::Result::SearchMarcMap
+
+=cut
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+=head1 TABLE: C<search_marc_map>
+
+=cut
+
+__PACKAGE__->table("search_marc_map");
+
+=head1 ACCESSORS
+
+=head2 id
+
+  data_type: 'integer'
+  is_auto_increment: 1
+  is_nullable: 0
+
+=head2 index_name
+
+  data_type: 'enum'
+  extra: {list => ["biblios","authorities"]}
+  is_nullable: 0
+
+what storage index this map is for
+
+=head2 marc_type
+
+  data_type: 'enum'
+  extra: {list => ["marc21","unimarc","normarc"]}
+  is_nullable: 0
+
+what MARC type this map is for
+
+=head2 marc_field
+
+  data_type: 'varchar'
+  is_nullable: 0
+  size: 255
+
+the MARC specifier for this field
+
+=cut
+
+__PACKAGE__->add_columns(
+  "id",
+  { data_type => "integer", is_auto_increment => 1, is_nullable => 0 },
+  "index_name",
+  {
+    data_type => "enum",
+    extra => { list => ["biblios", "authorities"] },
+    is_nullable => 0,
+  },
+  "marc_type",
+  {
+    data_type => "enum",
+    extra => { list => ["marc21", "unimarc", "normarc"] },
+    is_nullable => 0,
+  },
+  "marc_field",
+  { data_type => "varchar", is_nullable => 0, size => 255 },
+);
+
+=head1 PRIMARY KEY
+
+=over 4
+
+=item * L</id>
+
+=back
+
+=cut
+
+__PACKAGE__->set_primary_key("id");
+
+=head1 UNIQUE CONSTRAINTS
+
+=head2 C<index_name>
+
+=over 4
+
+=item * L</index_name>
+
+=item * L</marc_field>
+
+=item * L</marc_type>
+
+=back
+
+=cut
+
+__PACKAGE__->add_unique_constraint("index_name", ["index_name", "marc_field", "marc_type"]);
+
+=head1 RELATIONS
+
+=head2 search_marc_to_fields
+
+Type: has_many
+
+Related object: L<Koha::Schema::Result::SearchMarcToField>
+
+=cut
+
+__PACKAGE__->has_many(
+  "search_marc_to_fields",
+  "Koha::Schema::Result::SearchMarcToField",
+  { "foreign.search_marc_map_id" => "self.id" },
+  { cascade_copy => 0, cascade_delete => 0 },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.07042 @ 2015-10-12 16:41:47
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:nKMOxnAJST3zNN6Kxj2ynA
+
+__PACKAGE__->many_to_many("search_fields", "search_marc_to_fields", "search_field");
+
+1;

Koha/Schema/Result/SearchMarcToField.pm

@@ -0,0 +1,128 @@
+use utf8;
+package Koha::Schema::Result::SearchMarcToField;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+=head1 NAME
+
+Koha::Schema::Result::SearchMarcToField
+
+=cut
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+=head1 TABLE: C<search_marc_to_field>
+
+=cut
+
+__PACKAGE__->table("search_marc_to_field");
+
+=head1 ACCESSORS
+
+=head2 search_marc_map_id
+
+  data_type: 'integer'
+  is_foreign_key: 1
+  is_nullable: 0
+
+=head2 search_field_id
+
+  data_type: 'integer'
+  is_foreign_key: 1
+  is_nullable: 0
+
+=head2 facet
+
+  data_type: 'tinyint'
+  default_value: 0
+  is_nullable: 1
+
+true if a facet field should be generated for this
+
+=head2 suggestible
+
+  data_type: 'tinyint'
+  default_value: 0
+  is_nullable: 1
+
+true if this field can be used to generate suggestions for browse
+
+=head2 sort
+
+  data_type: 'tinyint'
+  is_nullable: 1
+
+true/false creates special sort handling, null doesn't
+
+=cut
+
+__PACKAGE__->add_columns(
+  "search_marc_map_id",
+  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
+  "search_field_id",
+  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
+  "facet",
+  { data_type => "tinyint", default_value => 0, is_nullable => 1 },
+  "suggestible",
+  { data_type => "tinyint", default_value => 0, is_nullable => 1 },
+  "sort",
+  { data_type => "tinyint", is_nullable => 1 },
+);
+
+=head1 PRIMARY KEY
+
+=over 4
+
+=item * L</search_marc_map_id>
+
+=item * L</search_field_id>
+
+=back
+
+=cut
+
+__PACKAGE__->set_primary_key("search_marc_map_id", "search_field_id");
+
+=head1 RELATIONS
+
+=head2 search_field
+
+Type: belongs_to
+
+Related object: L<Koha::Schema::Result::SearchField>
+
+=cut
+
+__PACKAGE__->belongs_to(
+  "search_field",
+  "Koha::Schema::Result::SearchField",
+  { id => "search_field_id" },
+  { is_deferrable => 1, on_delete => "CASCADE", on_update => "CASCADE" },
+);
+
+=head2 search_marc_map
+
+Type: belongs_to
+
+Related object: L<Koha::Schema::Result::SearchMarcMap>
+
+=cut
+
+__PACKAGE__->belongs_to(
+  "search_marc_map",
+  "Koha::Schema::Result::SearchMarcMap",
+  { id => "search_marc_map_id" },
+  { is_deferrable => 1, on_delete => "CASCADE", on_update => "CASCADE" },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.07042 @ 2015-10-12 16:41:47
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ccnCYwDdtPGZ/VArAe1kPA
+
+
+# You can replace this text with custom code or comments, and it will be preserved on regeneration
+1;

Koha/SearchEngine.pm

@@ -0,0 +1,46 @@
+package Koha::SearchEngine;
+# This handles generic search-engine related functions
+
+# Copyright 2015 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use Modern::Perl;
+use Readonly;
+
+=head1 NAME
+
+Koha::SearchEngine - non-engine-specific data and functions
+
+=head1 VARIABLES / CONSTANTS
+
+=head2 BIBLIOS_INDEX
+
+Use this constant when creating a new L<Koha::SearchEngine::Search> instance
+to indicate that you want to be working with the biblio index.
+
+=head2 AUTHORITIES_INDEX
+
+Use this constant when creating a new L<Koha::SearchEngine::Search> instance to
+indicate that you want to be working with the authorities index.
+
+=cut
+
+# Search engine implementations should compare against these to determine
+# what bit of storage is being requested. They will be sensible strings so
+# may be used for, e.g., directory names.
+Readonly our $BIBLIOS_INDEX     => 'biblios';
+Readonly our $AUTHORITIES_INDEX => 'authorities';

Koha/SearchEngine/Elasticsearch/QueryBuilder.pm

@@ -0,0 +1,771 @@
+package Koha::SearchEngine::Elasticsearch::QueryBuilder;
+
+# This file is part of Koha.
+#
+# Copyright 2014 Catalyst IT Ltd.
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+
+=head1 NAME
+
+Koha::SearchEngine::Elasticsearch::QueryBuilder - constructs elasticsearch
+query objects from user-supplied queries
+
+=head1 DESCRIPTION
+
+This provides the functions that take a user-supplied search query, and
+provides something that can be given to elasticsearch to get answers.
+
+=head1 SYNOPSIS
+
+    use Koha::SearchEngine::Elasticsearch::QueryBuilder;
+    $builder = Koha::SearchEngine::Elasticsearch->new({ index => $index });
+    my $simple_query = $builder->build_query("hello");
+    # This is currently undocumented because the original code is undocumented
+    my $adv_query = $builder->build_advanced_query($indexes, $operands, $operators);
+
+=head1 METHODS
+
+=cut
+
+use base qw(Koha::ElasticSearch);
+use Carp;
+use JSON;
+use List::MoreUtils qw/ each_array /;
+use Modern::Perl;
+use URI::Escape;
+
+use C4::Context;
+use Data::Dumper;    # TODO remove
+
+=head2 build_query
+
+    my $simple_query = $builder->build_query("hello", %options)
+
+This will build a query that can be issued to elasticsearch from the provided
+string input. This expects a lucene style search form (see
+L<http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax>
+for details.)
+
+It'll make an attempt to respect the various query options.
+
+Additional options can be provided with the C<%options> hash.
+
+=over 4
+
+=item sort
+
+This should be an arrayref of hashrefs, each containing a C<field> and an
+C<direction> (optional, defaults to C<asc>.) The results will be sorted
+according to these values. Valid values for C<direction> are 'asc' and 'desc'.
+
+=back
+
+=cut
+
+sub build_query {
+    my ( $self, $query, %options ) = @_;
+
+    my $stemming         = C4::Context->preference("QueryStemming")        || 0;
+    my $auto_truncation  = C4::Context->preference("QueryAutoTruncate")    || 0;
+    my $weight_fields    = C4::Context->preference("QueryWeightFields")    || 0;
+    my $fuzzy_enabled    = C4::Context->preference("QueryFuzzy")           || 0;
+
+    $query = '*' unless defined $query;
+
+    my $res;
+    $res->{query} = {
+        query_string => {
+            query            => $query,
+            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
+            default_operator => 'AND',
+            default_field    => '_all',
+            lenient          => JSON::true,
+        }
+    };
+
+    if ( $options{sort} ) {
+        foreach my $sort ( @{ $options{sort} } ) {
+            my ( $f, $d ) = @$sort{qw/ field direction /};
+            die "Invalid sort direction, $d"
+              if $d && ( $d ne 'asc' && $d ne 'desc' );
+            $d = 'asc' unless $d;
+
+            # TODO account for fields that don't have a 'phrase' type
+
+            $f = $self->_sort_field($f);
+            push @{ $res->{sort} }, { "$f.phrase" => { order => $d } };
+        }
+    }
+
+    # See _convert_facets in Search.pm for how these get turned into
+    # things that Koha can use.
+    $res->{facets} = {
+        author   => { terms => { field => "author__facet" } },
+        subject  => { terms => { field => "subject__facet" } },
+        itype    => { terms => { field => "itype__facet" } },
+        location => { terms => { field => "homebranch__facet" } },
+        'su-geo' => { terms => { field => "su-geo__facet" } },
+        se       => { terms => { field => "se__facet" } },
+    };
+    if ( my $ef = $options{expanded_facet} ) {
+        $res->{facets}{$ef}{terms}{size} = C4::Context->preference('FacetMaxCount');
+    };
+    return $res;
+}
+
+=head2 build_browse_query
+
+    my $browse_query = $builder->build_browse_query($field, $query);
+
+This performs a "starts with" style query on a particular field. The field
+to be searched must have been indexed with an appropriate mapping as a
+"phrase" subfield, which pretty much everything has.
+
+=cut
+
+# XXX this isn't really a browse query like we want in the end
+sub build_browse_query {
+    my ( $self, $field, $query ) = @_;
+
+    my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0;
+
+    return { query => '*' } if !defined $query;
+
+    # TODO this should come from Koha::Elasticsearch
+    my %field_whitelist = (
+        title  => 1,
+        author => 1,
+    );
+    $field = 'title' if !exists $field_whitelist{$field};
+    my $sort = $self->_sort_field($field);
+    my $res = {
+        query => {
+            match_phrase_prefix => {
+                "$field.phrase" => {
+                    query     => $query,
+                    operator  => 'or',
+                    fuzziness => $fuzzy_enabled ? 'auto' : '0',
+                }
+            }
+        },
+        sort => [ { "$sort.phrase" => { order => "asc" } } ],
+    };
+}
+
+=head2 build_query_compat
+
+    my (
+        $error,             $query, $simple_query, $query_cgi,
+        $query_desc,        $limit, $limit_cgi,    $limit_desc,
+        $stopwords_removed, $query_type
+      )
+      = $builder->build_query_compat( \@operators, \@operands, \@indexes,
+        \@limits, \@sort_by, $scan, $lang );
+
+This handles a search using the same api as L<C4::Search::buildQuery> does.
+
+A very simple query will go in with C<$operands> set to ['query'], and
+C<$sort_by> set to ['pubdate_dsc']. This simple case will return with
+C<$query> set to something that can perform the search, C<$simple_query>
+set to just the search term, C<$query_cgi> set to something that can
+reproduce this search, and C<$query_desc> set to something else.
+
+=cut
+
+sub build_query_compat {
+    my ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan,
+        $lang, $params )
+      = @_;
+
+#die Dumper ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan, $lang );
+    my @sort_params  = $self->_convert_sort_fields(@$sort_by);
+    my @index_params = $self->_convert_index_fields(@$indexes);
+    my $limits       = $self->_fix_limit_special_cases($orig_limits);
+
+    # Merge the indexes in with the search terms and the operands so that
+    # each search thing is a handy unit.
+    unshift @$operators, undef;    # The first one can't have an op
+    my @search_params;
+    my $ea = each_array( @$operands, @$operators, @index_params );
+    while ( my ( $oand, $otor, $index ) = $ea->() ) {
+        next if ( !defined($oand) || $oand eq '' );
+        push @search_params, {
+            operand => $self->_clean_search_term($oand),      # the search terms
+            operator => defined($otor) ? uc $otor : undef,    # AND and so on
+            $index ? %$index : (),
+        };
+    }
+
+    # We build a string query from limits and the queries. An alternative
+    # would be to pass them separately into build_query and let it build
+    # them into a structured ES query itself. Maybe later, though that'd be
+    # more robust.
+    my $query_str = join( ' AND ',
+        join( ' ', $self->_create_query_string(@search_params) ) || (),
+        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
+
+    # If there's no query on the left, let's remove the junk left behind
+    $query_str =~ s/^ AND //;
+    my %options;
+    $options{sort} = \@sort_params;
+    $options{expanded_facet} = $params->{expanded_facet};
+    my $query = $self->build_query( $query_str, %options );
+
+    #die Dumper($query);
+    # We roughly emulate the CGI parameters of the zebra query builder
+    my $query_cgi;
+    $query_cgi = 'idx=kw&q=' . uri_escape_utf8( $operands->[0] ) if @$operands;
+    my $simple_query;
+    $simple_query = $operands->[0] if @$operands == 1;
+    my $query_desc   = $simple_query;
+    my $limit        = $self->_join_queries( $self->_convert_index_strings(@$limits));
+    my $limit_cgi = ( $orig_limits and @$orig_limits )
+      ? '&limit=' . join( '&limit=', map { uri_escape_utf8($_) } @$orig_limits )
+      : '';
+    my $limit_desc;
+    $limit_desc = "$limit" if $limit;
+    return (
+        undef,  $query,     $simple_query, $query_cgi, $query_desc,
+        $limit, $limit_cgi, $limit_desc,   undef,      undef
+    );
+}
+
+=head2 build_authorities_query
+
+    my $query = $builder->build_authorities_query(\%search);
+
+This takes a nice description of an authority search and turns it into a black-box
+query that can then be passed to the appropriate searcher.
+
+The search description is a hashref that looks something like:
+
+    {
+        searches => [
+            {
+                where    => 'Heading',    # search the main entry
+                operator => 'exact',        # require an exact match
+                value    => 'frogs',        # the search string
+            },
+            {
+                where    => '',             # search all entries
+                operator => '',             # default keyword, right truncation
+                value    => 'pond',
+            },
+        ],
+        sort => {
+            field => 'Heading',
+            order => 'desc',
+        },
+        authtypecode => 'TOPIC_TERM',
+    }
+
+=cut
+
+sub build_authorities_query {
+    my ( $self, $search ) = @_;
+
+    # Start by making the query parts
+    my @query_parts;
+    my @filter_parts;
+    foreach my $s ( @{ $search->{searches} } ) {
+        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
+        $wh = '_all' if $wh eq '';
+        if ( $op eq 'is' || $op eq '=' ) {
+
+            # look for something that matches completely
+            # note, '=' is about numerical vals. May need special handling.
+            # _allphrase is a special field that only groups the exact
+            # matches. Also, we lowercase our search because the ES
+            # index lowercases its values, and term searches don't get the
+            # search analyzer applied to them.
+            push @filter_parts, { term => { "$wh.phrase" => lc $val } };
+        }
+        elsif ( $op eq 'exact' ) {
+
+            # left and right truncation, otherwise an exact phrase
+            push @query_parts, { match_phrase => { $wh => $val } };
+        }
+        elsif ( $op eq 'start' ) {
+
+            # startswith search
+            push @query_parts, { wildcard => { "$wh.phrase" => lc "$val*" } };
+        }
+        else {
+            # regular wordlist stuff
+            push @query_parts, { match => { $wh => $val } };
+        }
+    }
+
+    # Merge the query and filter parts appropriately
+    # 'should' behaves like 'or', if we want 'and', use 'must'
+    my $query_part  = { bool => { should => \@query_parts } };
+    my $filter_part = { bool => { should => \@filter_parts } };
+
+    # We need to add '.phrase' to all the sort headings otherwise it'll sort
+    # based on the tokenised form.
+    my %s;
+    if ( exists $search->{sort} ) {
+        foreach my $k ( keys %{ $search->{sort} } ) {
+            my $f = $self->_sort_field($k);
+            $s{"$f.phrase"} = $search->{sort}{$k};
+        }
+        $search->{sort} = \%s;
+    }
+
+    # extract the sort stuff
+    my %sort;
+    %sort = ( sort => [ $search->{sort} ] ) if exists $search->{sort};
+    my $query;
+    if (@filter_parts) {
+        $query =
+          { query =>
+              { filtered => { filter => $filter_part, query => $query_part } }
+          };
+    }
+    else {
+        $query = { query => $query_part };
+    }
+    $query = { %$query, %sort };
+    return $query;
+}
+
+
+=head2 build_authorities_query_compat
+
+    my ($query) =
+      $builder->build_authorities_query_compat( \@marclist, \@and_or,
+        \@excluding, \@operator, \@value, $authtypecode, $orderby );
+
+This builds a query for searching for authorities, in the style of
+L<C4::AuthoritiesMarc::SearchAuthorities>.
+
+Arguments:
+
+=over 4
+
+=item marclist
+
+An arrayref containing where the particular term should be searched for.
+Options are: mainmainentry, mainentry, match, match-heading, see-from, and
+thesaurus. If left blank, any field is used.
+
+=item and_or
+
+Totally ignored. It is never used in L<C4::AuthoritiesMarc::SearchAuthorities>.
+
+=item excluding
+
+Also ignored.
+
+=item operator
+
+What form of search to do. Options are: is (phrase, no trunction, whole field
+must match), = (number exact match), exact (phrase, but with left and right
+truncation). If left blank, then word list, right truncted, anywhere is used.
+
+=item value
+
+The actual user-provided string value to search for.
+
+=item authtypecode
+
+The authority type code to search within. If blank, then all will be searched.
+
+=item orderby
+
+The order to sort the results by. Options are Relevance, HeadingAsc,
+HeadingDsc, AuthidAsc, AuthidDsc.
+
+=back
+
+marclist, operator, and value must be the same length, and the values at
+index /i/ all relate to each other.
+
+This returns a query, which is a black box object that can be passed to the
+appropriate search object.
+
+=cut
+
+sub build_authorities_query_compat {
+    my ( $self, $marclist, $and_or, $excluding, $operator, $value,
+        $authtypecode, $orderby )
+      = @_;
+
+    # This turns the old-style many-options argument form into a more
+    # extensible hash form that is understood by L<build_authorities_query>.
+    my @searches;
+
+    my %koha_to_index_name = (
+        mainmainentry   => 'Heading-Main',
+        mainentry       => 'Heading',
+        match           => 'Match',
+        'match-heading' => 'Match-heading',
+        'see-from'      => 'Match-heading-see-from',
+        thesaurus       => 'Subject-heading-thesaurus',
+        any              => '',
+    );
+
+    # Make sure everything exists
+    foreach my $m (@$marclist) {
+        confess "Invalid marclist field provided: $m" unless exists $koha_to_index_name{$m};
+    }
+    for ( my $i = 0 ; $i < @$value ; $i++ ) {
+        push @searches,
+          {
+            where    => $koha_to_index_name{$marclist->[$i]},
+            operator => $operator->[$i],
+            value    => $value->[$i],
+          };
+    }
+
+    my %sort;
+    my $sort_field =
+        ( $orderby =~ /^Heading/ ) ? 'Heading'
+      : ( $orderby =~ /^Auth/ )    ? 'Local-Number'
+      :                              undef;
+    if ($sort_field) {
+        my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';
+        %sort = ( $sort_field => $sort_order, );
+    }
+    my %search = (
+        searches     => \@searches,
+        authtypecode => $authtypecode,
+    );
+    $search{sort} = \%sort if %sort;
+    my $query = $self->build_authorities_query( \%search );
+    return $query;
+}
+
+=head2 _convert_sort_fields
+
+    my @sort_params = _convert_sort_fields(@sort_by)
+
+Converts the zebra-style sort index information into elasticsearch-style.
+
+C<@sort_by> is the same as presented to L<build_query_compat>, and it returns
+something that can be sent to L<build_query>.
+
+=cut
+
+sub _convert_sort_fields {
+    my ( $self, @sort_by ) = @_;
+
+    # Turn the sorting into something we care about.
+    my %sort_field_convert = (
+        acqdate     => 'acqdate',
+        author      => 'author',
+        call_number => 'callnum',
+        popularity  => 'issues',
+        relevance   => undef,       # default
+        title       => 'title',
+        pubdate     => 'pubdate',
+    );
+    my %sort_order_convert =
+      ( qw( dsc desc ), qw( asc asc ), qw( az asc ), qw( za desc ) );
+
+    # Convert the fields and orders, drop anything we don't know about.
+    grep { $_->{field} } map {
+        my ( $f, $d ) = split /_/;
+        {
+            field     => $sort_field_convert{$f},
+            direction => $sort_order_convert{$d}
+        }
+    } @sort_by;
+}
+
+=head2 _convert_index_fields
+
+    my @index_params = $self->_convert_index_fields(@indexes);
+
+Converts zebra-style search index notation into elasticsearch-style.
+
+C<@indexes> is an array of index names, as presented to L<build_query_compat>,
+and it returns something that can be sent to L<build_query>.
+
+B<TODO>: this will pull from the elasticsearch mappings table to figure out
+types.
+
+=cut
+
+our %index_field_convert = (
+    'kw'      => '_all',
+    'ti'      => 'title',
+    'au'      => 'author',
+    'su'      => 'subject',
+    'nb'      => 'isbn',
+    'se'      => 'title-series',
+    'callnum' => 'callnum',
+    'itype'   => 'itype',
+    'ln'      => 'ln',
+    'branch'  => 'homebranch',
+    'fic'     => 'lf',
+    'mus'     => 'rtype',
+    'aud'     => 'ta',
+    'hi'      => 'Host-Item-Number',
+);
+
+sub _convert_index_fields {
+    my ( $self, @indexes ) = @_;
+
+    my %index_type_convert =
+      ( __default => undef, phr => 'phrase', rtrn => 'right-truncate' );
+
+    # Convert according to our table, drop anything that doesn't convert.
+    # If a field starts with mc- we save it as it's used (and removed) later
+    # when joining things, to indicate we make it an 'OR' join.
+    # (Sorry, this got a bit ugly after special cases were found.)
+    grep { $_->{field} } map {
+        my ( $f, $t ) = split /,/;
+        my $mc = '';
+        if ($f =~ /^mc-/) {
+            $mc = 'mc-';
+            $f =~ s/^mc-//;
+        }
+        my $r = {
+            field => $index_field_convert{$f},
+            type  => $index_type_convert{ $t // '__default' }
+        };
+        $r->{field} = ($mc . $r->{field}) if $mc && $r->{field};
+        $r;
+    } @indexes;
+}
+
+=head2 _convert_index_strings
+
+    my @searches = $self->_convert_index_strings(@searches);
+
+Similar to L<_convert_index_fields>, this takes strings of the form
+B<field:search term> and rewrites the field from zebra-style to
+elasticsearch-style. Anything it doesn't understand is returned verbatim.
+
+=cut
+
+sub _convert_index_strings {
+    my ( $self, @searches ) = @_;
+    my @res;
+    foreach my $s (@searches) {
+        next if $s eq '';
+        my ( $field, $term ) = $s =~ /^\s*([\w,-]*?):(.*)/;
+        unless ( defined($field) && defined($term) ) {
+            push @res, $s;
+            next;
+        }
+        my ($conv) = $self->_convert_index_fields($field);
+        unless ( defined($conv) ) {
+            push @res, $s;
+            next;
+        }
+        push @res, $conv->{field} . ":"
+          . $self->_modify_string_by_type( %$conv, operand => $term );
+    }
+    return @res;
+}
+
+=head2 _convert_index_strings_freeform
+
+    my $search = $self->_convert_index_strings_freeform($search);
+
+This is similar to L<_convert_index_strings>, however it'll search out the
+things to change within the string. So it can handle strings such as
+C<(su:foo) AND (su:bar)>, converting the C<su> appropriately.
+
+If there is something of the form "su,complete-subfield" or something, the
+second part is stripped off as we can't yet handle that. Making it work
+will have to wait for a real query parser.
+
+=cut
+
+sub _convert_index_strings_freeform {
+    my ( $self, $search ) = @_;
+    while ( my ( $zeb, $es ) = each %index_field_convert ) {
+        $search =~ s/\b$zeb(?:,[\w-]*)?:/$es:/g;
+    }
+    return $search;
+}
+
+=head2 _modify_string_by_type
+
+    my $str = $self->_modify_string_by_type(%index_field);
+
+If you have a search term (operand) and a type (phrase, right-truncated), this
+will convert the string to have the function in lucene search terms, e.g.
+wrapping quotes around it.
+
+=cut
+
+sub _modify_string_by_type {
+    my ( $self, %idx ) = @_;
+
+    my $type = $idx{type} || '';
+    my $str = $idx{operand};
+    return $str unless $str;    # Empty or undef, we can't use it.
+
+    $str .= '*' if $type eq 'right-truncate';
+    $str = '"' . $str . '"' if $type eq 'phrase';
+    return $str;
+}
+
+=head2 _join_queries
+
+    my $query_str = $self->_join_queries(@query_parts);
+
+This takes a list of query parts, that might be search terms on their own, or
+booleaned together, or specifying fields, or whatever, wraps them in
+parentheses, and ANDs them all together. Suitable for feeding to the ES
+query string query.
+
+Note: doesn't AND them together if they specify an index that starts with "mc"
+as that was a special case in the original code for dealing with multiple
+choice options (you can't search for something that has an itype of A and
+and itype of B otherwise.)
+
+=cut
+
+sub _join_queries {
+    my ( $self, @parts ) = @_;
+
+    my @norm_parts = grep { defined($_) && $_ ne '' && $_ !~ /^mc-/ } @parts;
+    my @mc_parts =
+      map { s/^mc-//r } grep { defined($_) && $_ ne '' && $_ =~ /^mc-/ } @parts;
+    return () unless @norm_parts + @mc_parts;
+    return ( @norm_parts, @mc_parts )[0] if @norm_parts + @mc_parts == 1;
+    my $grouped_mc =
+      @mc_parts ? '(' . ( join ' OR ', map { "($_)" } @mc_parts ) . ')' : ();
+
+    # Handy trick: $x || () inside a join means that if $x ends up as an
+    # empty string, it gets replaced with (), which makes join ignore it.
+    # (bad effect: this'll also happen to '0', this hopefully doesn't matter
+    # in this case.)
+    join( ' AND ',
+        join( ' AND ', map { "($_)" } @norm_parts ) || (),
+        $grouped_mc || () );
+}
+
+=head2 _make_phrases
+
+    my @phrased_queries = $self->_make_phrases(@query_parts);
+
+This takes the supplied queries and forces them to be phrases by wrapping
+quotes around them. It understands field prefixes, e.g. 'subject:' and puts
+the quotes outside of them if they're there.
+
+=cut
+
+sub _make_phrases {
+    my ( $self, @parts ) = @_;
+    map { s/^\s*(\w*?:)(.*)$/$1"$2"/r } @parts;
+}
+
+=head2 _create_query_string
+
+    my @query_strings = $self->_create_query_string(@queries);
+
+Given a list of hashrefs, it will turn them into a lucene-style query string.
+The hash should contain field, type (both for the indexes), operator, and
+operand.
+
+=cut
+
+sub _create_query_string {
+    my ( $self, @queries ) = @_;
+
+    map {
+        my $otor  = $_->{operator} ? $_->{operator} . ' ' : '';
+        my $field = $_->{field}    ? $_->{field} . ':'    : '';
+
+        my $oand = $self->_modify_string_by_type(%$_);
+        "$otor($field$oand)";
+    } @queries;
+}
+
+=head2 _clean_search_term
+
+    my $term = $self->_clean_search_term($term);
+
+This cleans a search term by removing any funny characters that may upset
+ES and give us an error. It also calls L<_convert_index_strings_freeform>
+to ensure those parts are correct.
+
+=cut
+
+sub _clean_search_term {
+    my ( $self, $term ) = @_;
+
+    # Some hardcoded searches (like with authorities) produce things like
+    # 'an=123', when it ought to be 'an:123' for our purposes.
+    $term =~ s/=/:/g;
+    $term = $self->_convert_index_strings_freeform($term);
+    $term =~ s/[{}]/"/g;
+    return $term;
+}
+
+=head2 _fix_limit_special_cases
+
+    my $limits = $self->_fix_limit_special_cases($limits);
+
+This converts any special cases that the limit specifications have into things
+that are more readily processable by the rest of the code.
+
+The argument should be an arrayref, and it'll return an arrayref.
+
+=cut
+
+sub _fix_limit_special_cases {
+    my ( $self, $limits ) = @_;
+
+    my @new_lim;
+    foreach my $l (@$limits) {
+
+        # This is set up by opac-search.pl
+        if ( $l =~ /^yr,st-numeric,ge=/ ) {
+            my ( $start, $end ) =
+              ( $l =~ /^yr,st-numeric,ge=(.*) and yr,st-numeric,le=(.*)$/ );
+            next unless defined($start) && defined($end);
+            push @new_lim, "copydate:[$start TO $end]";
+        }
+        elsif ( $l =~ /^yr,st-numeric=/ ) {
+            my ($date) = ( $l =~ /^yr,st-numeric=(.*)$/ );
+            next unless defined($date);
+            push @new_lim, "copydate:$date";
+        }
+        elsif ( $l =~ /^available$/ ) {
+            push @new_lim, 'onloan:false';
+        }
+        else {
+            push @new_lim, $l;
+        }
+    }
+    return \@new_lim;
+}
+
+=head2 _sort_field
+
+    my $field = $self->_sort_field($field);
+
+Given a field name, this works out what the actual name of the version to sort
+on should be. Often it's the same, sometimes it involves sticking "__sort" on
+the end. Maybe it'll be something else in the future, who knows?
+
+=cut
+
+sub _sort_field {
+    my ($self, $f) = @_;
+    if ($self->sort_fields()->{$f}) {
+        $f .= '__sort';
+    }
+    return $f;
+}
+
+1;

Koha/SearchEngine/Elasticsearch/Search.pm

@@ -0,0 +1,458 @@
+package Koha::SearchEngine::Elasticsearch::Search;
+
+# Copyright 2014 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+=head1 NAME
+
+Koha::SearchEngine::ElasticSearch::Search - search functions for Elasticsearch
+
+=head1 SYNOPSIS
+
+    my $searcher =
+      Koha::SearchEngine::ElasticSearch::Search->new( { index => $index } );
+    my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new(
+        { index => $index } );
+    my $query = $builder->build_query('perl');
+    my $results = $searcher->search($query);
+    print "There were " . $results->total . " results.\n";
+    $results->each(sub {
+        push @hits, @_[0];
+    });
+
+=head1 METHODS
+
+=cut
+
+use Modern::Perl;
+
+use base qw(Koha::ElasticSearch);
+use C4::Context;
+use Koha::ItemTypes;
+use Koha::AuthorisedValues;
+use Koha::SearchEngine::QueryBuilder;
+use MARC::Record;
+use Catmandu::Store::ElasticSearch;
+
+use Data::Dumper; #TODO remove
+use Carp qw(cluck);
+
+Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
+
+=head2 search
+
+    my $results = $searcher->search($query, $page, $count, %options);
+
+Run a search using the query. It'll return C<$count> results, starting at page
+C<$page> (C<$page> counts from 1, anything less that, or C<undef> becomes 1.)
+C<$count> is also the number of entries on a page.
+
+C<%options> is a hash containing extra options:
+
+=over 4
+
+=item offset
+
+If provided, this overrides the C<$page> value, and specifies the record as
+an offset (i.e. the number of the record to start with), rather than a page.
+
+=back
+
+Returns
+
+=cut
+
+sub search {
+    my ($self, $query, $page, $count, %options) = @_;
+
+    my $params = $self->get_elasticsearch_params();
+    my %paging;
+    # 20 is the default number of results per page
+    $paging{limit} = $count || 20;
+    # ES/Catmandu doesn't want pages, it wants a record to start from.
+    if (exists $options{offset}) {
+        $paging{start} = $options{offset};
+    } else {
+        $page = (!defined($page) || ($page <= 0)) ? 0 : $page - 1;
+        $paging{start} = $page * $paging{limit};
+    }
+    $self->store(
+        Catmandu::Store::ElasticSearch->new(
+            %$params,
+        )
+    ) unless $self->store;
+    my $results = eval {
+        $self->store->bag->search( %$query, %paging );
+    };
+    if ($@) {
+        die $self->process_error($@);
+    }
+    return $results;
+}
+
+=head2 count
+
+    my $count = $searcher->count($query);
+
+This mimics a search request, but just gets the result count instead. That's
+faster than pulling all the data in, usually.
+
+=cut
+
+sub count {
+    my ( $self, $query ) = @_;
+
+    my $params = $self->get_elasticsearch_params();
+    $self->store(
+        Catmandu::Store::ElasticSearch->new( %$params, trace_calls => 0, ) )
+      unless $self->store;
+
+    my $search = $self->store->bag->search( %$query);
+    my $count = $search->total() || 0;
+    return $count;
+}
+
+=head2 search_compat
+
+    my ( $error, $results, $facets ) = $search->search_compat(
+        $query,            $simple_query, \@sort_by,       \@servers,
+        $results_per_page, $offset,       $expanded_facet, $branches,
+        $query_type,       $scan
+      )
+
+A search interface somewhat compatible with L<C4::Search->getRecords>. Anything
+that is returned in the query created by build_query_compat will probably
+get ignored here, along with some other things (like C<@servers>.)
+
+=cut
+
+sub search_compat {
+    my (
+        $self,     $query,            $simple_query, $sort_by,
+        $servers,  $results_per_page, $offset,       $expanded_facet,
+        $branches, $query_type,       $scan
+    ) = @_;
+
+    my %options;
+    $options{offset} = $offset;
+    $options{expanded_facet} = $expanded_facet;
+    my $results = $self->search($query, undef, $results_per_page, %options);
+
+    # Convert each result into a MARC::Record
+    my (@records, $index);
+    $index = $offset; # opac-search expects results to be put in the
+        # right place in the array, according to $offset
+    $results->each(sub {
+            # The results come in an array for some reason
+            my $marc_json = $_[0]->{record};
+            my $marc = $self->json2marc($marc_json);
+            $records[$index++] = $marc;
+        });
+    # consumers of this expect a name-spaced result, we provide the default
+    # configuration.
+    my %result;
+    $result{biblioserver}{hits} = $results->total;
+    $result{biblioserver}{RECORDS} = \@records;
+    return (undef, \%result, $self->_convert_facets($results->{facets}, $expanded_facet));
+}
+
+=head2 search_auth_compat
+
+    my ( $results, $total ) =
+      $searcher->search_auth_compat( $query, $page, $count, %options );
+
+This has a similar calling convention to L<search>, however it returns its
+results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>.
+
+=cut
+
+sub search_auth_compat {
+    my $self = shift;
+
+    # TODO handle paging
+    my $database = Koha::Database->new();
+    my $schema   = $database->schema();
+    my $res      = $self->search(@_);
+    my $bib_searcher = Koha::SearchEngine::Elasticsearch::Search->new({index => 'biblios'});
+    my @records;
+    $res->each(
+        sub {
+            my %result;
+            my $record    = $_[0];
+            my $marc_json = $record->{record};
+
+            # I wonder if these should be real values defined in the mapping
+            # rather than hard-coded conversions.
+            # Our results often come through as nested arrays, to fix this
+            # requires changes in catmandu.
+            my $authid = $record->{ 'Local-number' }[0][0];
+            $result{authid} = $authid;
+
+            # TODO put all this info into the record at index time so we
+            # don't have to go and sort it all out now.
+            my $authtypecode = $record->{authtype};
+            my $rs           = $schema->resultset('AuthType')
+              ->search( { authtypecode => $authtypecode } );
+
+            # FIXME there's an assumption here that we will get a result.
+            # the original code also makes an assumption that some provided
+            # authtypecode may sometimes be used instead of the one stored
+            # with the record. It's not documented why this is the case, so
+            # it's not reproduced here yet.
+            my $authtype           = $rs->single;
+            my $auth_tag_to_report = $authtype->auth_tag_to_report;
+            my $marc               = $self->json2marc($marc_json);
+            my $mainentry          = $marc->field($auth_tag_to_report);
+            my $reported_tag;
+            if ($mainentry) {
+                foreach ( $mainentry->subfields() ) {
+                    $reported_tag .= '$' . $_->[0] . $_->[1];
+                }
+            }
+            # Turn the resultset into a hash
+            my %authtype_cols;
+            foreach my $col ($authtype->result_source->columns) {
+                $authtype_cols{$col} = $authtype->get_column($col);
+            }
+            $result{authtype}     = $authtype->authtypetext;
+            $result{reported_tag} = $reported_tag;
+
+            # Reimplementing BuildSummary is out of scope because it'll be hard
+            $result{summary} =
+              C4::AuthoritiesMarc::BuildSummary( $marc, $result{authid},
+                $authtypecode );
+            $result{used} = $self->count_auth_use($bib_searcher, $authid);
+            push @records, \%result;
+        }
+    );
+    return ( \@records, $res->total );
+}
+
+=head2 count_auth_use
+
+    my $count = $auth_searcher->count_auth_use($bib_searcher, $authid);
+
+This runs a search to determine the number of records that reference the
+specified authid. C<$bib_searcher> must be something compatible with
+elasticsearch, as the query is built in this function.
+
+=cut
+
+sub count_auth_use {
+    my ($self, $bib_searcher, $authid) = @_;
+
+    my $query = {
+        query => {
+            filtered => {
+                query  => { match_all => {} },
+                filter => { term      => { an => $authid } }
+            }
+        }
+    };
+    $bib_searcher->count($query);
+}
+
+=head2 simple_search_compat
+
+    my ( $error, $marcresults, $total_hits ) =
+      $searcher->simple_search( $query, $offset, $max_results );
+
+This is a simpler interface to the searching, intended to be similar enough to
+L<C4::Search::SimpleSearch>.
+
+Arguments:
+
+=over 4
+
+=item C<$query>
+
+A thing to search for. It could be a simple string, or something constructed
+with the appropriate QueryBuilder module.
+
+=item C<$offset>
+
+How many results to skip from the start of the results.
+
+=item C<$max_results>
+
+The max number of results to return. The default is 100 (because unlimited
+is a pretty terrible thing to do.)
+
+=back
+
+Returns:
+
+=over 4
+
+=item C<$error>
+
+if something went wrong, this'll contain some kind of error
+message.
+
+=item C<$marcresults>
+
+an arrayref of MARC::Records (note that this is different from the
+L<C4::Search> version which will return plain XML, but too bad.)
+
+=item C<$total_hits>
+
+the total number of results that this search could have returned.
+
+=back
+
+=cut
+
+sub simple_search_compat {
+    my ($self, $query, $offset, $max_results) = @_;
+
+    return ('No query entered', undef, undef) unless $query;
+
+    my %options;
+    $options{offset} = $offset // 0;
+    $max_results //= 100;
+
+    unless (ref $query) {
+        # We'll push it through the query builder to sanitise everything.
+        my $qb = Koha::SearchEngine::QueryBuilder->new({index => $self->index});
+        (undef,$query) = $qb->build_query_compat(undef, [$query]);
+    }
+    my $results = $self->search($query, undef, $max_results, %options);
+    my @records;
+    $results->each(sub {
+            # The results come in an array for some reason
+            my $marc_json = $_[0]->{record};
+            my $marc = $self->json2marc($marc_json);
+            push @records, $marc;
+        });
+    return (undef, \@records, $results->total);
+}
+
+=head2 json2marc
+
+    my $marc = $self->json2marc($marc_json);
+
+Converts the form of marc (based on its JSON, but as a Perl structure) that
+Catmandu stores into a MARC::Record object.
+
+=cut
+
+sub json2marc {
+    my ( $self, $marcjson ) = @_;
+
+    my $marc = MARC::Record->new();
+    $marc->encoding('UTF-8');
+
+    # fields are like:
+    # [ '245', '1', '2', 'a' => 'Title', 'b' => 'Subtitle' ]
+    # conveniently, this is the form that MARC::Field->new() likes
+    foreach my $field (@$marcjson) {
+        next if @$field < 5;    # Shouldn't be possible, but...
+        if ( $field->[0] eq 'LDR' ) {
+            $marc->leader( $field->[4] );
+        }
+        else {
+            my $marc_field = MARC::Field->new(@$field);
+            $marc->append_fields($marc_field);
+        }
+    }
+    return $marc;
+}
+
+=head2 _convert_facets
+
+    my $koha_facets = _convert_facets($es_facets, $expanded_facet);
+
+Converts elasticsearch facets types to the form that Koha expects.
+It expects the ES facet name to match the Koha type, for example C<itype>,
+C<au>, C<su-to>, etc.
+
+C<$expanded_facet> is the facet that we want to show FacetMaxCount entries for, rather
+than just 5 like normal.
+
+=cut
+
+sub _convert_facets {
+    my ( $self, $es, $exp_facet ) = @_;
+
+    return if !$es;
+
+    # These should correspond to the ES field names, as opposed to the CCL
+    # things that zebra uses.
+    # TODO let the library define the order using the interface.
+    my %type_to_label = (
+        author   => { order => 1, label => 'Authors', },
+        itype    => { order => 2, label => 'ItemTypes', },
+        location => { order => 3, label => 'Location', },
+        'su-geo' => { order => 4, label => 'Places', },
+        se       => { order => 5, label => 'Series', },
+        subject  => { order => 6, label => 'Topics', },
+    );
+
+    # We also have some special cases, e.g. itypes that need to show the
+    # value rather than the code.
+    my @itypes = Koha::ItemTypes->search;
+    my @locations = Koha::AuthorisedValues->search( { category => 'LOC' } );
+    my $opac = C4::Context->interface eq 'opac' ;
+    my %special = (
+        itype    => { map { $_->itemtype         => $_->description } @itypes },
+        location => { map { $_->authorised_value => ( $opac ? ( $_->lib_opac || $_->lib ) : $_->lib ) } @locations },
+    );
+    my @facets;
+    $exp_facet //= '';
+    while ( my ( $type, $data ) = each %$es ) {
+        next if !exists( $type_to_label{$type} );
+
+        # We restrict to the most popular $limit !results
+        my $limit = ( $type eq $exp_facet ) ? C4::Context->preference('FacetMaxCount') : 5;
+        my $facet = {
+            type_id    => $type . '_id',
+            expand     => $type,
+            expandable => ( $type ne $exp_facet )
+              && ( @{ $data->{terms} } > $limit ),
+            "type_label_$type_to_label{$type}{label}" => 1,
+            type_link_value                    => $type,
+            order      => $type_to_label{$type}{order},
+        };
+        $limit = @{ $data->{terms} } if ( $limit > @{ $data->{terms} } );
+        foreach my $term ( @{ $data->{terms} }[ 0 .. $limit - 1 ] ) {
+            my $t = $term->{term};
+            my $c = $term->{count};
+            my $label;
+            if ( exists( $special{$type} ) ) {
+                $label = $special{$type}->{$t} // $t;
+            }
+            else {
+                $label = $t;
+            }
+            push @{ $facet->{facets} }, {
+                facet_count       => $c,
+                facet_link_value  => $t,
+                facet_title_value => $t . " ($c)",
+                facet_label_value => $label,        # TODO either truncate this,
+                     # or make the template do it like it should anyway
+                type_link_value => $type,
+            };
+        }
+        push @facets, $facet if exists $facet->{facets};
+    }
+
+    @facets = sort { $a->{order} cmp $b->{order} } @facets;
+    return \@facets;
+}
+
+
+1;

Koha/SearchEngine/QueryBuilder.pm

@@ -0,0 +1,60 @@
+package Koha::SearchEngine::QueryBuilder;
+
+# This file is part of Koha.
+#
+# Copyright 2014 Catalyst IT
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+
+# This is a shim that gives you the appropriate query builder for your
+# system preference.
+
+=head1 NAME
+
+Koha::SearchEngine::QueryBuilder - instantiates the query builder that
+corresponds to the C<SearchEngine> system preference
+
+=head1 DESCRIPTION
+
+This allows you to be agnostic about what the search engine configuration is
+and just get whatever querybuilder you need.
+
+=head1 SYNOPSIS
+
+    use Koha::SearchEngine::QueryBuilder;
+    my $qb = Koha::SearchEngine::QueryBuilder->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
+
+=head1 METHODS
+
+=head2 new
+
+Creates a new C<QueryBuilder> of whatever the relevant type is.
+
+=cut
+
+use C4::Context;
+use Modern::Perl;
+use Carp;
+
+sub new {
+    my $engine = C4::Context->preference("SearchEngine");
+    my $file = "Koha/SearchEngine/${engine}/QueryBuilder.pm";
+    my $class = "Koha::SearchEngine::${engine}::QueryBuilder";
+    croak "SearchEngine system preference not set" unless $engine;
+    require $file;
+    shift @_;
+    return $class->new(@_);
+}
+
+1;

Koha/SearchEngine/Search.pm

@@ -0,0 +1,58 @@
+package Koha::SearchEngine::Search;
+
+# This file is part of Koha.
+#
+# Copyright 2015 Catalyst IT
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+
+# This is a shim that gives you the appropriate search object for your
+# system preference.
+
+=head1 NAME
+
+Koha::SearchEngine::Search - instantiate the search object that corresponds to
+the C<SearchEngine> system preference.
+
+=head1 DESCRIPTION
+
+This allows you to be agnostic about what the search engine configuration is
+and just get whatever search object you need.
+
+=head1 SYNOPSIS
+
+    use Koha::SearchEngine::Search;
+    my $searcher = Koha::SearchEngine::Search->new();
+
+=head1 METHODS
+
+=head2 new
+
+Creates a new C<Search> of whatever the relevant type is.
+
+=cut
+
+use C4::Context;
+use Modern::Perl;
+
+sub new {
+    my $engine = C4::Context->preference("SearchEngine") // 'Zebra';
+    my $file = "Koha/SearchEngine/${engine}/Search.pm";
+    my $class = "Koha::SearchEngine::${engine}::Search";
+    require $file;
+    shift @_;
+    return $class->new(@_);
+}
+
+1;

Koha/SearchEngine/Zebra/QueryBuilder.pm

@@ -0,0 +1,56 @@
+package Koha::SearchEngine::Zebra::QueryBuilder;
+
+# This file is part of Koha.
+#
+# Copyright 2012 BibLibre
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+
+use base qw(Class::Accessor);
+use Modern::Perl;
+use Moose::Role;
+use C4::Search;
+use C4::AuthoritiesMarc;
+
+sub build_query {
+    shift;
+    C4::Search::buildQuery @_;
+}
+
+sub build_query_compat {
+    # Because this passes directly on to C4::Search, we have no trouble being
+    # compatible.
+    build_query(@_);
+}
+
+sub build_authorities_query {
+    shift;
+    C4::AuthoritiesMarc::SearchAuthorities(@_);
+    return {
+        marclist     => $_[0],
+        and_or       => $_[1],
+        excluding    => $_[2],
+        operator     => $_[3],
+        value        => $_[4],
+        authtypecode => $_[5],
+        orderby      => $_[6],
+    };
+}
+
+sub build_authorities_query_compat {
+    # Pass straight through as well
+    build_authorities_query(@_);
+}
+
+1;

Koha/SearchEngine/Zebra/Search.pm

@@ -0,0 +1,100 @@
+package Koha::SearchEngine::Zebra::Search;
+
+# This file is part of Koha.
+#
+# Copyright 2012 BibLibre
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+
+# I don't think this ever worked right
+#use Moose::Role;
+#with 'Koha::SearchEngine::SearchRole';
+
+use Modern::Perl;
+
+use base qw(Class::Accessor);
+
+use C4::Search; # :(
+use C4::AuthoritiesMarc;
+
+=head1 NAME
+
+Koha::SearchEngine::Zebra::Search - Search implementation for Zebra
+
+=head1 METHODS
+
+=head2 search
+
+=cut
+
+sub search {
+    my ($self,$query_string) = @_;
+
+     my $query = Data::SearchEngine::Query->new(
+       count => 10,
+       page => 1,
+       query => $query_string,
+     );
+
+    my $results = $self->searchengine->search($query);
+
+    foreach my $item (@{ $results->items }) {
+        my $title = $item->get_value('ste_title');
+        #utf8::encode($title);
+        print "$title\n";
+    }
+}
+
+=head2 search_compat
+
+This passes straight through to C4::Search::getRecords.
+
+=cut
+
+sub search_compat {
+    shift; # get rid of $self
+
+    return getRecords(@_);
+}
+
+=head2 simple_search_compat
+
+This passes straight through to C4::Search::SimpleSearch.
+
+=cut
+
+
+sub simple_search_compat {
+    shift;
+    return C4::Search::SimpleSearch(@_);
+}
+
+=head2 search_auth_compat
+
+This passes the search query on to C4::AuthoritiesMarc::SearchAuthorities
+
+=cut
+
+sub search_auth_compat {
+    my ( $self, $q, $startfrom, $resperpage ) = @_;
+
+    my @params = (
+        @{$q}{ 'marclist', 'and_or', 'excluding', 'operator', 'value' },
+        $startfrom - 1,
+        $resperpage, @{$q}{ 'authtypecode', 'orderby' }
+    );
+    C4::AuthoritiesMarc::SearchAuthorities(@params);
+}
+
+1;

Koha/SearchField.pm

@@ -0,0 +1,49 @@
+package Koha::SearchField;
+
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use Modern::Perl;
+
+use Carp;
+
+use Koha::Database;
+
+use base qw(Koha::Object);
+
+=head1 NAME
+
+Koha::SearchField - Koha SearchField Object class
+
+=head1 API
+
+=head2 Class Methods
+
+=cut
+
+sub add_to_search_marc_maps {
+    my ( $self, $search_field, $params ) = @_;
+    return $self->_result()->add_to_search_marc_maps($search_field->_result, $params);
+}
+
+=head3 type
+
+=cut
+
+sub _type {
+    return 'SearchField';
+}
+
+1;

Koha/SearchFields.pm

@@ -0,0 +1,50 @@
+package Koha::SearchFields;
+
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use Modern::Perl;
+
+use Carp;
+
+use Koha::Database;
+
+use Koha::SearchField;
+
+use base qw(Koha::Objects);
+
+=head1 NAME
+
+Koha::SearchFields - Koha SearchField Object set class
+
+=head1 API
+
+=head2 Class Methods
+
+=cut
+
+=head3 type
+
+=cut
+
+sub _type {
+    return 'SearchField';
+}
+
+sub object_class {
+    return 'Koha::SearchField';
+}
+
+1;

Koha/SearchMarcMap.pm

@@ -0,0 +1,50 @@
+package Koha::SearchMarcMap;
+
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use Modern::Perl;
+
+use Carp;
+
+use Koha::Database;
+
+use base qw(Koha::Object);
+
+=head1 NAME
+
+Koha::SearchMarcMap - Koha SearchMarcMap Object class
+
+=head1 API
+
+=head2 Class Methods
+
+=cut
+
+sub add_to_search_fields {
+    my(  $self, $params) = @_;
+    $self->_result->add_to_search_fields($params);
+
+}
+
+=head3 type
+
+=cut
+
+sub _type {
+    return 'SearchMarcMap';
+}
+
+1;

Koha/SearchMarcMaps.pm

@@ -0,0 +1,50 @@
+package Koha::SearchMarcMaps;
+
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use Modern::Perl;
+
+use Carp;
+
+use Koha::Database;
+
+use Koha::SearchMarcMap;
+
+use base qw(Koha::Objects);
+
+=head1 NAME
+
+Koha::SearchMarcMaps - Koha SearchMarcMap Object set class
+
+=head1 API
+
+=head2 Class Methods
+
+=cut
+
+=head3 type
+
+=cut
+
+sub _type {
+    return 'SearchMarcMap';
+}
+
+sub object_class {
+    return 'Koha::SearchMarcMap';
+}
+
+1;

acqui/neworderbiblio.pl

@@ -68,6 +68,9 @@ use C4::Members qw/ GetMember /;
 use C4::Budgets qw/ GetBudgetHierarchy /;
 
 use Koha::Acquisition::Bookseller;
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
+use Koha::SearchEngine::QueryBuilder;
 
 my $input = new CGI;
 
@@ -99,13 +102,14 @@ my @operands = $query;
 my $QParser;
 $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
 my $builtquery;
+my $builder  = Koha::SearchEngine::QueryBuilder->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
+my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
 if ($QParser) {
     $builtquery = $query;
 } else {
-    my ( $builterror,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);
-    ( $builterror,$builtquery,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type) = buildQuery(undef,\@operands);
+    ( undef,$builtquery,undef,undef,undef,undef,undef,undef,undef,undef) = $builder->build_query_compat(undef,\@operands);
 }
-my ( $error, $marcresults, $total_hits ) = SimpleSearch( $builtquery, $results_per_page * ( $page - 1 ), $results_per_page );
+my ( $error, $marcresults, $total_hits ) = $searcher->simple_search_compat($builtquery, $results_per_page * ($page - 1), $results_per_page);
 
 if (defined $error) {
     $template->param(

authorities/authorities-home.pl

@@ -34,6 +34,8 @@ use C4::Biblio;
 use C4::Search::History;
 
 use Koha::Authority::Types;
+use Koha::SearchEngine::Search;
+use Koha::SearchEngine::QueryBuilder;
 
 my $query = new CGI;
 my $dbh   = C4::Context->dbh;
@@ -77,13 +79,25 @@ if ( $op eq "do_search" ) {
     my $startfrom      = $query->param('startfrom')      || 1;
     my $resultsperpage = $query->param('resultsperpage') || 20;
 
-    my ( $results, $total ) = SearchAuthorities(
-        [$marclist],  [$and_or],
-        [$excluding], [$operator],
-        [$value], ( $startfrom - 1 ) * $resultsperpage,
-        $resultsperpage, $authtypecode,
-        $orderby
+    my $builder = Koha::SearchEngine::QueryBuilder->new(
+        { index => $Koha::SearchEngine::AUTHORITIES_INDEX } );
+    my $searcher = Koha::SearchEngine::Search->new(
+        { index => $Koha::SearchEngine::AUTHORITIES_INDEX } );
+    my $search_query = $builder->build_authorities_query_compat(
+        [$marclist], [$and_or], [$excluding], [$operator],
+        [$value], $authtypecode, $orderby
     );
+    $startfrom = $startfrom // 0;
+    my ( $results, $total ) =
+      $searcher->search_auth_compat( $search_query, $startfrom,
+        $resultsperpage );
+    #my ( $results, $total ) = SearchAuthorities(
+    #    [$marclist],  [$and_or],
+    #    [$excluding], [$operator],
+    #    [$value], ( $startfrom - 1 ) * $resultsperpage,
+    #    $resultsperpage, $authtypecode,
+    #    $orderby
+    #);
 
 
     ( $template, $loggedinuser, $cookie ) = get_template_and_user(

authorities/authorities-list.pl

@@ -5,6 +5,8 @@ use C4::Context;
 use C4::AuthoritiesMarc;
 use utf8;
 use open qw[ :std :encoding(utf8) ];
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
 
 my $dbh=C4::Context->dbh;
 my $datatypes_query = $dbh->prepare(<<ENDSQL);
@@ -24,7 +26,8 @@ foreach my $authority (@$dataauthorities){
   my $query;
   $query= "an=".$authority->{'authid'};
   # search for biblios mapped
-  my ($err,$res,$used) = C4::Search::SimpleSearch($query,0,10);
+  my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
+  my ($err,undef,$used) = $searcher->simple_search_compat($query,0,1);
   if (defined $err) {
       $used = 0;
   }

catalogue/search.pl

@@ -154,6 +154,8 @@ use C4::Search::History;
 
 use Koha::LibraryCategories;
 use Koha::Virtualshelves;
+use Koha::SearchEngine::Search;
+use Koha::SearchEngine::QueryBuilder;
 
 use URI::Escape;
 
@@ -203,30 +205,6 @@ if($cgi->cookie("holdfor")){
     );
 }
 
-## URI Re-Writing
-# Deprecated, but preserved because it's interesting :-)
-# The same thing can be accomplished with mod_rewrite in
-# a more elegant way
-#
-#my $rewrite_flag;
-#my $uri = $cgi->url(-base => 1);
-#my $relative_url = $cgi->url(-relative=>1);
-#$uri.="/".$relative_url."?";
-#warn "URI:$uri";
-#my @cgi_params_list = $cgi->param();
-#my $url_params = $cgi->Vars;
-#
-#for my $each_param_set (@cgi_params_list) {
-#    $uri.= join "",  map "\&$each_param_set=".$_, split("\0",$url_params->{$each_param_set}) if $url_params->{$each_param_set};
-#}
-#warn "New URI:$uri";
-# Only re-write a URI if there are params or if it already hasn't been re-written
-#unless (($cgi->param('r')) || (!$cgi->param()) ) {
-#    print $cgi->redirect(     -uri=>$uri."&r=1",
-#                            -cookie => $cookie);
-#    exit;
-#}
-
 # load the branches
 my $branches = GetBranches();
 
@@ -295,6 +273,9 @@ foreach my $advanced_srch_type (@advanced_search_types) {
     }
 }
 $template->param(advancedsearchesloop => \@advancedsearchesloop);
+my $types = C4::Context->preference("AdvancedSearchTypes") || "itemtypes";
+my $advancedsearchesloop = prepare_adv_search_types($types);
+$template->param(advancedsearchesloop => $advancedsearchesloop);
 
 # The following should only be loaded if we're bringing up the advanced search template
 if ( $template_type eq 'advsearch' ) {
@@ -486,8 +467,21 @@ my $expanded_facet = $params->{'expand'};
 # Define some global variables
 my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);
 
+my $builder = Koha::SearchEngine::QueryBuilder->new(
+    { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
+my $searcher = Koha::SearchEngine::Search->new(
+    { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
+
+my @results;
+
 ## I. BUILD THE QUERY
-( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type) = buildQuery(\@operators,\@operands,\@indexes,\@limits,\@sort_by,$scan,$lang);
+(
+    $error,             $query, $simple_query, $query_cgi,
+    $query_desc,        $limit, $limit_cgi,    $limit_desc,
+    $query_type
+  )
+  = $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits,
+    \@sort_by, $scan, $lang );
 
 ## parse the query_cgi string and put it into a form suitable for <input>s
 my @query_inputs;
@@ -529,15 +523,14 @@ my $facets; # this object stores the faceted results that display on the left-ha
 my $results_hashref;
 
 eval {
-    ($error, $results_hashref, $facets) = getRecords($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$itemtypes,$query_type,$scan);
+    my $itemtypes = GetItemTypes;
+    ( $error, $results_hashref, $facets ) = $searcher->search_compat(
+        $query,            $simple_query, \@sort_by,       \@servers,
+        $results_per_page, $offset,       $expanded_facet, $branches,
+        $itemtypes,        $query_type,   $scan
+    );
 };
 
-# This sorts the facets into alphabetical order
-if ($facets) {
-    foreach my $f (@$facets) {
-        $f->{facets} = [ sort { uc($a->{facet_label_value}) cmp uc($b->{facet_label_value}) } @{ $f->{facets} } ];
-    }
-}
 if ($@ || $error) {
     $template->param(query_error => $error.$@);
     output_html_with_http_headers $cgi, $cookie, $template->output;
@@ -767,3 +760,86 @@ $template->param(
 );
 
 output_html_with_http_headers $cgi, $cookie, $template->output;
+
+
+=head2 prepare_adv_search_types
+
+    my $type = C4::Context->preference("AdvancedSearchTypes") || "itemtypes";
+    my @advanced_search_types = prepare_adv_search_types($type);
+
+Different types can be searched for in the advanced search. This takes the
+system preference that defines these types and parses it into an arrayref for
+the template.
+
+"itemtypes" is handled specially, as itemtypes aren't an authorised value.
+It also accounts for the "item-level_itypes" system preference.
+
+=cut
+
+sub prepare_adv_search_types {
+    my ($types) = @_;
+
+    my @advanced_search_types = split( /\|/, $types );
+
+    # the index parameter is different for item-level itemtypes
+    my $itype_or_itemtype =
+      ( C4::Context->preference("item-level_itypes") ) ? 'itype' : 'itemtype';
+    my $itemtypes = GetItemTypes;
+
+    my ( $cnt, @result );
+    foreach my $advanced_srch_type (@advanced_search_types) {
+        $advanced_srch_type =~ s/^\s*//;
+        $advanced_srch_type =~ s/\s*$//;
+        if ( $advanced_srch_type eq 'itemtypes' ) {
+
+       # itemtype is a special case, since it's not defined in authorized values
+            my @itypesloop;
+            foreach my $thisitemtype (
+                sort {
+                    $itemtypes->{$a}->{'description'}
+                      cmp $itemtypes->{$b}->{'description'}
+                } keys %$itemtypes
+              )
+            {
+                my %row = (
+                    number      => $cnt++,
+                    ccl         => "$itype_or_itemtype,phr",
+                    code        => $thisitemtype,
+                    description => $itemtypes->{$thisitemtype}->{'description'},
+                    imageurl    => getitemtypeimagelocation(
+                        'intranet', $itemtypes->{$thisitemtype}->{'imageurl'}
+                    ),
+                );
+                push @itypesloop, \%row;
+            }
+            my %search_code = (
+                advanced_search_type => $advanced_srch_type,
+                code_loop            => \@itypesloop
+            );
+            push @result, \%search_code;
+        }
+        else {
+            # covers all the other cases: non-itemtype authorized values
+            my $advsearchtypes = GetAuthorisedValues($advanced_srch_type);
+            my @authvalueloop;
+            for my $thisitemtype (@$advsearchtypes) {
+                my %row = (
+                    number      => $cnt++,
+                    ccl         => $advanced_srch_type,
+                    code        => $thisitemtype->{authorised_value},
+                    description => $thisitemtype->{'lib'},
+                    imageurl    => getitemtypeimagelocation(
+                        'intranet', $thisitemtype->{'imageurl'}
+                    ),
+                );
+                push @authvalueloop, \%row;
+            }
+            my %search_code = (
+                advanced_search_type => $advanced_srch_type,
+                code_loop            => \@authvalueloop
+            );
+            push @result, \%search_code;
+        }
+    }
+    return \@result;
+}

cataloguing/addbooks.pl

@@ -34,6 +34,9 @@ use C4::Output;
 use C4::Koha;
 use C4::Search;
 
+use Koha::SearchEngine::Search;
+use Koha::SearchEngine::QueryBuilder;
+
 my $input = new CGI;
 
 my $success = $input->param('biblioitem');
@@ -74,15 +77,17 @@ if ($query) {
     my $QParser;
     $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
     my $builtquery;
+    my $builder = Koha::SearchEngine::QueryBuilder->new(
+        { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
+    my $searcher = Koha::SearchEngine::Search->new(
+        { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
     if ($QParser) {
         $builtquery = $query;
     } else {
-        my ( $builterror,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);
-        ( $builterror,$builtquery,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type) = buildQuery(undef,\@operands);
+        ( undef,$builtquery,undef,undef,undef,undef,undef,undef,undef,undef) = $builder->build_query_compat(undef,\@operands);
     }
-
     # find results
-    my ( $error, $marcresults, $total_hits ) = SimpleSearch($builtquery, $results_per_page * ($page - 1), $results_per_page);
+    my ( $error, $marcresults, $total_hits ) = $searcher->simple_search_compat($builtquery, $results_per_page * ($page - 1), $results_per_page);
 
     if ( defined $error ) {
         $template->param( error => $error );

cataloguing/value_builder/marc21_linking_section.pl

@@ -35,6 +35,9 @@ use C4::Branch;
 
 use Koha::ItemTypes;
 
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
+
 my $builder = sub {
     my ( $params ) = @_;
     my $function_name = $params->{id};
@@ -179,8 +182,13 @@ my $launcher = sub {
         } else {
             $op = 'and';
         }
+        my $searcher = Koha::SearchEngine::Search->new(
+            { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
         $search = 'kw:' . $search . " $op mc-itemtype:" . $itype if $itype;
-        my ( $errors, $results, $total_hits ) = SimpleSearch( $search, $startfrom * $resultsperpage, $resultsperpage );
+        my ( $errors, $results, $total_hits ) =
+          $searcher->simple_search_compat( $search,
+            $startfrom * $resultsperpage,
+            $resultsperpage );
         if ( defined $errors ) {
             $results = [];
         }

cataloguing/value_builder/unimarc_field_4XX.pl

@@ -35,6 +35,14 @@ use C4::Branch;    # GetBranches
 
 use Koha::ItemTypes;
 
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
+
+sub plugin_parameters {
+    my ( $dbh, $record, $tagslib, $i, $tabloop ) = @_;
+    return "";
+}
+
 sub plugin_javascript {
     my ( $dbh, $record, $tagslib, $field_number, $tabloop ) = @_;
     my $function_name = $field_number;
@@ -361,7 +369,8 @@ sub plugin {
             $op = 'and';
         }
         $search = 'kw:'.$search." $op mc-itemtype:".$itype if $itype;
-        my ( $errors, $results, $total_hits ) = SimpleSearch($search, $startfrom * $resultsperpage, $resultsperpage );
+        my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
+        my ( $errors, $results, $total_hits ) = $searcher->simple_search_compat($search, $startfrom * $resultsperpage, $resultsperpage );
         if (defined $errors ) {
             $results = [];
         }

circ/circulation.pl

@@ -49,6 +49,8 @@ use Koha::DateUtils;
 use Koha::Database;
 use Koha::Patron::Messages;
 use Koha::Patron::Images;
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
 
 use Date::Calc qw(
   Today
@@ -353,8 +355,9 @@ if (@$barcodes) {
     {
      $template_params->{FALLBACK} = 1;
 
+        my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
         my $query = "kw=" . $barcode;
-        my ( $searcherror, $results, $total_hits ) = SimpleSearch($query);
+        my ( $searcherror, $results, $total_hits ) = $searcher->simple_search_compat($query, 0, 10);
 
         # if multiple hits, offer options to librarian
         if ( $total_hits > 0 ) {

installer/data/mysql/atomicupdate/elastic_search.perl

@@ -0,0 +1,88 @@
+#!/usr/bin/perl
+
+use Modern::Perl;
+
+use C4::Context;
+use YAML::Syck;
+
+use Koha::SearchMarcMaps;
+use Koha::SearchFields;
+
+my $dbh = C4::Context->dbh;
+
+
+$dbh->do(q|INSERT IGNORE INTO systempreferences (variable,value,explanation,options,type)
+        VALUES('SearchEngine','Zebra','Choose format to display postal addresses','','Choice')|);
+
+
+$dbh->do(q|DROP TABLE IF EXISTS search_marc_to_field|);
+$dbh->do(q|DROP TABLE IF EXISTS search_marc_map|);
+$dbh->do(q|DROP TABLE IF EXISTS search_field|);
+
+# This specifies the fields that will be stored in the search engine.
+$dbh->do(q|
+    CREATE TABLE `search_field` (
+      `id` int(11) NOT NULL AUTO_INCREMENT,
+      `name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',
+      `label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display',
+      `type` ENUM('string', 'date', 'number', 'boolean', 'sum') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',
+      PRIMARY KEY (`id`),
+      UNIQUE KEY (`name`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+|);
+
+# This contains a MARC field specifier for a given index, marc type, and marc
+# field.
+$dbh->do(q|
+    CREATE TABLE `search_marc_map` (
+        id int(11) NOT NULL AUTO_INCREMENT,
+        index_name ENUM('biblios','authorities') NOT NULL COMMENT 'what storage index this map is for',
+        marc_type ENUM('marc21', 'unimarc', 'normarc') NOT NULL COMMENT 'what MARC type this map is for',
+        marc_field VARCHAR(255) NOT NULL COMMENT 'the MARC specifier for this field',
+        PRIMARY KEY(`id`),
+        unique key( index_name, marc_field, marc_type),
+        INDEX (`index_name`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+|);
+
+# This joins the two search tables together. We can have any combination:
+# one marc field could have many search fields (maybe you want one value
+# to go to 'author' and 'corporate-author) and many marc fields could go
+# to one search field (e.g. all the various author fields going into
+# 'author'.)
+#
+# a note about the sort field:
+# * if all the entries for a mapping are 'null', nothing special is done with that mapping.
+# * if any of the entries are not null, then a __sort field is created in ES for this mapping. In this case:
+#   * any mapping with sort == false WILL NOT get copied into a __sort field
+#   * any mapping with sort == true or is null WILL get copied into a __sort field
+#   * any sorts on the field name will be applied to $fieldname.'__sort' instead.
+# this means that we can have search for author that includes 1xx, 245$c, and 7xx, but the sort only applies to 1xx.
+$dbh->do(q|
+    CREATE TABLE `search_marc_to_field` (
+        search_marc_map_id int(11) NOT NULL,
+        search_field_id int(11) NOT NULL,
+        facet boolean DEFAULT FALSE COMMENT 'true if a facet field should be generated for this',
+        suggestible boolean DEFAULT FALSE COMMENT 'true if this field can be used to generate suggestions for browse',
+        sort boolean DEFAULT NULL COMMENT 'true/false creates special sort handling, null doesn''t',
+        PRIMARY KEY(search_marc_map_id, search_field_id),
+        FOREIGN KEY(search_marc_map_id) REFERENCES search_marc_map(id) ON DELETE CASCADE ON UPDATE CASCADE,
+        FOREIGN KEY(search_field_id) REFERENCES search_field(id) ON DELETE CASCADE ON UPDATE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+|);
+
+my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
+my $indexes = LoadFile( $mappings_yaml );
+
+while ( my ( $index_name, $fields ) = each %$indexes ) {
+    while ( my ( $field_name, $data ) = each %$fields ) {
+        my $field_type = $data->{type};
+        my $field_label = $data->{label};
+        my $mappings = $data->{mappings};
+        my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' });
+        for my $mapping ( @$mappings ) {
+            my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
+            $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet}, suggestible => $mapping->{suggestible}, sort => $mapping->{sort} } );
+        }
+    }
+}

installer/data/mysql/kohastructure.sql

@@ -1946,6 +1946,19 @@ CREATE TABLE saved_reports (
    PRIMARY KEY  (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
 
+--
+-- Table structure for table 'search_field'
+--
+
+DROP TABLE IF EXISTS search_field;
+CREATE TABLE `search_field` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',
+  `label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display',
+  `type` ENUM('string', 'date', 'number', 'boolean', 'sum') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
 
 --
 -- Table structure for table `search_history`
@@ -1966,6 +1979,36 @@ CREATE TABLE IF NOT EXISTS `search_history` ( -- patron's opac search history
   PRIMARY KEY  (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci COMMENT='Opac search history results';
 
+--
+-- Table structure for table 'search_marc_map'
+--
+
+DROP TABLE IF EXISTS search_marc_map;
+CREATE TABLE `search_marc_map` (
+  id int(11) NOT NULL AUTO_INCREMENT,
+  index_name ENUM('biblios','authorities') NOT NULL COMMENT 'what storage index this map is for',
+  marc_type ENUM('marc21', 'unimarc', 'normarc') NOT NULL COMMENT 'what MARC type this map is for',
+  marc_field VARCHAR(255) NOT NULL COMMENT 'the MARC specifier for this field',
+  PRIMARY KEY(`id`),
+  UNIQUE key(index_name, marc_field, marc_type),
+  INDEX (`index_name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+
+--
+-- Table structure for table 'search_marc_to_field'
+--
+
+DROP TABLE IF EXISTS search_marc_to_field;
+CREATE TABLE `search_marc_to_field` (
+  search_marc_map_id int(11) NOT NULL,
+  search_field_id int(11) NOT NULL,
+  facet boolean DEFAULT FALSE COMMENT 'true if a facet field should be generated for this',
+  suggestible boolean DEFAULT FALSE COMMENT 'true if this field can be used to generate suggestions for browse',
+  sort boolean DEFAULT NULL COMMENT 'true/false creates special sort handling, null doesn''t',
+  PRIMARY KEY(search_marc_map_id, search_field_id),
+  FOREIGN KEY(search_marc_map_id) REFERENCES search_marc_map(id) ON DELETE CASCADE ON UPDATE CASCADE,
+  FOREIGN KEY(search_field_id) REFERENCES search_field(id) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
 
 --
 -- Table structure for table `serial`

installer/data/mysql/sysprefs.sql

@@ -421,6 +421,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `
 ('RoutingSerials','1',NULL,'If ON, serials routing is enabled','YesNo'),
 ('SCOUserCSS','',NULL,'Add CSS to be included in the SCO module in an embedded <style> tag.','free'),
 ('SCOUserJS','',NULL,'Define custom javascript for inclusion in the SCO module','free'),
+('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
 ('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
 ('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
 ('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),

installer/data/mysql/updatedatabase.pl

@@ -12255,7 +12255,6 @@ if ( $column_has_been_used ) {
         SetVersion($DBversion);
 }
 
-
 # DEVELOPER PROCESS, search for anything to execute in the db_update directory
 # SEE bug 13068
 # if there is anything in the atomicupdate, read and execute it.

koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref

@@ -211,3 +211,11 @@ Administration:
                   subscription: "subscription"
             - will be shown on the <a href="http://hea.koha-community.org">Hea Koha community website</a>.
             - Note that this value has no effect if the UsageStats system preference is set to "Don't share"
+    Search Engine:
+        -
+            - pref: SearchEngine
+              default: Zebra
+              choices:
+                Zebra: Zebra
+                Elasticsearch: Elasticsearch
+            - is the search engine used.

labels/label-item-search.pl

@@ -36,6 +36,8 @@ use C4::Creators::Lib qw(html_table);
 use C4::Debug;
 use Koha::DateUtils;
 
+use Koha::SearchEngine::Search;
+
 BEGIN {
     $debug = $debug || $cgi_debug;
     if ($debug) {
@@ -107,14 +109,14 @@ if ( $op eq "do_search" ) {
     }
 
     my $offset = $startfrom > 1 ? $startfrom - 1 : 0;
-    ( $error, $marcresults, $total_hits ) =
-      SimpleSearch( $ccl_query, $offset, $resultsperpage );
+    my $searcher = Koha::SearchEngine::Search->new({index => 'biblios'});
+    ( $error, $marcresults, $total_hits ) = $searcher->simple_search_compat($ccl_query, $offset, $resultsperpage);
 
     if (!defined $error && @{$marcresults} ) {
         $show_results = @{$marcresults};
     }
     else {
-        $debug and warn "ERROR label-item-search: no results from SimpleSearch";
+        $debug and warn "ERROR label-item-search: no results from simple_search_compat";
 
         # leave $show_results undef
     }

misc/migration_tools/bulkmarcimport.pl

@@ -31,6 +31,9 @@ use Getopt::Long;
 use IO::File;
 use Pod::Usage;
 
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
+
 use open qw( :std :encoding(UTF-8) );
 binmode( STDOUT, ":encoding(UTF-8)" );
 my ( $input_marc_file, $number, $offset) = ('',0,0);
@@ -215,6 +218,17 @@ if ($logfile){
    $loghandle= IO::File->new($logfile, $writemode) ;
    print $loghandle "id;operation;status\n";
 }
+
+my $searcher = Koha::SearchEngine::Search->new(
+    {
+        index => (
+              $authorities
+            ? $Koha::SearchEngine::AUTHORITIES_INDEX
+            : $Koha::SearchEngine::BIBLIOS_INDEX
+        )
+    }
+);
+
 RECORD: while (  ) {
     my $record;
     # get records
@@ -268,7 +282,7 @@ RECORD: while (  ) {
         my $query = build_query( $match, $record );
         my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
         $debug && warn $query;
-        my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
+        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
         # changed to warn so able to continue with one broken record
         if ( defined $error ) {
             warn "unable to search the database for duplicates : $error";

misc/migration_tools/checkNonIndexedBiblios.pl

@@ -34,9 +34,10 @@ BEGIN {
 # Koha modules used
 use MARC::Record;
 use C4::Context;
-use C4::Search;
 use Getopt::Long;
 
+use Koha::SearchEngine::Search;
+
 my ( $help, $confirm, $zebraqueue, $silent,$stealth );
 
 GetOptions(
@@ -62,13 +63,14 @@ $| = 1;
 my $sth = $dbh->prepare("SELECT biblionumber FROM biblio");
 my $sth_insert = $dbh->prepare("INSERT INTO zebraqueue (biblio_auth_number,operation,server,done) VALUES (?,'specialUpdate','biblioserver',0)");
 
+my $searcher = Koha::SearchEngine::Search->new({index => 'biblios'});
+
 # We get all biblios
 $sth->execute;
 my ($nbhits);
-
 # We check for each biblio
 while ( my ($biblionumber) = $sth->fetchrow ) {
-    (undef,undef,$nbhits) = SimpleSearch("Local-number=$biblionumber");
+    (undef,undef,$nbhits) = $searcher->simple_search_compat("Local-number=$biblionumber");
     print "biblionumber $biblionumber not indexed\n" unless $nbhits || $stealth;
 # If -z option we put the biblio in zebraqueue
     if ($zebraqueue && !$nbhits){
@@ -105,4 +107,4 @@ Syntax:
 \t./batchCheckNonIndexedBiblios.pl -s (s like silent => don't throws a warn every 1000 biblios to show progress)
 \t./batchCheckNonIndexedBiblios.pl -st (s like stealth => don't throws a warn every 1000 biblios to show progress and no warn for the non indexed biblionumbers, just the total)
 _USAGE_
-}
+}

misc/migration_tools/remove_unused_authorities.pl

@@ -27,6 +27,8 @@ use C4::Context;
 use C4::AuthoritiesMarc;
 use Getopt::Long;
 
+use Koha::SearchEngine::Search;
+
 my @authtypes;
 my $want_help = 0;
 my $test = 0;
@@ -66,11 +68,12 @@ $rqselect->execute;
 my $counter=0;
 my $totdeleted=0;
 my $totundeleted=0;
+my $searcher = Koha::SearchEngine::Search->new({index => 'biblios'});
 while (my $data=$rqselect->fetchrow_hashref){
     my $query;
     $query= "an=".$data->{'authid'};
     # search for biblios mapped
-    my ($err,$res,$used) = C4::Search::SimpleSearch($query,0,10);
+    my ($err,$res,$used) = $searcher->simple_search_compat($query,0,10);
     if (defined $err) {
         warn "error: $err on search $query\n";
         next;

misc/search_tools/rebuild_elastic_search.pl

@@ -0,0 +1,212 @@
+#!/usr/bin/perl
+
+# This inserts records from a Koha database into elastic search
+
+# Copyright 2014 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+=head1 NAME
+
+rebuild_elastic_search.pl - inserts records from a Koha database into Elasticsearch
+
+=head1 SYNOPSIS
+
+B<rebuild_elastic_search.pl>
+[B<-c|--commit>=C<count>]
+[B<-v|--verbose>]
+[B<-h|--help>]
+[B<--man>]
+
+=head1 DESCRIPTION
+
+Inserts records from a Koha database into Elasticsearch.
+
+=head1 OPTIONS
+
+=over
+
+=item B<-c|--commit>=C<count>
+
+Specify how many records will be batched up before they're added to Elasticsearch.
+Higher should be faster, but will cause more RAM usage. Default is 5000.
+
+=item B<-d|--delete>
+
+Delete the index and recreate it before indexing.
+
+=item B<-a|--authorities>
+
+Index the authorities only. Combining this with B<-b> is the same as
+specifying neither and so both get indexed.
+
+=item B<-b|--biblios>
+
+Index the biblios only. Combining this with B<-a> is the same as
+specifying neither and so both get indexed.
+
+=item B<-bn|--bnumber>
+
+Only index the supplied biblionumber, mostly for testing purposes. May be
+repeated. This also applies to authorities via authid, so if you're using it,
+you probably only want to do one or the other at a time.
+
+=item B<-v|--verbose>
+
+By default, this program only emits warnings and errors. This makes it talk
+more. Add more to make it even more wordy, in particular when debugging.
+
+=item B<-h|--help>
+
+Help!
+
+=item B<--man>
+
+Full documentation.
+
+=back
+
+=cut
+
+use autodie;
+use Getopt::Long;
+use C4::Context;
+use Koha::Authority;
+use Koha::BiblioUtils;
+use Koha::ElasticSearch::Indexer;
+use MARC::Field;
+use MARC::Record;
+use Modern::Perl;
+use Pod::Usage;
+
+use Data::Dumper; # TODO remove
+
+my $verbose = 0;
+my $commit = 5000;
+my ($delete, $help, $man);
+my ($index_biblios, $index_authorities);
+my (@biblionumbers);
+
+GetOptions(
+    'c|commit=i'       => \$commit,
+    'd|delete'         => \$delete,
+    'a|authorities' => \$index_authorities,
+    'b|biblios' => \$index_biblios,
+    'bn|bnumber=i' => \@biblionumbers,
+    'v|verbose+'       => \$verbose,
+    'h|help'           => \$help,
+    'man'              => \$man,
+);
+
+# Default is to do both
+unless ($index_authorities || $index_biblios) {
+    $index_authorities = $index_biblios = 1;
+}
+
+pod2usage(1) if $help;
+pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
+
+sanity_check();
+
+my $next;
+if ($index_biblios) {
+    _log(1, "Indexing biblios\n");
+    if (@biblionumbers) {
+        $next = sub {
+            my $r = shift @biblionumbers;
+            return () unless defined $r;
+            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
+        };
+    } else {
+        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
+        $next = sub {
+            $records->next();
+        }
+    }
+    do_reindex($next, $Koha::ElasticSearch::BIBLIOS_INDEX);
+}
+if ($index_authorities) {
+    _log(1, "Indexing authorities\n");
+    if (@biblionumbers) {
+        $next = sub {
+            my $r = shift @biblionumbers;
+            return () unless defined $r;
+            my $a = Koha::Authority->get_from_authid($r);
+            return ($r, $a->record);
+        };
+    } else {
+        my $records = Koha::Authority->get_all_authorities_iterator();
+        $next = sub {
+            $records->next();
+        }
+    }
+    do_reindex($next, $Koha::ElasticSearch::AUTHORITIES_INDEX);
+}
+
+sub do_reindex {
+    my ( $next, $index_name ) = @_;
+
+    my $indexer = Koha::ElasticSearch::Indexer->new( { index => $index_name } );
+    if ($delete) {
+
+        # We know it's safe to not recreate the indexer because update_index
+        # hasn't been called yet.
+        $indexer->drop_index();
+    }
+
+    my $count        = 0;
+    my $commit_count = $commit;
+    my ( @id_buffer, @commit_buffer );
+    while ( my $record = $next->() ) {
+        my $id     = $record->id;
+        my $record = $record->record;
+        _log( 1, "$id\n" );
+        $count++;
+
+        push @id_buffer,     $id;
+        push @commit_buffer, $record;
+        if ( !( --$commit_count ) ) {
+            _log( 2, "Committing...\n" );
+            $indexer->update_index( \@id_buffer, \@commit_buffer );
+            $commit_count  = $commit;
+            @id_buffer     = ();
+            @commit_buffer = ();
+        }
+    }
+
+    # There are probably uncommitted records
+    $indexer->update_index( \@id_buffer, \@commit_buffer );
+    _log( 1, "$count records indexed.\n" );
+}
+
+# Checks some basic stuff to ensure that it's sane before we start.
+sub sanity_check {
+    # Do we have an elasticsearch block defined?
+    my $conf = C4::Context->config('elasticsearch');
+    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
+}
+
+# Output progress information.
+#
+#   _log($level, $msg);
+#
+# Will output $msg if the verbosity setting is set to $level or more. Will
+# not include a trailing newline.
+sub _log {
+    my ($level, $msg) = @_;
+
+    print $msg if ($verbose >= $level);
+}

opac/opac-authorities-home.pl

@@ -32,6 +32,8 @@ use C4::Koha;
 use C4::Search::History;
 
 use Koha::Authority::Types;
+use Koha::SearchEngine::Search;
+use Koha::SearchEngine::QueryBuilder;
 
 my $query        = new CGI;
 my $op           = $query->param('op') || '';
@@ -58,10 +60,19 @@ if ( $op eq "do_search" ) {
     $resultsperpage = $query->param('resultsperpage');
     $resultsperpage = 20 if ( !defined $resultsperpage );
     my @tags;
-    my ( $results, $total, @fields ) =
-      SearchAuthorities( \@marclist, \@and_or, \@excluding, \@operator,
-        \@value, $startfrom * $resultsperpage,
-        $resultsperpage, $authtypecode, $orderby );
+    my $builder = Koha::SearchEngine::QueryBuilder->new(
+        { index => $Koha::SearchEngine::AUTHORITIES_INDEX } );
+    my $searcher = Koha::SearchEngine::Search->new(
+        { index => $Koha::SearchEngine::AUTHORITIES_INDEX } );
+    my $search_query = $builder->build_authorities_query_compat( \@marclist, \@and_or,
+        \@excluding, \@operator, \@value, $authtypecode, $orderby );
+#    use Data::Dumper;
+#    die Dumper(\@marclist, \@and_or,
+#        \@excluding, \@operator, \@value, $authtypecode, $orderby, $query);
+    # The searchengine API expects pages to start at page 1
+    $startfrom = $startfrom // 0;
+    my ( $results, $total ) =
+      $searcher->search_auth_compat( $search_query, $startfrom+1, $resultsperpage );
     ( $template, $loggedinuser, $cookie ) = get_template_and_user(
         {
             template_name   => "opac-authoritiessearchresultlist.tt",
@@ -116,8 +127,9 @@ if ( $op eq "do_search" ) {
         $to = ( ( $startfrom + 1 ) * $resultsperpage );
     }
     unless (C4::Context->preference('OPACShowUnusedAuthorities')) {
-        my @usedauths = grep { $_->{used} > 0 } @$results;
-        $results = \@usedauths;
+#        TODO implement usage counts
+#        my @usedauths = grep { $_->{used} > 0 } @$results;
+#        $results = \@usedauths;
     }
 
     # Opac search history
@@ -151,7 +163,6 @@ if ( $op eq "do_search" ) {
     }
 
     $template->param( result => $results ) if $results;
-    $template->param( FIELDS => \@fields );
     $template->param( orderby => $orderby );
     $template->param(
         startfrom      => $startfrom,

opac/opac-search.pl

@@ -30,6 +30,17 @@ use Modern::Perl;
 use C4::Context;
 use List::MoreUtils q/any/;
 
+use Data::Dumper; # TODO remove
+
+use Koha::SearchEngine::Search;
+use Koha::SearchEngine::QueryBuilder;
+
+my $searchengine = C4::Context->preference("SearchEngine");
+my ($builder, $searcher);
+#$searchengine = 'Zebra'; # XXX
+$builder  = Koha::SearchEngine::QueryBuilder->new({index => 'biblios'});
+$searcher = Koha::SearchEngine::Search->new({index => 'biblios'});
+
 use C4::Output;
 use C4::Auth qw(:DEFAULT get_session);
 use C4::Languages qw(getLanguages);
@@ -525,7 +536,7 @@ my ($error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_
 my @results;
 
 ## I. BUILD THE QUERY
-( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type) = buildQuery(\@operators,\@operands,\@indexes,\@limits,\@sort_by, 0, $lang);
+( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type) = $builder->build_query_compat(\@operators,\@operands,\@indexes,\@limits,\@sort_by, 0, $lang, { expanded_facet => $expanded_facet });
 
 sub _input_cgi_parse {
     my @elements;
@@ -605,15 +616,8 @@ if ($tag) {
     $pasarParams .= '&simple_query=' . uri_escape_utf8($simple_query);
     $pasarParams .= '&query_type=' . uri_escape_utf8($query_type) if ($query_type);
     eval {
-        ($error, $results_hashref, $facets) = getRecords($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$itemtypes_nocategory,$query_type,$scan,1);
-    };
-}
-# This sorts the facets into alphabetical order
-if ($facets) {
-    foreach my $f (@$facets) {
-        $f->{facets} = [ sort { uc($a->{facet_label_value}) cmp uc($b->{facet_label_value}) } @{ $f->{facets} } ];
-    }
-    @$facets = sort {$a->{expand} cmp $b->{expand}} @$facets;
+        ($error, $results_hashref, $facets) = $searcher->search_compat($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$itemtypes,$query_type,$scan,1);
+};
 }
 
 # use Data::Dumper; print STDERR "-" x 25, "\n", Dumper($results_hashref);

serials/subscription-bib-search.pl

@@ -58,6 +58,9 @@ use C4::Search;
 use C4::Biblio;
 use C4::Debug;
 
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
+
 my $input = new CGI;
 my $op = $input->param('op') || q{};
 my $dbh = C4::Context->dbh;
@@ -96,8 +99,9 @@ if ( $op eq "do_search" && $query ) {
     $resultsperpage = $input->param('resultsperpage');
     $resultsperpage = 20 if ( !defined $resultsperpage );
 
+    my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
     my ( $error, $marcrecords, $total_hits ) =
-      SimpleSearch( $query, $startfrom * $resultsperpage, $resultsperpage );
+      $searcher->simple_search_compat( $query, $startfrom * $resultsperpage, $resultsperpage );
     my $total = 0;
     if ( defined $marcrecords ) {
         $total = scalar @{$marcrecords};

t/Koha/ItemType.t

@@ -0,0 +1,46 @@
+#!/usr/bin/perl
+#
+# Copyright 2014 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use Modern::Perl;
+
+use Test::More tests => 8;
+
+BEGIN {
+    use_ok('Koha::ItemType');
+}
+
+my $data = {
+    itemtype       => 'CODE',
+    description    => 'description',
+    rentalcharge   => 'rentalcharge',
+    imageurl       => 'imageurl',
+    summary        => 'summary',
+    checkinmsg     => 'checkinmsg',
+    checkinmsgtype => 'checkinmsgtype',
+};
+
+my $type = Koha::ItemType->new($data);
+
+is( $type->itemtype,       'CODE',           'itemtype/code' );
+is( $type->description,    'description',    'description' );
+is( $type->rentalcharge,   'rentalcharge',   'rentalcharge' );
+is( $type->imageurl,       'imageurl',       'imageurl' );
+is( $type->summary,        'summary',        'summary' );
+is( $type->checkinmsg,     'checkinmsg',     'checkinmsg' );
+is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );

t/db_dependent/Koha/BiblioUtils.t

@@ -1,5 +1,7 @@
 #!/usr/bin/perl
-
+#
+# Copyright 2014 Catalyst IT
+#
 # This file is part of Koha.
 #
 # Koha is free software; you can redistribute it and/or modify it

t/db_dependent/Koha/ItemTypes.t

@@ -0,0 +1,79 @@
+#!/usr/bin/perl
+#
+# Copyright 2014 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use Modern::Perl;
+
+use Test::More tests => 18;
+use Data::Dumper;
+use Koha::Database;
+
+BEGIN {
+    use_ok('Koha::ItemType');
+    use_ok('Koha::ItemTypes');
+}
+
+my $database = Koha::Database->new();
+my $schema   = $database->schema();
+$schema->txn_begin;
+
+Koha::ItemType->new(
+    {
+        itemtype       => 'type1',
+        description    => 'description',
+        rentalcharge   => '0.00',
+        imageurl       => 'imageurl',
+        summary        => 'summary',
+        checkinmsg     => 'checkinmsg',
+        checkinmsgtype => 'checkinmsgtype',
+    }
+)->store;
+
+Koha::ItemType->new(
+    {
+        itemtype       => 'type2',
+        description    => 'description',
+        rentalcharge   => '0.00',
+        imageurl       => 'imageurl',
+        summary        => 'summary',
+        checkinmsg     => 'checkinmsg',
+        checkinmsgtype => 'checkinmsgtype',
+    }
+)->store;
+
+my $type = Koha::ItemTypes->find('type1');
+ok( defined($type), 'first result' );
+is( $type->itemtype,       'type1',          'itemtype/code' );
+is( $type->description,    'description',    'description' );
+is( $type->rentalcharge,   '0.0000',             'rentalcharge' );
+is( $type->imageurl,       'imageurl',       'imageurl' );
+is( $type->summary,        'summary',        'summary' );
+is( $type->checkinmsg,     'checkinmsg',     'checkinmsg' );
+is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );
+
+$type = Koha::ItemTypes->find('type2');
+ok( defined($type), 'second result' );
+is( $type->itemtype,       'type2',          'itemtype/code' );
+is( $type->description,    'description',    'description' );
+is( $type->rentalcharge,   '0.0000',             'rentalcharge' );
+is( $type->imageurl,       'imageurl',       'imageurl' );
+is( $type->summary,        'summary',        'summary' );
+is( $type->checkinmsg,     'checkinmsg',     'checkinmsg' );
+is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );
+
+$schema->txn_rollback;

t/db_dependent/Koha_ElasticSearch.t

@@ -0,0 +1,23 @@
+# Copyright 2015 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>
+
+use strict;
+use warnings;
+
+use Test::More tests => 1;                      # last test to print
+
+use_ok('Koha::ElasticSearch');

t/db_dependent/Koha_ElasticSearch_Indexer.t

@@ -0,0 +1,52 @@
+# Copyright 2015 Catalyst IT
+#
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+
+use strict;
+use warnings;
+
+use Test::More tests => 5;    # last test to print
+use MARC::Record;
+
+use_ok('Koha::ElasticSearch::Indexer');
+
+my $indexer;
+ok(
+    $indexer = Koha::ElasticSearch::Indexer->new(
+        {
+            'nodes' => ['localhost:9200'],
+            'index' => 'mydb'
+        }
+    ),
+    'Creating new indexer object'
+);
+
+my $marc_record = MARC::Record->new();
+my $field = MARC::Field->new( '001', '1234567' );
+$marc_record->append_fields($field);
+$field = MARC::Field->new( '020', '', '', 'a' => '1234567890123' );
+$marc_record->append_fields($field);
+$field = MARC::Field->new( '245', '', '', 'a' => 'Title' );
+$marc_record->append_fields($field);
+
+my $records = [$marc_record];
+ok( my $converted = $indexer->_convert_marc_to_json($records),
+    'Convert some records' );
+
+is( $converted->count, 1, 'One converted record' );
+
+ok( $indexer->update_index(undef,$records), 'Update Index' );

t/db_dependent/Koha_SearchEngine_Elasticsearch_Search.t

@@ -0,0 +1,48 @@
+# Copyright 2015 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+use strict;
+use warnings;
+
+use Test::More tests => 10;    # last test to print
+use Koha::SearchEngine::Elasticsearch::QueryBuilder;
+
+my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new( { index => 'mydb' } );
+
+use_ok('Koha::SearchEngine::Elasticsearch::Search');
+
+ok(
+    my $searcher = Koha::SearchEngine::Elasticsearch::Search->new(
+        { 'nodes' => ['localhost:9200'], 'index' => 'mydb' }
+    ),
+    'Creating a Koha::ElasticSearch::Search object'
+);
+
+is( $searcher->index, 'mydb', 'Testing basic accessor' );
+
+ok( my $query = $builder->build_query('easy'), 'Build a search query');
+
+ok( my $results = $searcher->search( $query) , 'Do a search ' );
+
+ok( my $marc = $searcher->json2marc( $results->first ), 'Convert JSON to MARC');
+
+is (my $count = $searcher->count( $query ), 0 , 'Get a count of the results, without returning results ');
+
+ok ($results = $searcher->search_compat( $query ), 'Test search_compat' );
+
+ok (($results,$count) = $searcher->search_auth_compat ( $query ), 'Test search_auth_compat' );
+
+is ( $count = $searcher->count_auth_use($searcher,1), 0, 'Testing count_auth_use');