package Koha::SearchEngine::Elasticsearch::QueryBuilder;

# This file is part of Koha.
#
# Copyright 2014 Catalyst IT Ltd.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <https://www.gnu.org/licenses>.

=head1 NAME

Koha::SearchEngine::Elasticsearch::QueryBuilder - constructs elasticsearch
query objects from user-supplied queries

=head1 DESCRIPTION

This provides the functions that take a user-supplied search query, and
provides something that can be given to elasticsearch to get answers.

=head1 SYNOPSIS

    use Koha::SearchEngine::Elasticsearch::QueryBuilder;
    $builder = Koha::SearchEngine::Elasticsearch->new({ index => $index });
    my $simple_query = $builder->build_query("hello");
    # This is currently undocumented because the original code is undocumented
    my $adv_query = $builder->build_advanced_query($indexes, $operands, $operators);

=head1 METHODS

=cut

use base qw(Koha::SearchEngine::Elasticsearch);
use JSON;
use List::MoreUtils qw( each_array );
use Modern::Perl;
use URI::Escape qw( uri_escape_utf8 );

use C4::Context;
use Koha::Exceptions;
use Koha::Authority;
use Koha::Caches;
use Koha::AuthorisedValueCategories;

our %index_field_convert = (
    'kw'                      => '',
    'ab'                      => 'abstract',
    'au'                      => 'author',
    'lcn'                     => 'local-classification',
    'callnum'                 => 'local-classification',
    'record-type'             => 'rtype',
    'mc-rtype'                => 'rtype',
    'mus'                     => 'rtype',
    'lc-card'                 => 'lc-card-number',
    'sn'                      => 'local-number',
    'biblionumber'            => 'local-number',
    'yr'                      => 'date-of-publication',
    'pubdate'                 => 'date-of-publication',
    'acqdate'                 => 'date-of-acquisition',
    'date/time-last-modified' => 'date-time-last-modified',
    'dtlm'                    => 'date-time-last-modified',
    'diss'                    => 'dissertation-information',
    'nb'                      => 'isbn',
    'ns'                      => 'issn',
    'music-number'            => 'identifier-publisher-for-music',
    'number-music-publisher'  => 'identifier-publisher-for-music',
    'music'                   => 'identifier-publisher-for-music',
    'ident'                   => 'identifier-standard',
    'cpn'                     => 'corporate-name',
    'cfn'                     => 'conference-name',
    'pn'                      => 'personal-name',
    'pb'                      => 'publisher',
    'pv'                      => 'provider',
    'nt'                      => 'note',
    'notes'                   => 'note',
    'rcn'                     => 'record-control-number',
    'cni'                     => 'control-number-identifier',
    'cnum'                    => 'control-number',
    'su'                      => 'subject',
    'su-to'                   => 'subject',

    #'su-geo' => 'subject',
    'su-ut'                 => 'subject',
    'ti'                    => 'title',
    'se'                    => 'title-series',
    'ut'                    => 'title-uniform',
    'an'                    => 'koha-auth-number',
    'authority-number'      => 'koha-auth-number',
    'at'                    => 'authtype',
    'he'                    => 'heading',
    'rank'                  => 'relevance',
    'phr'                   => 'st-phrase',
    'wrdl'                  => 'st-word-list',
    'rt'                    => 'right-truncation',
    'rtrn'                  => 'right-truncation',
    'ltrn'                  => 'left-truncation',
    'rltrn'                 => 'left-and-right',
    'mc-itemtype'           => 'itemtype',
    'mc-ccode'              => 'ccode',
    'branch'                => 'homebranch',
    'mc-loc'                => 'location',
    'loc'                   => 'location',
    'stocknumber'           => 'number-local-acquisition',
    'inv'                   => 'number-local-acquisition',
    'bc'                    => 'barcode',
    'mc-itype'              => 'itype',
    'aub'                   => 'author-personal-bibliography',
    'auo'                   => 'author-in-order',
    'ff8-22'                => 'ta',
    'aud'                   => 'ta',
    'audience'              => 'ta',
    'frequency-code'        => 'ff8-18',
    'illustration-code'     => 'ff8-18-21',
    'regularity-code'       => 'ff8-19',
    'type-of-serial'        => 'ff8-21',
    'format'                => 'ff8-23',
    'conference-code'       => 'ff8-29',
    'festschrift-indicator' => 'ff8-30',
    'index-indicator'       => 'ff8-31',
    'fiction'               => 'lf',
    'fic'                   => 'lf',
    'literature-code'       => 'lf',
    'biography'             => 'bio',
    'ff8-34'                => 'bio',
    'biography-code'        => 'bio',
    'l-format'              => 'ff7-01-02',
    'lex'                   => 'lexile-number',
    'hi'                    => 'host-item-number',
    'itu'                   => 'index-term-uncontrolled',
    'itg'                   => 'index-term-genre',
);
my $field_name_pattern   = '[\w\-]+';
my $multi_field_pattern  = "(?:\\.$field_name_pattern)*";
my $es_advanced_searches = [];

=head2 get_index_field_convert

    my @index_params = Koha::SearchEngine::Elasticsearch::QueryBuilder->get_index_field_convert();

Converts zebra-style search index notation into elasticsearch-style.

C<@indexes> is an array of index names, as presented to L<build_query_compat>,
and it returns something that can be sent to L<build_query>.

B<TODO>: this will pull from the elasticsearch mappings table to figure out
types.

=cut

sub get_index_field_convert() {
    return \%index_field_convert;
}

=head2 build_query

    my $simple_query = $builder->build_query("hello", %options)

This will build a query that can be issued to elasticsearch from the provided
string input. This expects a lucene style search form (see
L<http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax>
for details.)

It'll make an attempt to respect the various query options.

Additional options can be provided with the C<%options> hash.

=over 4

=item sort

This should be an arrayref of hashrefs, each containing a C<field> and an
C<direction> (optional, defaults to C<asc>.) The results will be sorted
according to these values. Valid values for C<direction> are 'asc' and 'desc'.

=back

=cut

sub build_query {
    my ( $self, $query, %options ) = @_;

    my $stemming        = C4::Context->preference("QueryStemming")     || 0;
    my $auto_truncation = C4::Context->preference("QueryAutoTruncate") || 0;
    my $fuzzy_enabled   = C4::Context->preference("QueryFuzzy")        || 0;

    $query = '*' unless defined $query;

    my $res;
    my $fields = $self->_search_fields(
        {
            is_opac         => $options{is_opac},
            weighted_fields => $options{weighted_fields},
        }
    );
    if ( $options{whole_record} ) {
        push @$fields, 'marc_data_array.*';
    }
    my $query_string = {
        query            => $query,
        fuzziness        => $fuzzy_enabled ? 'auto' : '0',
        default_operator => 'AND',
        fields           => $fields,
        lenient          => JSON::true,
        analyze_wildcard => JSON::true,
    };
    $query_string->{type} = 'cross_fields' if C4::Context->preference('ElasticsearchCrossFields');

    $res->{query} = { bool => { must => [ { query_string => $query_string } ] } };

    $res->{query}->{bool}->{should} = $options{field_match_boost_query} if $options{field_match_boost_query};

    if ( $options{sort} ) {
        foreach my $sort ( @{ $options{sort} } ) {
            my ( $f, $d ) = @$sort{qw/ field direction /};
            die "Invalid sort direction, $d"
                if $d && ( $d ne 'asc' && $d ne 'desc' );
            $d = 'asc' unless $d;

            $f = $self->_sort_field($f);
            push @{ $res->{sort} }, { $f => { order => $d } };
        }
    }

    # If not sorting by anything explicitly search by score
    if ( !defined $res->{sort} ) {
        push @{ $res->{sort} }, { _score => { order => 'desc' } };
    }

    # Add a tie breaker in case of equally relevant records
    push @{ $res->{sort} }, { 'local-number__sort' => { order => 'desc' } };

    unless ( $options{skip_facets} ) {

        # See _convert_facets in Search.pm for how these get turned into
        # things that Koha can use.
        my $size   = C4::Context->preference('FacetMaxCount');
        my @facets = Koha::SearchEngine::Elasticsearch->get_facet_fields;
        for my $f (@facets) {
            my $name = $f->name;
            $res->{aggregations}->{$name} = { terms => { field => "${name}__facet", size => $size } };
        }

    }

    $res = _rebuild_to_es_advanced_query($res) if @$es_advanced_searches;
    return $res;
}

=head2 build_query_compat

    my (
        $error,             $query, $simple_query, $query_cgi,
        $query_desc,        $limit, $limit_cgi,    $limit_desc,
        $stopwords_removed, $query_type
      )
      = $builder->build_query_compat( \@operators, \@operands, \@indexes,
        \@limits, \@sort_by, $scan, $lang, $params );

This handles a search using the same api as L<C4::Search::buildQuery> does.

A very simple query will go in with C<$operands> set to ['query'], and
C<$sort_by> set to ['pubdate_dsc']. This simple case will return with
C<$query> set to something that can perform the search, C<$simple_query>
set to just the search term, C<$query_cgi> set to something that can
reproduce this search, and C<$query_desc> set to something else.

=cut

sub build_query_compat {
    my (
        $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan,
        $lang, $params
    ) = @_;

    my $query;
    my $query_str              = '';
    my $search_param_query_str = '';
    my $limits                 = ();
    if ($scan) {
        ( $query, $query_str ) = $self->_build_scan_query( $operands, $indexes );
        $search_param_query_str = $query_str;
    } else {
        my @sort_params  = $self->_convert_sort_fields(@$sort_by);
        my @index_params = $self->_convert_index_fields(@$indexes);
        my $field_match_boost_query =
            C4::Context->preference('ElasticsearchBoostFieldMatch')
            ? $self->_build_field_match_boost_query( { operands => $operands, indexes => \@index_params } )
            : [];
        $limits = $self->_fix_limit_special_cases($orig_limits);
        if ( $params->{suppress} ) { push @$limits, "suppress:false"; }

        # Merge the indexes in with the search terms and the operands so that
        # each search thing is a handy unit.
        unshift @$operators, undef;    # The first one can't have an op
        my @search_params;
        my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
        my $ea       = each_array( @$operands, @$operators, @index_params );
        while ( my ( $oand, $otor, $index ) = $ea->() ) {
            next if ( !defined($oand) || $oand eq '' );
            $oand = $self->clean_search_term($oand);
            $oand = $self->_truncate_terms($oand)
                if $truncate && $self->_is_safe_to_auto_truncate( $index->{field}, $oand );
            push @search_params, {
                operand => $oand,                                 # the search terms
                operator => defined($otor) ? uc $otor : undef,    # AND and so on
                $index ? %$index : (),
            };
        }

        # We build a string query from limits and the queries. An alternative
        # would be to pass them separately into build_query and let it build
        # them into a structured ES query itself. Maybe later, though that'd be
        # more robust.
        my @search_param_query_array = $self->_create_query_string(@search_params);
        $search_param_query_str = join( ' ', @search_param_query_array );
        my $search_param_limit_str = $self->_join_queries( $self->_convert_index_strings(@$limits) );
        if ( @search_param_query_array > 1 && $search_param_limit_str ) {
            $search_param_query_str = "($search_param_query_str)";
        }
        $query_str = join(
            ' AND ',
            $search_param_query_str || (),
            $search_param_limit_str || ()
        );

        # If there's no query on the left, let's remove the junk left behind
        $query_str =~ s/^ AND //;
        my %options;
        $options{sort}                    = \@sort_params;
        $options{is_opac}                 = $params->{is_opac};
        $options{weighted_fields}         = $params->{weighted_fields};
        $options{whole_record}            = $params->{whole_record};
        $options{skip_facets}             = $params->{skip_facets};
        $options{field_match_boost_query} = $field_match_boost_query if @$field_match_boost_query;
        $query                            = $self->build_query( $query_str, %options );
    }

    # We roughly emulate the CGI parameters of the zebra query builder
    my $query_cgi = '';
    shift @$operators;    # Shift out the one we unshifted before
    my $ea = each_array( @$operands, @$operators, @$indexes );
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
        $query_cgi .= '&' if $query_cgi;
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '' ) . '&q=' . uri_escape_utf8($oand);
        $query_cgi .= '&op=' . uri_escape_utf8($otor) if $otor;
    }
    $query_cgi .= '&scan=1'                    if ($scan);
    $query_cgi .= '&weight_search=1'           if ( $params->{weighted_fields} );
    $query_cgi .= '&weight_search_submitted=1' if ( $params->{weight_search_submitted} );
    $query_cgi .= '&whole_record=1'            if ( $params->{whole_record} );

    my $simple_query;
    $simple_query = $operands->[0] if @$operands == 1;
    my $query_desc;
    if ($simple_query) {
        $query_desc = $simple_query;
    } else {
        $query_desc = $search_param_query_str;
    }
    my $limit = $self->_join_queries( $self->_convert_index_strings(@$limits) );
    my $limit_cgi =
        ( $orig_limits and @$orig_limits )
        ? '&limit=' . join( '&limit=', map { uri_escape_utf8($_) } @$orig_limits )
        : '';
    my $limit_desc;
    $limit_desc = "$limit" if $limit;

    return (
        undef,  $query,     $simple_query, $query_cgi, $query_desc,
        $limit, $limit_cgi, $limit_desc,   undef,      undef
    );
}

=head2 _build_field_match_boost_query

    my ($query, $query_str) = $builder->_build_field_match_boost_query({ operands => \@operands, indexes => \@indexes)

This will build an array of match queries for terms and indexes passed in and return a reference to the array.

=cut

sub _build_field_match_boost_query {
    my ( $self, $params ) = @_;
    my $indexes  = $params->{indexes};
    my $operands = $params->{operands};

    my $boost_amount = C4::Context->preference('ElasticsearchBoostFieldMatchAmount');
    my @boost_query;
    my $built_boost;
    my $ea = each_array( @$operands, @$indexes );
    while ( my ( $operand, $index ) = $ea->() ) {
        next unless $operand;
        $index = $index->{field} if ref $index eq 'HASH';
        $index = 'title-cover'   if ( !$index || $index eq 'kw' || $index eq 'ti' || $index eq 'title' );
        $built_boost = { match => { $index => { query => $operand } } };
        $built_boost->{match}->{$index}->{boost} = $boost_amount if $boost_amount;
        push @boost_query, $built_boost;
    }
    return \@boost_query;
}

=head2 build_authorities_query

    my $query = $builder->build_authorities_query(\%search);

This takes a nice description of an authority search and turns it into a black-box
query that can then be passed to the appropriate searcher.

The search description is a hashref that looks something like:

    {
        searches => [
            {
                where    => 'Heading',    # search the main entry
                operator => 'exact',        # require an exact match
                value    => 'frogs',        # the search string
            },
            {
                where    => '',             # search all entries
                operator => '',             # default keyword, right truncation
                value    => 'pond',
            },
        ],
        sort => {
            field => 'Heading',
            order => 'desc',
        },
        authtypecode => 'TOPIC_TERM',
    }

=cut

sub build_authorities_query {
    my ( $self, $search ) = @_;

    # Start by making the query parts
    my @query_parts;

    foreach my $s ( @{ $search->{searches} } ) {
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
        if ( defined $op && ( $op eq 'is' || $op eq '=' || $op eq 'exact' ) ) {
            if ($wh) {

                # Match the whole field, case insensitive, UTF normalized.
                push @query_parts, { term => { "$wh.ci_raw" => $val } };
            } else {

                # Match the whole field for all searchable fields, case insensitive,
                # UTF normalized.
                # Given that field data is "The quick brown fox"
                # "The quick brown fox" and "the quick brown fox" will match
                # but not "quick brown fox".
                push @query_parts, {
                    multi_match => {
                        query  => $val,
                        fields => $self->_search_fields( { subfield => 'ci_raw' } ),
                    }
                };
            }
        } elsif ( defined $op && $op eq 'start' ) {

            # Match the prefix within a field for all searchable fields.
            # Given that field data is "The quick brown fox"
            # "The quick bro" will match, but not "quick bro"

            # Does not seems to be a multi prefix query
            # so we need to create one
            if ($wh) {

                # Match prefix of the field.
                push @query_parts, { prefix => { "$wh.ci_raw" => $val } };
            } else {
                my @prefix_queries;
                foreach my $field ( @{ $self->_search_fields() } ) {
                    push @prefix_queries, { prefix => { "$field.ci_raw" => $val } };
                }
                push @query_parts, {
                    'bool' => {
                        'should'               => \@prefix_queries,
                        'minimum_should_match' => 1
                    }
                };
            }
        } else {

            # Query all searchable fields.
            # Given that field data is "The quick brown fox"
            # a search containing any of the words will match, regardless
            # of order.

            my @tokens = $self->_split_query($val);
            foreach my $token (@tokens) {
                $token = $self->_truncate_terms( $self->clean_search_term($token) );
            }
            my $query        = $self->_join_queries(@tokens);
            my $query_string = {
                query            => $query,
                lenient          => JSON::true,
                analyze_wildcard => JSON::true,
            };
            if ($wh) {
                $query_string->{default_field} = $wh;
            } else {
                $query_string->{fields} = $self->_search_fields();
            }
            push @query_parts, { query_string => $query_string };
        }
    }

    # Merge the query parts appropriately
    # 'should' behaves like 'or'
    # 'must' behaves like 'and'
    # We will obey the first and_or passed in here:
    # authfinder.pl is hardcoded to 'AND'
    # authorities-home.pl only allows searching a single index
    # for matching we 'OR' together multiple fields
    my $search_type   = defined $search->{and_or} && uc( $search->{and_or}->[0] ) eq 'OR' ? 'should' : 'must';
    my $elastic_query = {};
    $elastic_query->{bool}->{$search_type} = \@query_parts;

    # Filter by authtypecode if set
    if ( $search->{authtypecode} ) {
        $elastic_query->{bool}->{filter} = { term => { "authtype.raw" => $search->{authtypecode} } };
    }

    my $query = { query => $elastic_query };

    # Add the sort stuff
    $query->{sort} = [ $search->{sort} ] if exists $search->{sort};

    return $query;
}

=head2 build_authorities_query_compat

    my ($query) =
      $builder->build_authorities_query_compat( \@marclist, \@and_or,
        \@excluding, \@operator, \@value, $authtypecode, $orderby );

This builds a query for searching for authorities, in the style of
L<C4::AuthoritiesMarc::SearchAuthorities>.

Arguments:

=over 4

=item marclist

An arrayref containing where the particular term should be searched for.
Options are: mainmainentry, mainentry, match, match-heading, see-from, and
thesaurus. If left blank, any field is used.

=item and_or

This takes an array to match Zebra, however, we only care about the first value.
This will determine whether searches are combined as 'must' (AND) or 'should' (OR).

=item excluding

Also ignored.

=item operator

What form of search to do. Options are: is (phrase, no truncation, whole field
must match), = (number exact match), exact (phrase, no truncation, whole field
must match). If left blank, then word list, right truncated, anywhere is used.

=item value

The actual user-provided string value to search for.

=item authtypecode

The authority type code to search within. If blank, then all will be searched.

=item orderby

The order to sort the results by. Options are Relevance, HeadingAsc,
HeadingDsc, AuthidAsc, AuthidDsc.

=back

marclist, operator, and value must be the same length, and the values at
index /i/ all relate to each other.

This returns a query, which is a black box object that can be passed to the
appropriate search object.

=cut

our $koha_to_index_name = {
    mainmainentry           => 'heading-main',
    mainentry               => 'heading',
    match                   => 'match',
    'match-heading'         => 'match-heading',
    'see-from'              => 'match-heading-see-from',
    thesaurus               => 'subject-heading-thesaurus',
    'thesaurus-conventions' => 'subject-heading-thesaurus-conventions',
    any                     => '',
    all                     => ''
};

# Delegates to Koha::Authority's table so there is a single source of
# truth for thesaurus-name -> 008/11-code (bug 31925).
our $thesaurus_to_value = $Koha::Authority::MARC21_THESAURUS_TO_CONTROL_FIELD_008_11;

sub build_authorities_query_compat {
    my (
        $self,         $marclist, $and_or, $excluding, $operator, $value,
        $authtypecode, $orderby
    ) = @_;

    # This turns the old-style many-options argument form into a more
    # extensible hash form that is understood by L<build_authorities_query>.
    my @searches;
    my $mappings = $self->get_elasticsearch_mappings();

    # Convert to lower case
    $marclist = [ map( lc, @{$marclist} ) ];
    $orderby  = lc $orderby;

    my @indexes;

    # Make sure everything exists
    foreach my $m (@$marclist) {

        $m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
        push @indexes, $m;
        warn "Unknown search field $m in marclist"
            unless ( defined $mappings->{properties}->{$m} || $m eq '' || $m eq 'match-heading' );
    }
    for ( my $i = 0 ; $i < @$value ; $i++ ) {
        next unless $value->[$i];    #clean empty form values, ES doesn't like undefined searches
        $value->[$i] = $thesaurus_to_value->{ $value->[$i] }
            if ( defined $thesaurus_to_value->{ $value->[$i] } && $indexes[$i] eq 'subject-heading-thesaurus' );
        push @searches,
            {
            where    => $indexes[$i],
            operator => $operator->[$i],
            value    => $value->[$i],
            };
    }

    my %sort;
    my $sort_field =
          ( $orderby =~ /^heading/ ) ? 'heading__sort'
        : ( $orderby =~ /^auth/ )    ? 'local-number__sort'
        :                              undef;
    if ($sort_field) {
        my $sort_order = ( $orderby =~ /asc$/ ) ? 'asc' : 'desc';
        %sort = ( $sort_field => $sort_order, );
    }
    my %search = (
        searches     => \@searches,
        authtypecode => $authtypecode,
        and_or       => $and_or,
    );
    $search{sort} = \%sort if %sort;
    my $query = $self->build_authorities_query( \%search );
    return $query;
}

=head2 _build_scan_query

    my ($query, $query_str) = $builder->_build_scan_query(\@operands, \@indexes)

This will build an aggregation scan query that can be issued to elasticsearch from
the provided string input.

=cut

our %scan_field_convert = (
    'ti'      => 'title',
    'au'      => 'author',
    'su'      => 'subject',
    'se'      => 'title-series',
    'pb'      => 'publisher',
    'callnum' => 'local-classification',
);

sub _build_scan_query {
    my ( $self, $operands, $indexes ) = @_;

    my $term  = scalar(@$operands) == 0 ? ''        : $operands->[0];
    my $index = scalar(@$indexes) == 0  ? 'subject' : $indexes->[0];

    my ( $f, $d ) = split( /,/, $index );
    $index = $scan_field_convert{$f} || $f;

    my $res;
    $res->{query}        = { query_string => { query => '*' } };
    $res->{aggregations} = {
        $index => {
            terms => {
                field   => $index . '__facet',
                order   => { '_key' => 'asc' },
                include => $self->_create_regex_filter( $self->clean_search_term($term) ) . '.*'
            }
        }
    };
    return ( $res, $term );
}

=head2 _create_regex_filter

    my $filter = $builder->_create_regex_filter('term')

This will create a regex filter that can be used with an aggregation query.

=cut

sub _create_regex_filter {
    my ( $self, $term ) = @_;

    my $result = '';
    foreach my $c ( split( //, quotemeta($term) ) ) {
        my $lc = lc($c);
        my $uc = uc($c);
        $result .= $lc ne $uc ? '[' . $lc . $uc . ']' : $c;
    }
    return $result;
}

=head2 _convert_sort_fields

    my @sort_params = _convert_sort_fields(@sort_by)

Converts the zebra-style sort index information into elasticsearch-style.

C<@sort_by> is the same as presented to L<build_query_compat>, and it returns
something that can be sent to L<build_query>.

=cut

sub _convert_sort_fields {
    my ( $self, @sort_by ) = @_;

    # Turn the sorting into something we care about.
    my %sort_field_convert = (
        acqdate      => 'date-of-acquisition',
        author       => 'author',
        call_number  => 'cn-sort',
        popularity   => 'issues',
        relevance    => undef,                   # default
        title        => 'title',
        pubdate      => 'date-of-publication',
        biblionumber => 'local-number',
    );
    my %sort_order_convert = ( qw( desc desc ), qw( dsc desc ), qw( asc asc ), qw( az asc ), qw( za desc ) );

    # Convert the fields and orders, drop anything we don't know about.
    grep { $_->{field} } map {
        my ( $f, $d ) = /(.+)_(.+)/;
        defined $f && defined $d
            ? {
            field     => $sort_field_convert{$f},
            direction => $sort_order_convert{$d}
            }
            : { field => undef };
    } @sort_by;
}

=head2 _convert_index_fields

    my ($conv) = $self->_convert_index_fields($field);

Converts the zebra-style sort index information into elasticsearch-style.

 Convert according to our table, drop anything that doesn't convert.
 If a field starts with mc- we save it as it's used (and removed) later
 when joining things, to indicate we make it an 'OR' join.
 (Sorry, this got a bit ugly after special cases were found.)

=cut

sub _convert_index_fields {
    my ( $self, @indexes ) = @_;

    my %index_type_convert = ( __default => undef, phr => 'phrase', rtrn => 'right-truncate', 'st-year' => 'st-year' );

    @indexes = grep { $_ ne q{} } @indexes;    # Remove any blank indexes, i.e. keyword

    map {
        # Lower case all field names
        my ( $f, $t ) = map( lc, split /,/ );
        my $mc = '';
        if ( $f =~ /^mc-/ ) {
            $mc = 'mc-';
            $f =~ s/^mc-//;
        }
        my $r = {
            field => exists $index_field_convert{$f} ? $index_field_convert{$f} : $f,
            type  => $index_type_convert{ $t // '__default' }
        };
        $r->{field} .= '-all'
            if C4::Context->preference('SearchCancelledAndInvalidISBNandISSN') && $r->{field} =~ /^is[bs]n$/;
        $r->{field} = ( $mc . $r->{field} ) if $mc && $r->{field};
        $r->{field} || $r->{type} ? $r : undef;
    } @indexes;
}

=head2 _convert_index_strings

    my @searches = $self->_convert_index_strings(@searches);

Similar to L<_convert_index_fields>, this takes strings of the form
B<field:search term> and rewrites the field from zebra-style to
elasticsearch-style. Anything it doesn't understand is returned verbatim.

=cut

sub _convert_index_strings {
    my ( $self, @searches ) = @_;
    my @res;
    foreach my $s (@searches) {
        next if $s eq '';
        my ( $field, $term ) = $s =~ /^\s*([\w,-]*?):(.*)/;
        unless ( defined($field) && defined($term) ) {
            push @res, $s;
            next;
        }
        my ($conv) = $self->_convert_index_fields($field);
        unless ( defined($conv) ) {
            push @res, $s;
            next;
        }
        push @res,
            ( $conv->{field} ? $conv->{field} . ':' : '' ) . $self->_modify_string_by_type( %$conv, operand => $term );
    }
    return @res;
}

=head2 _convert_index_strings_freeform

    my $search = $self->_convert_index_strings_freeform($search);

This is similar to L<_convert_index_strings>, however it'll search out the
things to change within the string. So it can handle strings such as
C<(su:foo) AND (su:bar)>, converting the C<su> appropriately.

If there is something of the form "su,complete-subfield" or something, the
second part is stripped off as we can't yet handle that. Making it work
will have to wait for a real query parser.

=cut

sub _convert_index_strings_freeform {
    my ( $self, $search ) = @_;

    # @TODO: Currently will alter also fields contained within quotes:
    # `searching for "stuff cn:123"` for example will become
    # `searching for "stuff local-number:123"
    #
    # Fixing this is tricky, one possibility:
    # https://stackoverflow.com/questions/19193876/perl-regex-to-match-a-string-that-is-not-enclosed-in-quotes
    # Still not perfect, and will not handle escaped quotes within quotes and assumes balanced quotes.
    #
    # Another, not so elegant, solution could be to replace all quoted content with placeholders, and put
    # them back when processing is done.

    # Lower case field names
    $search =~ s/($field_name_pattern)(?:,[\w-]*)?($multi_field_pattern):/\L$1\E$2:/og;

    # Resolve possible field aliases
    $search =~
        s/($field_name_pattern)($multi_field_pattern):/(exists $index_field_convert{$1} ? $index_field_convert{$1} : $1).($1 eq 'kw' ? "$2" : "$2:")/oge;
    if ( C4::Context->preference('SearchCancelledAndInvalidISBNandISSN') ) {
        $search =~ s/\b(is[bs]n)(?=$multi_field_pattern:)/$1-all/g;
    }
    return $search;
}

=head2 _modify_string_by_type

    my $str = $self->_modify_string_by_type(%index_field);

If you have a search term (operand) and a type (phrase, right-truncated), this
will convert the string to have the function in lucene search terms, e.g.
wrapping quotes around it.

=cut

sub _modify_string_by_type {
    my ( $self, %idx ) = @_;

    my $type = $idx{type} || '';
    my $str  = $idx{operand};
    return $str unless $str;    # Empty or undef, we can't use it.

    $str .= '*'             if $type eq 'right-truncate';
    $str = '"' . $str . '"' if $type eq 'phrase' && $str !~ /^".*"$/;
    if ( $type eq 'st-year' ) {
        if ( $str =~ /^(.*)-(.*)$/ ) {
            my $from  = $1 || '*';
            my $until = $2 || '*';
            $str = "[$from TO $until]";
        }
    } elsif ( $type eq 'st-date-normalized' ) {
        if ( $str =~ /^(.*) - (.*)$/ ) {
            my $from  = $1 || '*';
            my $until = $2 || '*';
            $str = "[$from TO $until]";
        }
    }
    return $str;
}

=head2 _join_queries

    my $query_str = $self->_join_queries(@query_parts);

This takes a list of query parts, that might be search terms on their own, or
booleaned together, or specifying fields, or whatever, wraps them in
parentheses, and ANDs them all together. Suitable for feeding to the ES
query string query.

Note: doesn't AND them together if they specify an index that starts with "mc"
as that was a special case in the original code for dealing with multiple
choice options (you can't search for something that has an itype of A and
and itype of B otherwise.)

=cut

sub _join_queries {
    my ( $self, @parts ) = @_;

    my @norm_parts = grep { defined($_) && $_ ne '' && $_ !~ /^mc-/ } @parts;
    my @mc_parts =
        map { s/^mc-//r } grep { defined($_) && $_ ne '' && $_ =~ /^mc-/ } @parts;
    return () unless @norm_parts + @mc_parts;
    return ( @norm_parts, @mc_parts )[0] if @norm_parts + @mc_parts == 1;

    # Group limits by field, so they can be OR'ed together
    my %mc_limits;
    foreach my $mc_part (@mc_parts) {
        my ( $field, $value ) = split /:/, $mc_part, 2;
        $mc_limits{$field} //= [];
        push @{ $mc_limits{$field} }, $value;
    }

    @mc_parts = map { sprintf( '%s:(%s)', $_, join( ' OR ', @{ $mc_limits{$_} } ) ); } sort keys %mc_limits;

    @norm_parts = map { "($_)" } @norm_parts;

    return join( ' AND ', @norm_parts, @mc_parts );
}

=head2 _make_phrases

    my @phrased_queries = $self->_make_phrases(@query_parts);

This takes the supplied queries and forces them to be phrases by wrapping
quotes around them. It understands field prefixes, e.g. 'subject:' and puts
the quotes outside of them if they're there.

=cut

sub _make_phrases {
    my ( $self, @parts ) = @_;
    map { s/^\s*(\w*?:)(.*)$/$1"$2"/r } @parts;
}

=head2 _create_query_string

    my @query_strings = $self->_create_query_string(@queries);

Given a list of hashrefs, it will turn them into a lucene-style query string.
The hash should contain field, type (both for the indexes), operator, and
operand.

=cut

sub _create_query_string {
    my ( $self, @queries ) = @_;
    $es_advanced_searches = [];
    my @string_queries;
    foreach my $q (@queries) {
        if ( $q->{field} && $q->{field} eq 'geolocation' ) {
            push( @$es_advanced_searches, $q );
        } else {
            push( @string_queries, $q );
        }
    }

    @queries = @string_queries;

    map {
        my $otor  = $_->{operator} ? $_->{operator} . ' ' : '';
        my $field = $_->{field}    ? $_->{field} . ':'    : '';

        my $oand = $self->_modify_string_by_type(%$_);
        $oand = "($oand)"
            if $field && scalar( split( /\s+/, $oand ) ) > 1 && ( !defined $_->{type} || $_->{type} ne 'st-year' );
        "$otor($field$oand)";
    } @queries;
}

=head2 clean_search_term

    my $term = $self->clean_search_term($term);

This cleans a search term by removing any funny characters that may upset
ES and give us an error. It also calls L<_convert_index_strings_freeform>
to ensure those parts are correct.

=cut

sub clean_search_term {
    my ( $self, $term ) = @_;

    # Lookahead for checking if we are inside quotes
    my $lookahead = '(?=(?:[^\"]*+\"[^\"]*+\")*+[^\"]*+$)';

    # Some hardcoded searches (like with authorities) produce things like
    # 'an=123', when it ought to be 'an:123' for our purposes.
    $term =~ s/=/:/g;

    $term = $self->_convert_index_strings_freeform($term);

    # escape special characters
    $term = $self->_escape_special_characters($term);

    # Remove unbalanced quotes
    my $unquoted = $term;
    my $count    = ( $unquoted =~ tr/"/ / );
    if ( $count % 2 == 1 ) {
        $term = $unquoted;
    }
    $term = $self->_query_regex_escape_process($term);

    # because of _truncate_terms and if QueryAutoTruncate enabled
    # we will have any special operators ruined by _truncate_terms:
    # for ex. search for "test [6 TO 7]" will be converted to "test* [6* TO* 7]"
    # so no reason to keep ranges in QueryAutoTruncate==true case:
    my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
    unless ($truncate) {

        # replace all ranges with any square/curly brackets combinations to temporary substitutions (ex: "{a TO b]"" -> "~~LC~~a TO b~~RS~~")
        # (where L is for left and C is for Curly and so on)
        $term =~ s/
            (?<!\\)
            (?<backslashes>(?:[\\]{2})*)
            (?<leftbracket>\{|\[)
            (?<ranges>
                [^\s\[\]\{\}]+\ TO\ [^\s\[\]\{\}]+
                (?<!\\)
                (?:[\\]{2})*
            )
            (?<rightbracket>\}|\])
        /$+{backslashes}.'~~L'.($+{leftbracket} eq '[' ? 'S':'C').'~~'.$+{ranges}.'~~R'.($+{rightbracket} eq ']' ? 'S':'C').'~~'/gex;
    }

    # save all regex contents away before escaping brackets:
    # (same trick as with brackets above, just RE for 'RegularExpression')
    my @saved_regexes;
    my $rgx_i = 0;
    while (
        $term =~ s@(
                (?<!\\)(?:[\\]{2})*/
                (?:[^/]+|(?<=\\)(?:[\\]{2})*/)+
                (?<!\\)(?:[\\]{2})*/
            )$lookahead@~~RE$rgx_i~~@x
        )
    {
        @saved_regexes[ $rgx_i++ ] = $1;
    }

    # remove leading and trailing colons mixed with optional slashes and spaces
    $term =~ s/^([\s\\]*:\s*)+//;
    $term =~ s/([\s\\]*:\s*)+$//;

    # remove unquoted colons that have whitespace on either side of them
    $term =~ s/([\s\\]*:\s*)+(\s+)$lookahead/$2/g;
    $term =~ s/(\s+)([\s\\]*:\s*)+$lookahead/$1/g;

    # replace with spaces all repeated colons no matter how they surrounded with spaces and slashes
    $term =~ s/([\s\\]*:\s*){2,}$lookahead/ /g;

    # screen all followups for colons after first colon,
    # and correctly ignore unevenly backslashed:
    $term =~ s/((?<!\\)(?:[\\]{2})*:[^:\s]+(?<!\\)(?:[\\]{2})*)(?=:)/$1\\/g;

    # screen all exclamation signs that either are the last symbol or have white space after them
    # or are followed by close parentheses
    $term =~ s/(?:[\s\\]*!\s*)+(\s|$|\))/$1/g;

    # screen all brackets with backslash
    $term =~ s/(?<!\\)(?:[\\]{2})*([\{\}\[\]])$lookahead/\\$1/g;

    # Remove problematic punctuation  and escaped slashes surrounded by spaces if truncate.
    # This mainly relates to ISBD punctuation.
    # Note that slashes identified as RE limits will not be removed nor the contents of RE
    # will be altered (cf. @saved_regexes).  In other cases (i.e. escaped slashes
    # (cf. QueryRegexEscapeOptions) and unescaped slashes not identified as being limits
    # to a RE, i.e. the last odd unescaped slash), the slashes will be removed.
    $term =~ s/\s([&;,:\.=\-\/\s]|(\\\/))+\s$lookahead/ /g if $truncate;

    # restore all regex contents after escaping brackets:
    for ( my $i = 0 ; $i < @saved_regexes ; $i++ ) {
        $term =~ s/~~RE$i~~/$saved_regexes[$i]/;
    }
    unless ($truncate) {

        # restore temporary weird substitutions back to normal brackets
        $term =~
            s/~~L(C|S)~~([^\s\[\]\{\}]+ TO [^\s\[\]\{\}]+)~~R(C|S)~~/($1 eq 'S' ? '[':'{').$2.($3 eq 'S' ? ']':'}')/ge;
    }
    return $term;
}

=head2 _escape_special_characters

    $term = $self->_escape_special_characters($term);

Escapes characters in $term according to "ElasticsearchEscapeCharacters" /
"OpacElasticsearchEscapeCharacters" system preference setting.

=cut

sub _escape_special_characters {
    my ( $self, $term ) = @_;
    my $characters_to_escape;
    if ( C4::Context->interface eq 'opac' ) {
        $characters_to_escape = C4::Context->preference('OpacElasticsearchEscapeCharacters');
    } else {
        $characters_to_escape = C4::Context->preference('ElasticsearchEscapeCharacters');
    }
    if ($characters_to_escape) {
        $characters_to_escape =~ s/\s+//g;
        $term                 =~ s/[\Q$characters_to_escape\E]/\\$&/g if $characters_to_escape;
    }
    return $term;
}

=head2 _query_regex_escape_process

    my $query = $self->_query_regex_escape_process($query);

Processes query in accordance with current "QueryRegexEscapeOptions" system preference setting.

=cut

sub _query_regex_escape_process {
    my ( $self, $query ) = @_;
    my $regex_escape_options = C4::Context->preference("QueryRegexEscapeOptions");
    if ( $regex_escape_options ne 'dont_escape' ) {
        if ( $regex_escape_options eq 'escape' ) {

            # Will escape unescaped slashes (/) while preserving
            # unescaped slashes within quotes
            # @TODO: assumes quotes are always balanced and will
            # not handle escaped quotes properly, should perhaps be
            # replaced with a more general parser solution
            # so that this function is ever only provided with unquoted
            # query parts
            $query =~ s@(?:(?<!\\)((?:[\\]{2})*)(?=/))(?![^"]*"(?:[^"]*"[^"]*")*[^"]*$)@\\$1@g;
        } elsif ( $regex_escape_options eq 'unescape_escaped' ) {

            # Will unescape escaped slashes (\/) and escape
            # unescaped slashes (/) while preserving slashes within quotes
            # The same limitatations as above apply for handling of quotes
            $query =~
                s@(?:(?<!\\)(?:((?:[\\]{2})*[\\])|((?:[\\]{2})*))(?=/))(?![^"]*"(?:[^"]*"[^"]*")*[^"]*$)@($1 ? substr($1, 0, -1) : ($2 . "\\"))@ge;
        }
    }
    return $query;
}

=head2 _fix_limit_special_cases

    my $limits = $self->_fix_limit_special_cases($limits);

This converts any special cases that the limit specifications have into things
that are more readily processable by the rest of the code.

The argument should be an arrayref, and it'll return an arrayref.

=cut

sub _fix_limit_special_cases {
    my ( $self, $limits ) = @_;

    my @new_lim;
    foreach my $l (@$limits) {

        # This is set up by opac-search.pl
        if ( $l =~ /^yr,st-numeric,ge[=:]/ ) {
            my ( $start, $end ) = ( $l =~ /^yr,st-numeric,ge[=:](.*) and yr,st-numeric,le[=:](.*)$/ );
            next unless defined($start) && defined($end);
            push @new_lim, "date-of-publication:[$start TO $end]";
        } elsif ( $l =~ /^search_filter:/ ) {

            # Here we are going to get the query as a string, clean it, and take care of the part of the limit
            # Calling build_query_compat here is avoided because we generate more complex query structures
            my ($filter_id) = ( $l =~ /^search_filter:(.*)$/ );
            my $search_filter = Koha::SearchFilters->find($filter_id);
            next unless $search_filter;
            my ( $expanded_lim, $query_lim ) = $search_filter->expand_filter;

            # In the case of nested filters we need to expand them all
            foreach my $el ( @{ $self->_fix_limit_special_cases($expanded_lim) } ) {
                push @new_lim, $el;
            }

            # We need to clean the query part as we have built a string from the original search
            push @new_lim, $self->clean_search_term($query_lim);
        } elsif ( $l =~ /^yr,st-numeric[=:]/ ) {
            my ($date) = ( $l =~ /^yr,st-numeric[=:](.*)$/ );
            next unless defined($date);
            $date = $self->_modify_string_by_type( type => 'st-year', operand => $date );
            push @new_lim, "date-of-publication:$date";
        } elsif ( $l =~ 'multibranchlimit|^branch' ) {
            my $branchfield = C4::Context->preference('SearchLimitLibrary');
            my @branchcodes;
            if ( $l =~ 'multibranchlimit' ) {
                my ($group_id) = ( $l =~ /^multibranchlimit:(.*)$/ );
                my $search_group = Koha::Library::Groups->find($group_id);
                @branchcodes = map  { $_->branchcode } $search_group->all_libraries;
                @branchcodes = sort { $a cmp $b } @branchcodes;
            } else {
                @branchcodes = ( $l =~ /^branch:(.*)$/ );
            }

            if (@branchcodes) {

                # We quote the branchcodes here to prevent issues when codes are reserved words in ES, e.g. OR, AND, NOT, etc.
                if ( $branchfield eq "homebranch" ) {
                    push @new_lim, sprintf "(%s)", join " OR ", map { 'homebranch: "' . $_ . '"' } @branchcodes;
                } elsif ( $branchfield eq "holdingbranch" ) {
                    push @new_lim, sprintf "(%s)", join " OR ", map { 'holdingbranch: "' . $_ . '"' } @branchcodes;
                } else {
                    push @new_lim, sprintf "(%s OR %s)",
                        join( " OR ", map { 'homebranch: "' . $_ . '"' } @branchcodes ),
                        join( " OR ", map { 'holdingbranch: "' . $_ . '"' } @branchcodes );
                }
            }
        } elsif ( $l =~ /^available$/ ) {
            push @new_lim, 'available:true';
        } elsif ( $l =~ /^\s*(kw\b[\w,-]*?):(.*)/ ) {
            my ( $field, $term ) = ( $1, $2 );
            if ( defined($field) && defined($term) && $field =~ /,phr$/ ) {
                push @new_lim, "(\"$term\")";
            } else {
                push @new_lim, $term;
            }
        } elsif ( $l =~ /^acqdate,st-date-normalized=/ ) {
            my ($date) = ( $l =~ /^acqdate,st-date-normalized=(.*)$/ );
            next unless defined($date);
            $date = $self->_modify_string_by_type( type => 'st-date-normalized', operand => $date );
            push @new_lim, "date-of-acquisition.raw:$date";
        } else {
            my ( $field, $term ) = $l =~ /^\s*([\w,-]*?):(.*)/;
            $field =~ s/,phr$//;    #We are quoting all the limits as phrase, this prevents from quoting again later
            if ( defined($field) && defined($term) ) {
                push @new_lim, "$field:(\"$term\")";
            } else {
                push @new_lim, $l;
            }
        }
    }
    return \@new_lim;
}

=head2 _sort_field

    my $field = $self->_sort_field($field);

Given a field name, this works out what the actual name of the field to sort
on should be. A '__sort' suffix is added for fields with a sort version, and
for text fields either '.phrase' (for sortable versions) or '.raw' is appended
to avoid sorting on a tokenized value.

=cut

sub _sort_field {
    my ( $self, $f ) = @_;

    my $mappings  = $self->get_elasticsearch_mappings();
    my $textField = defined $mappings->{properties}{$f}{type} && $mappings->{properties}{$f}{type} eq 'text';
    if ( !defined $self->sort_fields()->{$f} || $self->sort_fields()->{$f} ) {
        $f .= '__sort';
    } else {

        # We need to add '.raw' to text fields without a sort field,
        # otherwise it'll sort based on the tokenised form.
        $f .= '.raw' if $textField;
    }
    return $f;
}

=head2 _truncate_terms

    my $query = $self->_truncate_terms($query);

Given a string query this function appends '*' wildcard  to all terms except
operands and double quoted strings.

=cut

sub _truncate_terms {
    my ( $self, $query ) = @_;

    my @tokens = $self->_split_query($query);

    # Filter out empty tokens
    my @words = grep { $_ !~ /^\s*$/ } @tokens;

    # Append '*' to words if needed, ie. if it ends in a word character and is not a keyword
    my @terms = map {
        my $w = $_;
        ( /\W$/ or grep { lc($w) eq $_ } qw/and or not/ ) ? $_ : "$_*";
    } @words;

    return join ' ', @terms;
}

=head2 _split_query

    my @token = $self->_split_query($query_str);

Given a string query this function splits it to tokens taking into account
any field prefixes and quoted strings.

=cut

my $tokenize_split_re = qr/((?:${field_name_pattern}${multi_field_pattern}:)?"[^"]+"|\s+)/;

sub _split_query {
    my ( $self, $query ) = @_;

    # '"donald duck" title:"the mouse" and peter" get split into
    # ['', '"donald duck"', '', ' ', '', 'title:"the mouse"', '', ' ', 'and', ' ', 'pete']
    my @tokens = split $tokenize_split_re, $query;

    # Filter out empty values
    @tokens = grep( /\S/, @tokens );

    # Filter out some known isolated trouble makers: -, ^, \, ~, +
    @tokens = grep( !/^[-\^\\~\+]*$/, @tokens );

    return @tokens;
}

=head2 _search_fields
    my $weighted_fields = $self->_search_fields({
        is_opac => 0,
        weighted_fields => 1,
        subfield => 'raw'
    });

Generate a list of searchable fields to be used for Elasticsearch queries
applied to multiple fields.

Returns an arrayref of field names for either OPAC or staff interface, with
possible weights and subfield appended to each field name depending on the
options provided.

=over 4

=item C<$params>

Hashref with options. The parameter C<is_opac> indicates whether the searchable
fields for OPAC or staff interface should be retrieved. If C<weighted_fields> is set
fields weights will be applied on returned fields. C<subfield> can be used to
provide a subfield that will be appended to fields as "C<field_name>.C<subfield>".

=back

=cut

sub _search_fields {
    my ( $self, $params ) = @_;
    $params //= {
        is_opac         => 0,
        weighted_fields => 0,
        whole_record    => 0,

        # This is a hack for authorities build_authorities_query
        # can hopefully be removed in the future
        subfield => undef,
    };
    my $cache = Koha::Caches->get_instance();
    my $cache_key =
        'elasticsearch_search_fields' . ( $params->{is_opac} ? '_opac' : '_staff_client' ) . "_" . $self->index;
    my $search_fields = $cache->get_from_cache( $cache_key, { unsafe => 1 } );
    if ( !$search_fields ) {

        # The reason we don't use Koha::SearchFields->search here is we don't
        # want or need resultset wrapped as Koha::SearchField object.
        # It does not make any sense in this context and would cause
        # unnecessary overhead sice we are only querying for data
        # Also would not work, or produce strange results, with the "columns"
        # option.
        my $schema = Koha::Database->schema;
        my $result = $schema->resultset('SearchField')->search(
            {
                $params->{is_opac}
                ? (
                    'opac' => 1,
                    )
                : ( 'staff_client' => 1 ),
                'type'                         => { '!=' => 'boolean' },
                'search_marc_map.index_name'   => $self->index,
                'search_marc_map.marc_type'    => C4::Context->preference('marcflavour'),
                'search_marc_to_fields.search' => 1,
            },
            {
                columns  => [qw/name weight/],
                collapse => 1,
                join     => { search_marc_to_fields => 'search_marc_map' },
            }
        );
        my @search_fields;
        while ( my $search_field = $result->next ) {
            push @search_fields, [
                lc $search_field->name,
                $search_field->weight ? $search_field->weight : ()
            ];
        }
        $search_fields = \@search_fields;
        $cache->set_in_cache( $cache_key, $search_fields );
    }
    if ( $params->{subfield} ) {
        my $subfield = $params->{subfield};
        $search_fields = [
            map {
                # Copy values to avoid mutating cached
                # data (since unsafe is used)
                my ( $field, $weight ) = @{$_};
                [ "${field}.${subfield}", $weight ];
            } @{$search_fields}
        ];
    }
    if ( $params->{weighted_fields} ) {
        return [ map { join( '^', @{$_} ) } @{$search_fields} ];
    } else {

        # Exclude weight from field
        return [ map { $_->[0] } @{$search_fields} ];
    }
}

=pod

=head2 _is_safe_to_auto_truncate

_is_safe_to_auto_truncate($index_field, $oand);

Checks if it is safe to auto truncate a search term within a given search field.

The search term should not be auto truncated when searching for identifiers, e.g.
koha-auth-number, record-control-number, local-number etc.  Also, non-text fields
must not be auto truncated (doing so would generate ES exception).

=cut

sub _is_safe_to_auto_truncate {
    my ( $self, $index_field, $oand ) = @_;

    # Do not auto truncate fields that should not be auto truncated,
    # primarily various types of identifiers, above all record identifiers.
    # Other search fields that should not be auto truncated can be defined
    # with ElasticsearchPreventAutoTruncate syspref.
    my %do_not_autotruncate_fields;
    my $cache                          = Koha::Caches->get_instance();
    my $cache_key                      = 'elasticsearch_search_do_not_autotruncate';
    my $do_not_autotruncate_fields_ref = $cache->get_from_cache( $cache_key, { unsafe => 1 } );
    %do_not_autotruncate_fields = %$do_not_autotruncate_fields_ref if $do_not_autotruncate_fields_ref;
    if ( !scalar( keys %do_not_autotruncate_fields ) ) {
        %do_not_autotruncate_fields =
            map { $_ => 1 } qw / biblioitemnumber host-item-number itemnumber koha-auth-number local-number /;

        # In addition, under no circumstances should non-text fields
        # be auto truncated.
        my $schema = Koha::Database->new()->schema();
        my $sfs =
            $schema->resultset('SearchField')
            ->search(
            { '-or' => [ { type => 'boolean' }, { type => 'number' }, { type => 'sum' }, { type => 'year' } ] } );
        while ( my $sf = $sfs->next ) {
            $do_not_autotruncate_fields{ $sf->name } = 1;
        }
        $cache->set_in_cache( $cache_key, \%do_not_autotruncate_fields );
    }

    # processing of the syspref is done outside cache since the systempreference
    # can be modified and the modification should be reflected in the
    # $do_not_autotruncate_fields array
    my $prevent_autotruncate = C4::Context->preference('ElasticsearchPreventAutoTruncate');
    for my $field ( split( /\s*[,;|]\s*/, $prevent_autotruncate ) ) {
        $do_not_autotruncate_fields{$field} = 1;
    }

    # search fields can be given as a explicit index name (e.g. from advanced
    # search):
    if ($index_field) {
        return 0 if grep { $index_field eq $_ } keys %do_not_autotruncate_fields;

        # OR can be given implicitly, as prefix in the operand (e.g. in links generated
        # by Koha like catalogue/search.pl?q=an:<authid>):
    } elsif ( $oand =~ /\b($field_name_pattern):/ ) {    # check field name prefixing operand
        my $field_name = $1;
        return 0 if grep { $field_name eq $_ } keys %do_not_autotruncate_fields;
    }

    # It is safe to auto truncate:
    return 1;
}

=head2 _rebuild_to_es_advanced_query

    $res = _rebuild_to_es_advanced_query( $res );

Passed in a query, this function converts a geolocation query into the necessary format for Elasticsearch

=cut

sub _rebuild_to_es_advanced_query {
    my ($res) = @_;
    my $query_string = $res->{query}->{query_string};
    $query_string->{query} = '*' unless $query_string->{query};
    delete $res->{query}->{query_string};

    my %filter;
    for my $advanced_query (@$es_advanced_searches) {
        if ( $advanced_query->{field} eq 'geolocation' ) {
            my ( $lat, $lon, $distance ) = map { $_ =~ /:(.*)\*/ } split( '\s+', $advanced_query->{operand} );
            $filter{geo_distance} = {
                distance    => $distance,
                geolocation => {
                    lat => $lat,
                    lon => $lon,
                }
            };
        } else {
            warn "unknown advanced ElasticSearch query: " . join( ', ', %$advanced_query );
        }
    }

    $res->{query} = {
        bool => {
            must   => { query_string => $query_string },
            filter => \%filter,
        }
    };

    return $res;
}

1;