1078 lines
33 KiB
Perl
1078 lines
33 KiB
Perl
package C4::Matcher;
|
|
|
|
# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
|
|
#
|
|
# This file is part of Koha.
|
|
#
|
|
# Koha is free software; you can redistribute it and/or modify it
|
|
# under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Koha is distributed in the hope that it will be useful, but
|
|
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with Koha; if not, see <https://www.gnu.org/licenses>.
|
|
|
|
use Modern::Perl;
|
|
|
|
use Koha::SearchEngine;
|
|
use Koha::SearchEngine::Search;
|
|
use Koha::SearchEngine::QueryBuilder;
|
|
use Koha::Util::Normalize qw(
|
|
ISBN
|
|
legacy_default
|
|
lower_case
|
|
remove_spaces
|
|
upper_case
|
|
);
|
|
|
|
=head1 NAME
|
|
|
|
C4::Matcher - find MARC records matching another one
|
|
|
|
=head1 SYNOPSIS
|
|
|
|
my @matchers = C4::Matcher::GetMatcherList();
|
|
|
|
my $matcher = C4::Matcher->new($record_type);
|
|
$matcher->threshold($threshold);
|
|
$matcher->code($code);
|
|
$matcher->description($description);
|
|
|
|
$matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
|
|
$matcher->add_simple_matchpoint('Date', 1000, '008', '', 7, 4, '');
|
|
$matcher->add_matchpoint('isbn', 1000, [ { tag => '020', subfields => 'a', norms => [] } ]);
|
|
|
|
$matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, '');
|
|
$matcher->add_required_check([ { tag => '245', subfields => 'a', norms => [] } ],
|
|
[ { tag => '245', subfields => 'a', norms => [] } ]);
|
|
|
|
my @matches = $matcher->get_matches($marc_record, $max_matches);
|
|
|
|
foreach $match (@matches) {
|
|
|
|
# matches already sorted in order of
|
|
# decreasing score
|
|
print "record ID: $match->{'record_id'};
|
|
print "score: $match->{'score'};
|
|
|
|
}
|
|
|
|
my $matcher_description = $matcher->dump();
|
|
|
|
=head1 FUNCTIONS
|
|
|
|
=cut
|
|
|
|
=head2 GetMatcherList
|
|
|
|
my @matchers = C4::Matcher::GetMatcherList();
|
|
|
|
Returns an array of hashrefs list all matchers
|
|
present in the database. Each hashref includes:
|
|
|
|
* matcher_id
|
|
* code
|
|
* description
|
|
|
|
=cut
|
|
|
|
sub GetMatcherList {
|
|
my $dbh = C4::Context->dbh;
|
|
|
|
my $sth = $dbh->prepare_cached("SELECT matcher_id, code, description FROM marc_matchers ORDER BY matcher_id");
|
|
$sth->execute();
|
|
my @results = ();
|
|
while ( my $row = $sth->fetchrow_hashref ) {
|
|
push @results, $row;
|
|
}
|
|
return @results;
|
|
}
|
|
|
|
=head2 GetMatcherId
|
|
|
|
my $matcher_id = C4::Matcher::GetMatcherId($code);
|
|
|
|
Returns the matcher_id of a code.
|
|
|
|
=cut
|
|
|
|
sub GetMatcherId {
|
|
my ($code) = @_;
|
|
my $dbh = C4::Context->dbh;
|
|
|
|
my $matcher_id = $dbh->selectrow_array( "SELECT matcher_id FROM marc_matchers WHERE code = ?", undef, $code );
|
|
return $matcher_id;
|
|
}
|
|
|
|
=head1 METHODS
|
|
|
|
=head2 new
|
|
|
|
my $matcher = C4::Matcher->new($record_type, $threshold);
|
|
|
|
Creates a new Matcher. C<$record_type> indicates which search
|
|
database to use, e.g., 'biblio' or 'authority' and defaults to
|
|
'biblio', while C<$threshold> is the minimum score required for a match
|
|
and defaults to 1000.
|
|
|
|
=cut
|
|
|
|
sub new {
|
|
my $class = shift;
|
|
my $self = {};
|
|
|
|
$self->{'id'} = undef;
|
|
|
|
if ( $#_ > -1 ) {
|
|
$self->{'record_type'} = shift;
|
|
} else {
|
|
$self->{'record_type'} = 'biblio';
|
|
}
|
|
|
|
if ( $#_ > -1 ) {
|
|
$self->{'threshold'} = shift;
|
|
} else {
|
|
$self->{'threshold'} = 1000;
|
|
}
|
|
|
|
$self->{'code'} = '';
|
|
$self->{'description'} = '';
|
|
|
|
$self->{'matchpoints'} = [];
|
|
$self->{'required_checks'} = [];
|
|
|
|
bless $self, $class;
|
|
return $self;
|
|
}
|
|
|
|
=head2 fetch
|
|
|
|
my $matcher = C4::Matcher->fetch($id);
|
|
|
|
Creates a matcher object from the version stored
|
|
in the database. If a matcher with the given
|
|
id does not exist, returns undef.
|
|
|
|
=cut
|
|
|
|
sub fetch {
|
|
my $class = shift;
|
|
my $id = shift;
|
|
my $dbh = C4::Context->dbh();
|
|
|
|
my $sth = $dbh->prepare_cached("SELECT * FROM marc_matchers WHERE matcher_id = ?");
|
|
$sth->execute($id);
|
|
my $row = $sth->fetchrow_hashref;
|
|
$sth->finish();
|
|
return unless defined $row;
|
|
|
|
my $self = {};
|
|
$self->{'id'} = $row->{'matcher_id'};
|
|
$self->{'record_type'} = $row->{'record_type'};
|
|
$self->{'code'} = $row->{'code'};
|
|
$self->{'description'} = $row->{'description'};
|
|
$self->{'threshold'} = int( $row->{'threshold'} );
|
|
bless $self, $class;
|
|
|
|
# matchpoints
|
|
$self->{'matchpoints'} = [];
|
|
$sth = $dbh->prepare_cached("SELECT * FROM matcher_matchpoints WHERE matcher_id = ? ORDER BY matchpoint_id");
|
|
$sth->execute( $self->{'id'} );
|
|
while ( my $row = $sth->fetchrow_hashref ) {
|
|
my $matchpoint = $self->_fetch_matchpoint( $row->{'matchpoint_id'} );
|
|
push @{ $self->{'matchpoints'} }, $matchpoint;
|
|
}
|
|
|
|
# required checks
|
|
$self->{'required_checks'} = [];
|
|
$sth = $dbh->prepare_cached("SELECT * FROM matchchecks WHERE matcher_id = ? ORDER BY matchcheck_id");
|
|
$sth->execute( $self->{'id'} );
|
|
while ( my $row = $sth->fetchrow_hashref ) {
|
|
my $source_matchpoint = $self->_fetch_matchpoint( $row->{'source_matchpoint_id'} );
|
|
my $target_matchpoint = $self->_fetch_matchpoint( $row->{'target_matchpoint_id'} );
|
|
my $matchcheck = {};
|
|
$matchcheck->{'source_matchpoint'} = $source_matchpoint;
|
|
$matchcheck->{'target_matchpoint'} = $target_matchpoint;
|
|
push @{ $self->{'required_checks'} }, $matchcheck;
|
|
}
|
|
|
|
return $self;
|
|
}
|
|
|
|
=head3 _fetch_matchpoint
|
|
|
|
my $matchpoint = $self->_fetch_matchpoint($matchpoint_id);
|
|
|
|
Loads a matchpoint and its components from the database and returns a
|
|
hashref with C<index>, C<score>, and C<components>.
|
|
|
|
=cut
|
|
|
|
sub _fetch_matchpoint {
|
|
my $self = shift;
|
|
my $matchpoint_id = shift;
|
|
|
|
my $dbh = C4::Context->dbh;
|
|
my $sth = $dbh->prepare_cached("SELECT * FROM matchpoints WHERE matchpoint_id = ?");
|
|
$sth->execute($matchpoint_id);
|
|
my $row = $sth->fetchrow_hashref;
|
|
my $matchpoint = {};
|
|
$matchpoint->{'index'} = $row->{'search_index'};
|
|
$matchpoint->{'score'} = int( $row->{'score'} );
|
|
$sth->finish();
|
|
|
|
$matchpoint->{'components'} = [];
|
|
$sth = $dbh->prepare_cached("SELECT * FROM matchpoint_components WHERE matchpoint_id = ? ORDER BY sequence");
|
|
$sth->execute($matchpoint_id);
|
|
while ( $row = $sth->fetchrow_hashref ) {
|
|
my $component = {};
|
|
$component->{'tag'} = $row->{'tag'};
|
|
$component->{'subfields'} = { map { $_ => 1 } split( //, $row->{'subfields'} ) };
|
|
$component->{'offset'} = int( $row->{'offset'} );
|
|
$component->{'length'} = int( $row->{'length'} );
|
|
$component->{'norms'} = [];
|
|
my $sth2 = $dbh->prepare_cached(
|
|
"SELECT *
|
|
FROM matchpoint_component_norms
|
|
WHERE matchpoint_component_id = ? ORDER BY sequence"
|
|
);
|
|
$sth2->execute( $row->{'matchpoint_component_id'} );
|
|
|
|
while ( my $row2 = $sth2->fetchrow_hashref ) {
|
|
push @{ $component->{'norms'} }, $row2->{'norm_routine'};
|
|
}
|
|
push @{ $matchpoint->{'components'} }, $component;
|
|
}
|
|
return $matchpoint;
|
|
}
|
|
|
|
=head2 store
|
|
|
|
my $id = $matcher->store();
|
|
|
|
Stores matcher in database. The return value is the ID
|
|
of the marc_matchers row. If the matcher was
|
|
previously retrieved from the database via the fetch()
|
|
method, the DB representation of the matcher
|
|
is replaced.
|
|
|
|
=cut
|
|
|
|
sub store {
|
|
my $self = shift;
|
|
|
|
if ( defined $self->{'id'} ) {
|
|
|
|
# update
|
|
$self->_del_matcher_components();
|
|
$self->_update_marc_matchers();
|
|
} else {
|
|
|
|
# create new
|
|
$self->_new_marc_matchers();
|
|
}
|
|
$self->_store_matcher_components();
|
|
return $self->{'id'};
|
|
}
|
|
|
|
=head3 _del_matcher_components
|
|
|
|
$self->_del_matcher_components();
|
|
|
|
Deletes all matchpoint component and norm rows for the current matcher
|
|
from the database (used before re-saving).
|
|
|
|
=cut
|
|
|
|
sub _del_matcher_components {
|
|
my $self = shift;
|
|
|
|
my $dbh = C4::Context->dbh();
|
|
my $sth = $dbh->prepare_cached("DELETE FROM matchpoints WHERE matcher_id = ?");
|
|
$sth->execute( $self->{'id'} );
|
|
$sth = $dbh->prepare_cached("DELETE FROM matchchecks WHERE matcher_id = ?");
|
|
$sth->execute( $self->{'id'} );
|
|
|
|
# foreign key delete cascades take care of deleting relevant rows
|
|
# from matcher_matchpoints, matchpoint_components, and
|
|
# matchpoint_component_norms
|
|
}
|
|
|
|
=head3 _update_marc_matchers
|
|
|
|
$self->_update_marc_matchers();
|
|
|
|
Updates the C<marc_matchers> row for the current matcher object.
|
|
|
|
=cut
|
|
|
|
sub _update_marc_matchers {
|
|
my $self = shift;
|
|
|
|
my $dbh = C4::Context->dbh();
|
|
my $sth = $dbh->prepare_cached(
|
|
"UPDATE marc_matchers
|
|
SET code = ?,
|
|
description = ?,
|
|
record_type = ?,
|
|
threshold = ?
|
|
WHERE matcher_id = ?"
|
|
);
|
|
$sth->execute(
|
|
$self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'},
|
|
$self->{'id'}
|
|
);
|
|
}
|
|
|
|
=head3 _new_marc_matchers
|
|
|
|
$self->_new_marc_matchers();
|
|
|
|
Inserts a new row into C<marc_matchers> and sets C<$self-E<gt>{'id'}>.
|
|
|
|
=cut
|
|
|
|
sub _new_marc_matchers {
|
|
my $self = shift;
|
|
|
|
my $dbh = C4::Context->dbh();
|
|
my $sth = $dbh->prepare_cached(
|
|
"INSERT INTO marc_matchers
|
|
(code, description, record_type, threshold)
|
|
VALUES (?, ?, ?, ?)"
|
|
);
|
|
$sth->execute( $self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'} );
|
|
$self->{'id'} = $dbh->{'mysql_insertid'};
|
|
}
|
|
|
|
=head3 _store_matcher_components
|
|
|
|
$self->_store_matcher_components($matchpoint_id, \@components);
|
|
|
|
Persists the component and norm rows for a matchpoint to the database.
|
|
|
|
=cut
|
|
|
|
sub _store_matcher_components {
|
|
my $self = shift;
|
|
|
|
my $dbh = C4::Context->dbh();
|
|
my $sth;
|
|
my $matcher_id = $self->{'id'};
|
|
foreach my $matchpoint ( @{ $self->{'matchpoints'} } ) {
|
|
my $matchpoint_id = $self->_store_matchpoint($matchpoint);
|
|
$sth = $dbh->prepare_cached(
|
|
"INSERT INTO matcher_matchpoints (matcher_id, matchpoint_id)
|
|
VALUES (?, ?)"
|
|
);
|
|
$sth->execute( $matcher_id, $matchpoint_id );
|
|
}
|
|
foreach my $matchcheck ( @{ $self->{'required_checks'} } ) {
|
|
my $source_matchpoint_id = $self->_store_matchpoint( $matchcheck->{'source_matchpoint'} );
|
|
my $target_matchpoint_id = $self->_store_matchpoint( $matchcheck->{'target_matchpoint'} );
|
|
$sth = $dbh->prepare_cached(
|
|
"INSERT INTO matchchecks
|
|
(matcher_id, source_matchpoint_id, target_matchpoint_id)
|
|
VALUES (?, ?, ?)"
|
|
);
|
|
$sth->execute( $matcher_id, $source_matchpoint_id, $target_matchpoint_id );
|
|
}
|
|
|
|
}
|
|
|
|
=head3 _store_matchpoint
|
|
|
|
my $matchpoint_id = $self->_store_matchpoint($matchpoint);
|
|
|
|
Inserts a matchpoint record into the database and returns its new ID.
|
|
|
|
=cut
|
|
|
|
sub _store_matchpoint {
|
|
my $self = shift;
|
|
my $matchpoint = shift;
|
|
|
|
my $dbh = C4::Context->dbh();
|
|
my $sth;
|
|
my $matcher_id = $self->{'id'};
|
|
$sth = $dbh->prepare_cached(
|
|
"INSERT INTO matchpoints (matcher_id, search_index, score)
|
|
VALUES (?, ?, ?)"
|
|
);
|
|
$sth->execute( $matcher_id, $matchpoint->{'index'}, $matchpoint->{'score'} || 0 );
|
|
my $matchpoint_id = $dbh->{'mysql_insertid'};
|
|
my $seqnum = 0;
|
|
|
|
foreach my $component ( @{ $matchpoint->{'components'} } ) {
|
|
$seqnum++;
|
|
$sth = $dbh->prepare_cached(
|
|
"INSERT INTO matchpoint_components
|
|
(matchpoint_id, sequence, tag, subfields, `offset`, length)
|
|
VALUES (?, ?, ?, ?, ?, ?)"
|
|
);
|
|
$sth->bind_param( 1, $matchpoint_id );
|
|
$sth->bind_param( 2, $seqnum );
|
|
$sth->bind_param( 3, $component->{'tag'} );
|
|
$sth->bind_param( 4, join "", sort keys %{ $component->{'subfields'} } );
|
|
$sth->bind_param( 5, $component->{'offset'} || 0 );
|
|
$sth->bind_param( 6, $component->{'length'} );
|
|
$sth->execute();
|
|
my $matchpoint_component_id = $dbh->{'mysql_insertid'};
|
|
my $normseq = 0;
|
|
|
|
foreach my $norm ( @{ $component->{'norms'} } ) {
|
|
$normseq++;
|
|
$sth = $dbh->prepare_cached(
|
|
"INSERT INTO matchpoint_component_norms
|
|
(matchpoint_component_id, sequence, norm_routine)
|
|
VALUES (?, ?, ?)"
|
|
);
|
|
$sth->execute( $matchpoint_component_id, $normseq, $norm );
|
|
}
|
|
}
|
|
return $matchpoint_id;
|
|
}
|
|
|
|
=head2 delete
|
|
|
|
C4::Matcher->delete($id);
|
|
|
|
Deletes the matcher of the specified ID
|
|
from the database.
|
|
|
|
=cut
|
|
|
|
sub delete {
|
|
my $class = shift;
|
|
my $matcher_id = shift;
|
|
|
|
my $dbh = C4::Context->dbh;
|
|
my $sth = $dbh->prepare("DELETE FROM marc_matchers WHERE matcher_id = ?");
|
|
$sth->execute($matcher_id); # relying on cascading deletes to clean up everything
|
|
}
|
|
|
|
=head2 record_type
|
|
|
|
$matcher->record_type('biblio');
|
|
my $record_type = $matcher->record_type();
|
|
|
|
Accessor method.
|
|
|
|
=cut
|
|
|
|
sub record_type {
|
|
my $self = shift;
|
|
@_ ? $self->{'record_type'} = shift : $self->{'record_type'};
|
|
}
|
|
|
|
=head2 threshold
|
|
|
|
$matcher->threshold(1000);
|
|
my $threshold = $matcher->threshold();
|
|
|
|
Accessor method.
|
|
|
|
=cut
|
|
|
|
sub threshold {
|
|
my $self = shift;
|
|
@_ ? $self->{'threshold'} = shift : $self->{'threshold'};
|
|
}
|
|
|
|
=head2 _id
|
|
|
|
$matcher->_id(123);
|
|
my $id = $matcher->_id();
|
|
|
|
Accessor method. Note that using this method
|
|
to set the DB ID of the matcher should not be
|
|
done outside of the editing CGI.
|
|
|
|
=cut
|
|
|
|
sub _id {
|
|
my $self = shift;
|
|
@_ ? $self->{'id'} = shift : $self->{'id'};
|
|
}
|
|
|
|
=head2 code
|
|
|
|
$matcher->code('ISBN');
|
|
my $code = $matcher->code();
|
|
|
|
Accessor method.
|
|
|
|
=cut
|
|
|
|
sub code {
|
|
my $self = shift;
|
|
@_ ? $self->{'code'} = shift : $self->{'code'};
|
|
}
|
|
|
|
=head2 description
|
|
|
|
$matcher->description('match on ISBN');
|
|
my $description = $matcher->description();
|
|
|
|
Accessor method.
|
|
|
|
=cut
|
|
|
|
sub description {
|
|
my $self = shift;
|
|
@_ ? $self->{'description'} = shift : $self->{'description'};
|
|
}
|
|
|
|
=head2 add_matchpoint
|
|
|
|
$matcher->add_matchpoint($index, $score, $matchcomponents);
|
|
|
|
Adds a matchpoint that may include multiple components. The $index
|
|
parameter identifies the index that will be searched, while $score
|
|
is the weight that will be added if a match is found.
|
|
|
|
$matchcomponents should be a reference to an array of matchpoint
|
|
components, each of which should be a hash containing the following
|
|
keys:
|
|
tag
|
|
subfields
|
|
offset
|
|
length
|
|
norms
|
|
|
|
The normalization_rules value should in turn be a reference to an
|
|
array, each element of which should be a reference to a
|
|
normalization subroutine (under C4::Normalize) to be applied
|
|
to the source string.
|
|
|
|
=cut
|
|
|
|
sub add_matchpoint {
|
|
my $self = shift;
|
|
my ( $index, $score, $matchcomponents ) = @_;
|
|
|
|
my $matchpoint = {};
|
|
$matchpoint->{'index'} = $index;
|
|
$matchpoint->{'score'} = $score;
|
|
$matchpoint->{'components'} = [];
|
|
foreach my $input_component ( @{$matchcomponents} ) {
|
|
push @{ $matchpoint->{'components'} }, _parse_match_component($input_component);
|
|
}
|
|
push @{ $self->{'matchpoints'} }, $matchpoint;
|
|
}
|
|
|
|
=head2 add_simple_matchpoint
|
|
|
|
$matcher->add_simple_matchpoint($index, $score, $source_tag,
|
|
$source_subfields, $source_offset,
|
|
$source_length, $source_normalizer);
|
|
|
|
|
|
Adds a simple matchpoint rule -- after composing a key based on the source tag and subfields,
|
|
normalized per the normalization function, search the index. All records retrieved
|
|
will receive the assigned score.
|
|
|
|
=cut
|
|
|
|
sub add_simple_matchpoint {
|
|
my $self = shift;
|
|
my ( $index, $score, $source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer ) = @_;
|
|
|
|
$self->add_matchpoint(
|
|
$index, $score,
|
|
[
|
|
{
|
|
tag => $source_tag, subfields => $source_subfields,
|
|
offset => $source_offset, 'length' => $source_length,
|
|
norms => [$source_normalizer]
|
|
}
|
|
]
|
|
);
|
|
}
|
|
|
|
=head2 add_required_check
|
|
|
|
$match->add_required_check($source_matchpoint, $target_matchpoint);
|
|
|
|
Adds a required check definition. A required check means that in
|
|
order for a match to be considered valid, the key derived from the
|
|
source (incoming) record must match the key derived from the target
|
|
(already in DB) record.
|
|
|
|
Unlike a regular matchpoint, only the first repeat of each tag
|
|
in the source and target match criteria are considered.
|
|
|
|
A typical example of a required check would be verifying that the
|
|
titles and publication dates match.
|
|
|
|
$source_matchpoint and $target_matchpoint are each a reference to
|
|
an array of hashes, where each hash follows the same definition
|
|
as the matchpoint component specification in add_matchpoint, i.e.,
|
|
|
|
tag
|
|
subfields
|
|
offset
|
|
length
|
|
norms
|
|
|
|
The normalization_rules value should in turn be a reference to an
|
|
array, each element of which should be a reference to a
|
|
normalization subroutine (under C4::Normalize) to be applied
|
|
to the source string.
|
|
|
|
=cut
|
|
|
|
sub add_required_check {
|
|
my $self = shift;
|
|
my ( $source_matchpoint, $target_matchpoint ) = @_;
|
|
|
|
my $matchcheck = {};
|
|
$matchcheck->{'source_matchpoint'}->{'index'} = '';
|
|
$matchcheck->{'source_matchpoint'}->{'score'} = 0;
|
|
$matchcheck->{'source_matchpoint'}->{'components'} = [];
|
|
$matchcheck->{'target_matchpoint'}->{'index'} = '';
|
|
$matchcheck->{'target_matchpoint'}->{'score'} = 0;
|
|
$matchcheck->{'target_matchpoint'}->{'components'} = [];
|
|
foreach my $input_component ( @{$source_matchpoint} ) {
|
|
push @{ $matchcheck->{'source_matchpoint'}->{'components'} }, _parse_match_component($input_component);
|
|
}
|
|
foreach my $input_component ( @{$target_matchpoint} ) {
|
|
push @{ $matchcheck->{'target_matchpoint'}->{'components'} }, _parse_match_component($input_component);
|
|
}
|
|
push @{ $self->{'required_checks'} }, $matchcheck;
|
|
}
|
|
|
|
=head2 add_simple_required_check
|
|
|
|
$matcher->add_simple_required_check($source_tag, $source_subfields,
|
|
$source_offset, $source_length, $source_normalizer,
|
|
$target_tag, $target_subfields, $target_offset,
|
|
$target_length, $target_normalizer);
|
|
|
|
Adds a required check, which requires that the normalized keys made from the source and targets
|
|
must match for a match to be considered valid.
|
|
|
|
=cut
|
|
|
|
sub add_simple_required_check {
|
|
my $self = shift;
|
|
my (
|
|
$source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
|
|
$target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer
|
|
) = @_;
|
|
|
|
$self->add_required_check(
|
|
[
|
|
{
|
|
tag => $source_tag, subfields => $source_subfields, offset => $source_offset,
|
|
'length' => $source_length,
|
|
norms => [$source_normalizer]
|
|
}
|
|
],
|
|
[
|
|
{
|
|
tag => $target_tag, subfields => $target_subfields, offset => $target_offset,
|
|
'length' => $target_length,
|
|
norms => [$target_normalizer]
|
|
}
|
|
]
|
|
);
|
|
}
|
|
|
|
=head2 get_matches
|
|
|
|
my @matches = $matcher->get_matches($marc_record, $max_matches);
|
|
foreach $match (@matches) {
|
|
# matches already sorted in order of
|
|
# decreasing score
|
|
print "record ID: $match->{'record_id'};
|
|
print "score: $match->{'score'};
|
|
}
|
|
|
|
Identifies all of the records matching the given MARC record. For a record already
|
|
in the database to be considered a match, it must meet the following criteria:
|
|
|
|
=over 2
|
|
|
|
=item 1. Total score from its matching field must exceed the supplied threshold.
|
|
|
|
=item 2. It must pass all required checks.
|
|
|
|
=back
|
|
|
|
Only the top $max_matches matches are returned. The returned array is sorted
|
|
in order of decreasing score, i.e., the best match is first.
|
|
|
|
=cut
|
|
|
|
sub get_matches {
|
|
my $self = shift;
|
|
my ( $source_record, $max_matches ) = @_;
|
|
|
|
my $matches = {};
|
|
|
|
foreach my $matchpoint ( @{ $self->{'matchpoints'} } ) {
|
|
my @source_keys = _get_match_keys( $source_record, $matchpoint );
|
|
|
|
next if scalar(@source_keys) == 0;
|
|
|
|
@source_keys = C4::Koha::GetVariationsOfISBNs(@source_keys)
|
|
if ( $matchpoint->{index} =~ /^isbn$/i
|
|
&& C4::Context->preference('AggressiveMatchOnISBN') );
|
|
|
|
@source_keys = C4::Koha::GetVariationsOfISSNs(@source_keys)
|
|
if ( $matchpoint->{index} =~ /^issn$/i
|
|
&& C4::Context->preference('AggressiveMatchOnISSN') );
|
|
|
|
# build query
|
|
my $query;
|
|
my $error;
|
|
my $searchresults;
|
|
my $total_hits;
|
|
if ( $self->{'record_type'} eq 'biblio' ) {
|
|
|
|
my $phr =
|
|
( C4::Context->preference('AggressiveMatchOnISBN') || C4::Context->preference('AggressiveMatchOnISSN') )
|
|
? ',phr'
|
|
: q{};
|
|
$query = join(
|
|
" OR ",
|
|
map { "$matchpoint->{'index'}$phr=\"$_\"" } @source_keys
|
|
);
|
|
|
|
#NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it.
|
|
|
|
# Use state variables to avoid recreating the objects every time.
|
|
# With Elasticsearch this also avoids creating a massive amount of
|
|
# ES connectors that would eventually run out of file descriptors.
|
|
state $query_builder =
|
|
Koha::SearchEngine::QueryBuilder->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
|
|
( undef, $query ) = $query_builder->build_query_compat(
|
|
undef, [$query], undef, undef, undef, undef, undef,
|
|
{ skip_facets => 1 }
|
|
);
|
|
state $searcher = Koha::SearchEngine::Search->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
|
|
( $error, $searchresults, $total_hits ) =
|
|
$searcher->simple_search_compat( $query, 0, $max_matches, undef, skip_normalize => 1 );
|
|
|
|
if ( defined $error ) {
|
|
warn "search failed ($query) $error";
|
|
} else {
|
|
foreach my $matched ( @{$searchresults} ) {
|
|
my $target_record = C4::Search::new_record_from_zebra( 'biblioserver', $matched );
|
|
my ( $biblionumber_tag, $biblionumber_subfield ) =
|
|
C4::Biblio::GetMarcFromKohaField("biblio.biblionumber");
|
|
my $id =
|
|
( $biblionumber_tag > 10 )
|
|
? $target_record->field($biblionumber_tag)->subfield($biblionumber_subfield)
|
|
: $target_record->field($biblionumber_tag)->data();
|
|
$matches->{$id}->{score} += $matchpoint->{score};
|
|
$matches->{$id}->{composite_scores}{ $matchpoint->{'index'} } += $matchpoint->{'score'};
|
|
$matches->{$id}->{record} = $target_record;
|
|
}
|
|
}
|
|
|
|
} elsif ( $self->{'record_type'} eq 'authority' ) {
|
|
my @marclist;
|
|
my @and_or;
|
|
my @excluding = [];
|
|
my @operator;
|
|
my @value;
|
|
foreach my $key (@source_keys) {
|
|
push @marclist, $matchpoint->{'index'};
|
|
push @and_or, 'or';
|
|
push @operator, 'exact';
|
|
push @value, $key;
|
|
}
|
|
|
|
# Use state variables to avoid recreating the objects every time.
|
|
# With Elasticsearch this also avoids creating a massive amount of
|
|
# ES connectors that would eventually run out of file descriptors.
|
|
state $builder =
|
|
Koha::SearchEngine::QueryBuilder->new( { index => $Koha::SearchEngine::AUTHORITIES_INDEX } );
|
|
state $searcher = Koha::SearchEngine::Search->new( { index => $Koha::SearchEngine::AUTHORITIES_INDEX } );
|
|
my $search_query = $builder->build_authorities_query_compat(
|
|
\@marclist, \@and_or, \@excluding, \@operator,
|
|
\@value, undef, 'AuthidAsc'
|
|
);
|
|
my ( $authresults, $total ) = $searcher->search_auth_compat( $search_query, 0, 20 );
|
|
|
|
foreach my $result (@$authresults) {
|
|
my $id = $result->{authid};
|
|
my $target_auth = Koha::Authorities->find($id);
|
|
next unless $target_auth;
|
|
my $target_record = eval { $target_auth->record };
|
|
my $invalid_record = $@ || !$target_record;
|
|
if ($invalid_record) {
|
|
warn
|
|
"Error parsing matched authority record $id, attempting to clean record, record will be skipped if it fails";
|
|
$target_record = eval { $target_auth->record_strip_nonxml };
|
|
}
|
|
next unless $target_record;
|
|
$matches->{$id}->{score} += $matchpoint->{'score'};
|
|
$matches->{$id}->{composite_scores}{ $matchpoint->{'index'} } += $matchpoint->{'score'};
|
|
$matches->{$id}->{record} = $target_record;
|
|
}
|
|
}
|
|
}
|
|
|
|
# get rid of any that don't meet the threshold
|
|
$matches =
|
|
{ map { ( $matches->{$_}->{score} >= $self->{'threshold'} ) ? ( $_ => $matches->{$_} ) : () } keys %$matches };
|
|
|
|
my @results = ();
|
|
if ( $self->{'record_type'} eq 'biblio' ) {
|
|
require C4::Biblio;
|
|
|
|
# get rid of any that don't meet the required checks
|
|
$matches = {
|
|
map {
|
|
_passes_required_checks( $source_record, $matches->{$_}->{'record'}, $self->{'required_checks'} )
|
|
? ( $_ => $matches->{$_} )
|
|
: ()
|
|
} keys %$matches
|
|
};
|
|
|
|
foreach my $id ( keys %$matches ) {
|
|
push @results, {
|
|
record_id => $id,
|
|
score => $matches->{$id}->{score},
|
|
composite_scores => $matches->{$id}->{composite_scores},
|
|
};
|
|
}
|
|
} elsif ( $self->{'record_type'} eq 'authority' ) {
|
|
require C4::AuthoritiesMarc;
|
|
|
|
# get rid of any that don't meet the required checks
|
|
$matches = {
|
|
map {
|
|
_passes_required_checks( $source_record, $matches->{$_}->{'record'}, $self->{'required_checks'} )
|
|
? ( $_ => $matches->{$_} )
|
|
: ()
|
|
} keys %$matches
|
|
};
|
|
|
|
foreach my $id ( keys %$matches ) {
|
|
push @results, {
|
|
record_id => $id,
|
|
score => $matches->{$id}->{score},
|
|
composite_scores => $matches->{$id}->{composite_scores},
|
|
};
|
|
}
|
|
}
|
|
@results = sort { $b->{'score'} cmp $a->{'score'} or $b->{'record_id'} cmp $a->{'record_id'} } @results;
|
|
if ( scalar(@results) > $max_matches ) {
|
|
@results = @results[ 0 .. $max_matches - 1 ];
|
|
}
|
|
return @results;
|
|
}
|
|
|
|
=head2 dump
|
|
|
|
$description = $matcher->dump();
|
|
|
|
Returns a reference to a structure containing all of the information
|
|
in the matcher object. This is mainly a convenience method to
|
|
aid setting up a HTML editing form.
|
|
|
|
=cut
|
|
|
|
sub dump {
|
|
my $self = shift;
|
|
|
|
my $result = {};
|
|
|
|
$result->{'matcher_id'} = $self->{'id'};
|
|
$result->{'code'} = $self->{'code'};
|
|
$result->{'description'} = $self->{'description'};
|
|
$result->{'record_type'} = $self->{'record_type'};
|
|
|
|
$result->{'matchpoints'} = [];
|
|
foreach my $matchpoint ( @{ $self->{'matchpoints'} } ) {
|
|
push @{ $result->{'matchpoints'} }, $matchpoint;
|
|
}
|
|
$result->{'matchchecks'} = [];
|
|
foreach my $matchcheck ( @{ $self->{'required_checks'} } ) {
|
|
push @{ $result->{'matchchecks'} }, $matchcheck;
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
=head3 _passes_required_checks
|
|
|
|
my $ok = _passes_required_checks($source_record, $target_record, \@matchchecks);
|
|
|
|
Returns true if the candidate C<$target_record> passes all required field
|
|
equality checks against C<$source_record>.
|
|
|
|
=cut
|
|
|
|
sub _passes_required_checks {
|
|
my ( $source_record, $target_record, $matchchecks ) = @_;
|
|
|
|
# no checks supplied == automatic pass
|
|
return 1 if $#{$matchchecks} == -1;
|
|
|
|
foreach my $matchcheck ( @{$matchchecks} ) {
|
|
my $source_key = join "", _get_match_keys( $source_record, $matchcheck->{'source_matchpoint'} );
|
|
my $target_key = join "", _get_match_keys( $target_record, $matchcheck->{'target_matchpoint'} );
|
|
return 0 unless $source_key eq $target_key;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
=head3 _get_match_keys
|
|
|
|
my @keys = _get_match_keys($source_record, $matchpoint);
|
|
|
|
Extracts and normalises the set of match key strings from C<$source_record>
|
|
for the given C<$matchpoint> definition.
|
|
|
|
=cut
|
|
|
|
sub _get_match_keys {
|
|
|
|
my $source_record = shift;
|
|
my $matchpoint = shift;
|
|
my $check_only_first_repeat = @_ ? shift : 0;
|
|
|
|
# If there is more than one component to the matchpoint (e.g.,
|
|
# matchpoint includes both 003 and 001), any repeats
|
|
# of the first component's tag are identified; repeats
|
|
# of the subsequent components' tags are appended to
|
|
# each parallel key derived from the first component,
|
|
# up to the number of repeats of the first component's tag.
|
|
#
|
|
# For example, if the record has one 003 and two 001s, only
|
|
# one key is retrieved because there is only one 003. The key
|
|
# will consist of the contents of the first 003 and first 001.
|
|
#
|
|
# If there are two 003s and two 001s, there will be two keys:
|
|
# first 003 + first 001
|
|
# second 003 + second 001
|
|
|
|
my @keys = ();
|
|
for ( my $i = 0 ; $i <= $#{ $matchpoint->{'components'} } ; $i++ ) {
|
|
my $component = $matchpoint->{'components'}->[$i];
|
|
my $j = -1;
|
|
|
|
my @fields = ();
|
|
my $tag = $component->{'tag'};
|
|
if ( $tag && $tag eq 'LDR' ) {
|
|
$fields[0] = $source_record->leader();
|
|
} else {
|
|
@fields = $source_record->field($tag);
|
|
}
|
|
|
|
FIELD: foreach my $field (@fields) {
|
|
$j++;
|
|
last FIELD if $j > 0 and $check_only_first_repeat;
|
|
last FIELD if $i > 0 and $j > $#keys;
|
|
|
|
my $string;
|
|
if ( !ref $field ) {
|
|
$string = "$field";
|
|
} elsif ( $field->is_control_field() ) {
|
|
$string = $field->data();
|
|
} elsif ( defined $component->{subfields} && keys %{ $component->{subfields} } ) {
|
|
$string = $field->as_string(
|
|
join( '', keys %{ $component->{subfields} } ), ' ' # ' ' as separator
|
|
);
|
|
} else {
|
|
$string = $field->as_string();
|
|
}
|
|
|
|
if ( $component->{'length'} > 0 ) {
|
|
$string = substr( $string, $component->{'offset'}, $component->{'length'} );
|
|
} elsif ( $component->{'offset'} ) {
|
|
$string = substr( $string, $component->{'offset'} );
|
|
}
|
|
|
|
my $norms = $component->{'norms'};
|
|
my $key = $string;
|
|
|
|
foreach my $norm ( @{$norms} ) {
|
|
if ( grep { $norm eq $_ } valid_normalization_routines() ) {
|
|
if ( $norm eq 'remove_spaces' ) {
|
|
$key = remove_spaces($key);
|
|
} elsif ( $norm eq 'upper_case' ) {
|
|
$key = upper_case($key);
|
|
} elsif ( $norm eq 'lower_case' ) {
|
|
$key = lower_case($key);
|
|
} elsif ( $norm eq 'legacy_default' ) {
|
|
$key = legacy_default($key);
|
|
} elsif ( $norm eq 'ISBN' ) {
|
|
$key = ISBN($key);
|
|
}
|
|
} else {
|
|
warn "Invalid normalization routine required ($norm)"
|
|
unless $norm eq 'none';
|
|
}
|
|
}
|
|
|
|
if ( $i == 0 ) {
|
|
push @keys, $key if $key;
|
|
} else {
|
|
$keys[$j] .= " $key" if $key;
|
|
}
|
|
}
|
|
}
|
|
return @keys;
|
|
}
|
|
|
|
=head3 _parse_match_component
|
|
|
|
my $component = _parse_match_component($input_component);
|
|
|
|
Parses a matchpoint component definition hashref and returns a normalised
|
|
component structure including tag, subfields, offset, length, and norms.
|
|
|
|
=cut
|
|
|
|
sub _parse_match_component {
|
|
my $input_component = shift;
|
|
|
|
my $component = {};
|
|
$component->{'tag'} = $input_component->{'tag'};
|
|
$component->{'subfields'} = { map { $_ => 1 } split( //, $input_component->{'subfields'} ) };
|
|
$component->{'offset'} = exists( $input_component->{'offset'} ) ? $input_component->{'offset'} : -1;
|
|
$component->{'length'} = $input_component->{'length'} ? $input_component->{'length'} : 0;
|
|
$component->{'norms'} = $input_component->{'norms'} ? $input_component->{'norms'} : [];
|
|
|
|
return $component;
|
|
}
|
|
|
|
=head2 valid_normalization_routines
|
|
|
|
Missing POD for valid_normalization_routines.
|
|
|
|
=cut
|
|
|
|
sub valid_normalization_routines {
|
|
|
|
return (
|
|
'remove_spaces',
|
|
'upper_case',
|
|
'lower_case',
|
|
'legacy_default',
|
|
'ISBN'
|
|
);
|
|
}
|
|
|
|
1;
|
|
__END__
|
|
|
|
=head1 AUTHOR
|
|
|
|
Koha Development Team <https://koha-community.org/>
|
|
|
|
Galen Charlton <galen.charlton@liblime.com>
|
|
|
|
=cut
|