gitea
/
Koha-clms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bug 22828: Elasticsearch - display errors encountered during indexing on the command line 

To test:
 1 - Use the Koha sample data, or insert a blank 245$b into a record (easiest way is using advanced cataloging editor
 2 - Reindex elasticsearch
 3 - Check the ES count on the about page
 4 - Check the count in the DB (SELECT count(*) FROM biblio)
 5 - They don't match!
 6 - perl misc/search_tools/rebuild_elastic_search.pl -v -v
 7 - No errors indicated
 8 - Apply patch
 9 - perl misc/search_tools/rebuild_elastic_search.pl -v
10 - You should be notified of an error
11 - perl misc/search_tools/rebuild_elastic_search.pl -v -v
12 - You should be notified of the specific biblio with an error and a (somewhat) readable reason
13 - perl misc/search_tools/rebuild_elastic_search.pl
14 - No output

Signed-off-by: Ere Maijala <ere.maijala@helsinki.fi>
Signed-off-by: Séverine QUEUNE <severine.queune@bulac.fr>
Signed-off-by: Bouzid Fergani <bouzid.fergani@inlibro.com>
Signed-off-by: Jonathan Druart <jonathan.druart@bugs.koha-community.org>
Signed-off-by: Martin Renvoize <martin.renvoize@ptfs-europe.com>
This commit is contained in:
Nick Clemens 2019-05-02 11:27:27 +00:00 committed by Martin Renvoize
parent 5fdd436558
commit 4249b20a39
No known key found for this signature in database
GPG Key ID: 422B469130441A0F
2 changed files with 31 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Koha/SearchEngine/Elasticsearch/Indexer.pm
View File

@ -117,15 +117,15 @@ sub update_index {
};
push @body, $document;
}
my $response;
if (@body) {
my $response = $elasticsearch->bulk(
$response = $elasticsearch->bulk(
index => $conf->{index_name},
type => 'data', # is just hard coded in Indexer.pm?
body => \@body
);
}
# TODO: handle response
return 1;
return $response;
}
=head2 set_index_status_ok

30
misc/search_tools/rebuild_elasticsearch.pl
View File

@ -206,6 +206,8 @@ if ($slice_index == 0) {
}
}
=head1 INTERNAL METHODS
=head2 _verify_index_state
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1);
@ -241,6 +243,7 @@ sub _verify_index_state {
_do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
Does the actual reindexing. $callback is a function that always returns the next record.
For each index we iterate through the records, committing at specified count
=cut
@ -266,7 +269,8 @@ sub _do_reindex {
push @commit_buffer, $record;
if ( !( --$commit_count ) ) {
_log( 1, "Committing $commit records...\n" );
$indexer->update_index( \@id_buffer, \@commit_buffer );
my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
_handle_response($response);
$commit_count = $commit;
@id_buffer = ();
@commit_buffer = ();
@ -276,7 +280,8 @@ sub _do_reindex {
# There are probably uncommitted records
_log( 1, "Committing final records...\n" );
$indexer->update_index( \@id_buffer, \@commit_buffer );
my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
_handle_response($response);
_log( 1, "Total $count records indexed\n" );
}
@ -294,6 +299,27 @@ sub _sanity_check {
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
}
=head2 _handle_response
Parse the return from update_index and display errors depending on verbosity of the script
=cut
sub _handle_response {
my ($response) = @_;
if( $response->{errors} eq 'true' ){
_log( 1, "There were errors during indexing\n" );
if ( $verbose > 1 ){
foreach my $item (@{$response->{items}}){
next unless defined $item->{index}->{error};
print "Record #" . $item->{index}->{_id} . " " .
$item->{index}->{error}->{reason} . " (" . $item->{index}->{error}->{type} . ") : " .
$item->{index}->{error}->{caused_by}->{type} . " (" . $item->{index}->{error}->{caused_by}->{reason} . ")\n";
}
}
}
}
=head2 _log
_log($level, "Message\n");