gitea
/
Koha-clms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bug 35466: Add skip_indexing parameter to bulkmarcimport.pl 

This patch adds a new optoin to skip indexing to bulkmarcimport.
It also fixes a bug where authorities were being indexed multiple times
during import.

To test:
 1 - Apply patch
 2 - Download the sample files on this bug
 3 - Perform asearch engine search that should retrieve all records( 'a' in Zebra, '*' in ES)
 4 - Note the total (435 in KTD)
 5 - perl misc/migration_tools/bulkmarcimport.pl -b -v --file=bug_35466_b.mrc
 6 - Search again, note increaed by 100
 7 - perl misc/migration_tools/bulkmarcimport.pl -b -v --file=bug_35466_b.mrc --skip_indexing
 8 - Search again, no increase
 9 - perl misc/search_tools/rebuild_elasticsearch.pl -v
10 - Search again, increase, records were added but not initially indexed
11 - Browse to authorities and search as for biblios
12 - perl misc/migration_tools/bulkmarcimport.pl -a -v --file=bug_35466_a.mrc
13 - Search again, note increase
14 - perl misc/migration_tools/bulkmarcimport.pl -a -v --file=bug_35466_a.mrc --skip_indexing
15 - Search again, no increase
16 - perl misc/search_tools/rebuild_elasticsearch.pl -v
17 - Search again, increase, records were added but not initially indexed
18 - Sign off! Hi5!

Signed-off-by: Phil Ringnalda <phil@chetcolibrary.org>
Signed-off-by: Thomas Klausner <domm@plix.at>
Signed-off-by: Katrin Fischer <katrin.fischer@bsz-bw.de>
(cherry picked from commit e01fb16455)
Signed-off-by: Lucas Gass <lucas@bywatersolutions.com>
This commit is contained in:
Nick Clemens 2024-08-05 12:48:04 +00:00 committed by Lucas Gass
parent 93d5737493
commit 76c1387263
1 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
misc/migration_tools/bulkmarcimport.pl
View File

@ -75,6 +75,7 @@ my $framework = '';
my $localcust;
my $marc_mod_template = '';
my $marc_mod_template_id = -1;
my $skip_indexing = 0;
$| = 1;
GetOptions(
@ -111,6 +112,7 @@ GetOptions(
'framework=s' => \$framework,
'custom:s' => \$localcust,
'marcmodtemplate:s' => \$marc_mod_template,
'si|skip_indexing' => \$skip_indexing,
);
$biblios ||= !$authorities;
@ -128,13 +130,15 @@ if ($all) {
my $using_elastic_search = ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' );
my $mod_biblio_options = {
disable_autolink => $using_elastic_search,
skip_record_index => $using_elastic_search,
skip_record_index => $using_elastic_search || $skip_indexing,
overlay_context => { source => 'bulkmarcimport' }
};
my $add_biblio_options = {
disable_autolink => $using_elastic_search,
skip_record_index => $using_elastic_search
skip_record_index => $using_elastic_search || $skip_indexing
};
my $mod_authority_options = { skip_record_index => $using_elastic_search || $skip_indexing };
my $add_authority_options = { skip_record_index => $using_elastic_search || $skip_indexing };
my @search_engine_record_ids;
my @search_engine_records;
@ -467,7 +471,12 @@ RECORD: foreach my $record ( @{$marc_records} ) {
if ($matched_record_id) {
if ($update) {
## Authority has an id and is in database: update
eval { ($authid) = ModAuthority( $matched_record_id, $record, $authtypecode ) };
eval {
($authid) = ModAuthority(
$matched_record_id, $record, $authtypecode,
$mod_authority_options,
);
};
if ($@) {
warn "ERROR: Update authority $matched_record_id failed: $@\n";
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
@ -487,7 +496,7 @@ RECORD: foreach my $record ( @{$marc_records} ) {
}
} elsif ($insert) {
## An authid is defined but no authority in database: insert
eval { ($authid) = AddAuthority( $record, undef, $authtypecode ) };
eval { ($authid) = AddAuthority( $record, undef, $authtypecode, $add_authority_options ); };
if ($@) {
warn "ERROR: Insert authority $originalid failed: $@\n";
printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile);
@ -682,7 +691,7 @@ RECORD: foreach my $record ( @{$marc_records} ) {
$schema->txn_commit;
$schema->txn_begin;
if ($indexer) {
$indexer->update_index( \@search_engine_record_ids, \@search_engine_records );
$indexer->update_index( \@search_engine_record_ids, \@search_engine_records ) unless $skip_indexing;
if ( C4::Context->preference('AutoLinkBiblios') ) {
foreach my $record (@search_engine_records) {
BiblioAutoLink( $record, $framework );