Bug 40577: Bulk biblio ES index update after auth change
When merging (or updating) authorities, the biblios using the auths might also be updated. If this happens, Koha used to create one background job for each biblio. If an auth is used in a lot of biblios, this creates a lot of background jobs, each to index one single biblio. This patch fixes that by collecting the affected biblios and putting all of them into one reindex background job. Test plan: * Apply the patch * run the unit test t/db_dependent/Authority/Merge.t * the test should now have 16 tests in total, with a new test #15 output like this: 1..3 ok 1 - 3 bibliographic record updated while merging two authorities ok 2 - we have only one more background job ok 3 - we have the correct biblio ids in the job to be indexed ok 15 - merge with ModBiblio only creates one indexer background job for multiple biblios ok 16 - no warnings Manual test plan (rather cumbersome): * Start KTD with ElasticSearch * Create an PERSON authority (eg "100a Test") * Create three biblios which use that authority in eg 100 * Access the DB to get the count of background jobs * `ktd --dbshell` * `select count(*) from background_jobs;` * the count should be the same as the number of biblios you created (eg 3). But whatever it is, remeber it * Edit the authority (change 100 to "Test new") * check if the change is reflected in the linked biblios (100a = "Test new") * Check the Background jobs in the DB again: * `select count(*) from background_jobs;` * should be three more than previously (eg 6) * You can use this SQL to verify that the last three jobs each contain one biblionumber (which is what we want to fix): * `select json_extract(data,'$.record_ids'),enqueued_on from background_jobs order by enqueued_on desc limit 3;` +-----------------------------------+---------------------+ | json_extract(data,'$.record_ids') | enqueued_on | +-----------------------------------+---------------------+ | [2299] | 2026-03-25 13:19:34 | | [2300] | 2026-03-25 13:19:34 | | [2301] | 2026-03-25 13:19:34 | +-----------------------------------+---------------------+ * Now apply the patch and run restart_all * Edit the authority again, setting 100a to "Test new patched". * The biblios should change accordingly * Check the Background jobs in the DB again: * `select count(*) from background_jobs;` * This should now be only 1 more then the last time (so eg 7) * Check the details of the background jobs: * `select json_extract(data,'$.record_ids'),enqueued_on from background_jobs order by enqueued_on desc limit 3;` * The most recent entry should now contain three items in one row, and the following two should be the same as in the previous query (i.e. the old jobs from the edit before the patch) +-----------------------------------+---------------------+ | json_extract(data,'$.record_ids') | enqueued_on | +-----------------------------------+---------------------+ | [2299, 2300, 2301] | 2026-03-25 13:29:17 | | [2300] | 2026-03-25 13:19:34 | | [2299] | 2026-03-25 13:19:34 | +-----------------------------------+---------------------+ Signed-off-by: Paul Derscheid <paul.derscheid@lmscloud.de> Signed-off-by: Baptiste Wojtkowski <baptiste.wojtkowski@biblibre.com> Signed-off-by: Lucas Gass <lucas@bywatersolutions.com>
This commit is contained in:
Thomas Klausner
Lucas Gass
parent
95267338d5
commit
1bb30a26cd
|
|
@ -1685,6 +1685,7 @@ sub merge {
|
|||
C4::Context->preference('IncludeSeeFromInSearches')
|
||||
|| C4::Context->preference('IncludeSeeAlsoFromInSearches')
|
||||
|| 0;
|
||||
my @biblios_to_index;
|
||||
while ( my $biblio = $biblios->next ) {
|
||||
my $marcrecord = $biblio->metadata->record;
|
||||
my $update = 0;
|
||||
|
|
@ -1784,13 +1785,20 @@ sub merge {
|
|||
&& $reindex_if_needed
|
||||
&& $syspref_include_see_from )
|
||||
{
|
||||
my $indexer = Koha::SearchEngine::Indexer->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
|
||||
$indexer->index_records( $biblio->biblionumber, "specialUpdate", "biblioserver" );
|
||||
push( @biblios_to_index, $biblio->biblionumber );
|
||||
}
|
||||
next if !$update;
|
||||
ModBiblio( $marcrecord, $biblio->biblionumber, $biblio->frameworkcode, { disable_autolink => 1 } );
|
||||
ModBiblio(
|
||||
$marcrecord, $biblio->biblionumber, $biblio->frameworkcode,
|
||||
{ disable_autolink => 1, skip_record_index => 1 }
|
||||
);
|
||||
push( @biblios_to_index, $biblio->biblionumber );
|
||||
$counteditedbiblio++;
|
||||
}
|
||||
if (@biblios_to_index) {
|
||||
my $indexer = Koha::SearchEngine::Indexer->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
|
||||
$indexer->index_records( \@biblios_to_index, "specialUpdate", "biblioserver" );
|
||||
}
|
||||
return $counteditedbiblio;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,12 +4,13 @@
|
|||
|
||||
use Modern::Perl;
|
||||
use Test::NoWarnings;
|
||||
use Test::More tests => 15;
|
||||
use Test::More tests => 16;
|
||||
|
||||
use Getopt::Long;
|
||||
use MARC::Record;
|
||||
use Test::MockModule;
|
||||
use Test::Warn;
|
||||
use Test::Deep qw( cmp_deeply bag );
|
||||
|
||||
use t::lib::Mocks;
|
||||
use t::lib::TestBuilder;
|
||||
|
|
@ -19,6 +20,7 @@ use Koha::Authorities;
|
|||
use Koha::Authority::ControlledIndicators;
|
||||
use Koha::Authority::MergeRequests;
|
||||
use Koha::Biblios;
|
||||
use Koha::BackgroundJobs;
|
||||
use Koha::Database;
|
||||
|
||||
BEGIN {
|
||||
|
|
@ -867,11 +869,73 @@ subtest 'ModBiblio calls from merge' => sub {
|
|||
}
|
||||
);
|
||||
}
|
||||
["ModBiblio called"], "real modification of bibliographic record, ModBiblio called";
|
||||
[ "ModBiblio called", "index_records called" ], "real modification of bibliographic record, ModBiblio called";
|
||||
is( $num_biblio_edited, 1, 'only one bibliographic record updated while merging two authorities' );
|
||||
|
||||
$schema->storage->txn_rollback;
|
||||
|
||||
# need to explicitly unmock ModBiblio here, because we have other mocks on C4::AuthoritiesMarc still in scope which cause problems on the next test;
|
||||
$biblio_module->unmock('ModBiblio');
|
||||
|
||||
};
|
||||
|
||||
subtest 'merge with ModBiblio only creates one indexer background job for multiple biblios' => sub {
|
||||
|
||||
plan tests => 3;
|
||||
|
||||
$schema->storage->txn_begin;
|
||||
|
||||
my $starting_job_count = Koha::BackgroundJobs->search->count;
|
||||
|
||||
t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );
|
||||
t::lib::Mocks::mock_preference( 'AuthorityControlledIndicators', '' );
|
||||
t::lib::Mocks::mock_preference( 'SearchEngine', 'Elasticsearch' );
|
||||
|
||||
# NOT mocking ModBiblio here, because we need to actually call ModBiblio to check that it is no longer adding a distinct background job
|
||||
|
||||
# create a new auth record
|
||||
my $auth_record = MARC::Record->new();
|
||||
$auth_record->insert_fields_ordered( MARC::Field->new( '109', '1', ' ', a => 'Brown,', 'c' => 'Father' ) );
|
||||
my $authid = AddAuthority( $auth_record, undef, $authtype1, { skip_record_index => 1 } );
|
||||
|
||||
# create 3 biblios using that auth
|
||||
my @biblio_ids;
|
||||
for my $c ( 1 .. 3 ) {
|
||||
my $biblio_record = MARC::Record->new();
|
||||
$biblio_record->insert_fields_ordered(
|
||||
MARC::Field->new( '109', '1', ' ', a => 'Brown,', 'c' => 'Father', 9 => $authid ),
|
||||
MARC::Field->new( '245', '0', '0', a => 'Test Biblio ' . $c )
|
||||
);
|
||||
my ($biblionumber) = AddBiblio( $biblio_record, '', { skip_record_index => 1 } );
|
||||
push( @biblio_ids, $biblionumber );
|
||||
}
|
||||
|
||||
# update the auth
|
||||
my $auth_record_modified = $auth_record->clone;
|
||||
$auth_record_modified->field('109')->update( x => 'Father (Fictitious character)' );
|
||||
|
||||
# merge the auth, which will update the 3 biblios
|
||||
my $num_biblio_edited = merge(
|
||||
{
|
||||
mergefrom => $authid, MARCfrom => $auth_record, mergeto => $authid, MARCto => $auth_record_modified,
|
||||
biblionumbers => \@biblio_ids,
|
||||
}
|
||||
);
|
||||
is( $num_biblio_edited, 3, '3 bibliographic record updated while merging two authorities' );
|
||||
|
||||
# the final test:
|
||||
# there should be only one background job with three biblio ids
|
||||
my $job_count = Koha::BackgroundJobs->search->count;
|
||||
is( $job_count, $starting_job_count + 1, 'we have only one more background job' );
|
||||
|
||||
my $job = Koha::BackgroundJobs->search( {}, { order_by => 'enqueued_on' } )->last;
|
||||
cmp_deeply(
|
||||
$job->decoded_data->{record_ids}, bag(@biblio_ids),
|
||||
'we have the correct biblio ids in the job to be indexed'
|
||||
);
|
||||
|
||||
$schema->storage->txn_rollback;
|
||||
|
||||
};
|
||||
|
||||
sub set_mocks {
|
||||
|
|
|
|||
Loading…
Reference in New Issue