From 4214c68f175fbb96c2c7c8959392599fbf16bf4b Mon Sep 17 00:00:00 2001 From: Tomas Cohen Arazi Date: Mon, 23 Dec 2024 16:04:04 -0300 Subject: [PATCH 1/6] Bug 38779: Add built record sources assets on install Signed-off-by: Tomas Cohen Arazi Signed-off-by: David Nind Signed-off-by: Katrin Fischer (cherry picked from commit 1de253eb508528bdfa90502b3073a73cfd36c478) Signed-off-by: Paul Derscheid Signed-off-by: Alex Buckley --- Makefile.PL | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.PL b/Makefile.PL index 62062d96b2..33d0b0c1b3 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -675,6 +675,7 @@ $file_map->{'koha-tmpl/intranet-tmpl/prog/css/mainpage.css'} = 'blib/INTRANET_TM $file_map->{'koha-tmpl/intranet-tmpl/prog/css/staff-global.css'} = 'blib/INTRANET_TMPL_DIR/prog/css/staff-global.css'; $file_map->{'koha-tmpl/intranet-tmpl/prog/js/vue/dist/erm.js'} = 'blib/INTRANET_TMPL_DIR/prog/js/vue/dist/erm.js'; $file_map->{'koha-tmpl/intranet-tmpl/prog/js/vue/dist/preservation.js'} = 'blib/INTRANET_TMPL_DIR/prog/js/vue/dist/preservation.js'; +$file_map->{'koha-tmpl/intranet-tmpl/prog/js/vue/dist/admin/record_sources.js'} = 'blib/INTRANET_TMPL_DIR/prog/js/vue/dist/admin/record_sources.js'; #OPAC_TMPL_DIR $file_map->{'koha-tmpl/opac-tmpl/bootstrap/css/opac.css'} = 'blib/OPAC_TMPL_DIR/bootstrap/css/opac.css'; $file_map->{'koha-tmpl/opac-tmpl/bootstrap/css/opac-rtl.css'} = 'blib/OPAC_TMPL_DIR/bootstrap/css/opac-rtl.css'; From 041e4368f9ebe12a14480babeec62ec543f072d5 Mon Sep 17 00:00:00 2001 From: Janusz Kaczmarek Date: Thu, 16 Jan 2025 21:04:28 +0000 Subject: [PATCH 2/6] Bug 38913: (bug 38416 follow-up) Elasticsearch indexing explodes with oversized records After Bug 38416 Elasticsearch indexing explodes with oversized records, especially with UTF encoded data. In Koha::SearchEngine::Elasticsearch::marc_records_to_documents a following snippet has been introduced: my $usmarc_record = $record->as_usmarc(); my $decoded_usmarc_record = MARC::Record->new_from_usmarc($usmarc_record); But if $record is oversized (> 99999 bytes), it is OK for MARC::Record object, but not for $record->as_usmarc. The produced ISO 2709 string is not correct and hence cannot be properly converted back to MARC::Record object by new_from_usmarc. The result in this case can be like: UTF-8 "\x85" does not map to Unicode at /usr/share/perl5/MARC/File/Encode.pm line 35. Since it is done without any eval / try, the whole reindex procedure (for instance rebuild_elasticsearch.pl) is being randomly interrupted with no explanation. Test plan: ========== Hard to reproduce. But the explanation together with discussion in Bug 38416 (from 2024-12-15) explains and justifies the need of this added eval. 1. Have a standard KTD installation with Elasticsearch. 2. Use the provided test record - add it to Koha with ./misc/migration_tools/bulkmarcimport.pl -b -file test.xml -m=MARCXML (have patience). During load process you should see a message like: UTF-8 "\xC4" does not map to Unicode at /usr/share/perl5/MARC/File/Encode.pm line 35. 3. The record should get biblionumber 439. Check in librarian interface with http://:8081/cgi-bin/koha/catalogue/detail.pl?biblionumber=439 that the record has been imported. However, you should not be able to make a search for this record. 4. Try to reindex with: ./misc/search_tools/rebuild_elasticsearch.pl -b -bn 439 You should get a message like: UTF-8 "\xC4" does not map to Unicode at /usr/share/perl5/MARC/File/Encode.pm line 35. Again, no search results. 5. Apply the patch ; restart_all. 6. Repeat reindex with: ./misc/search_tools/rebuild_elasticsearch.pl -b -bn 439 There should be no warning now and you should be able to find the record. Signed-off-by: Magnus Enger Followed the test plan. Works as advertised. Signed-off-by: David Cook Signed-off-by: Katrin Fischer (cherry picked from commit c7127fabf4ebe736a3563df4bd3e500691d0b632) Signed-off-by: Paul Derscheid Signed-off-by: Alex Buckley --- Koha/SearchEngine/Elasticsearch.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 79522b4f27..a2fd4dd0db 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -793,8 +793,9 @@ sub marc_records_to_documents { my $usmarc_record = $record->as_usmarc(); #NOTE: Try to round-trip the record to prove it will work for retrieval after searching - my $decoded_usmarc_record = MARC::Record->new_from_usmarc($usmarc_record); - if ( $decoded_usmarc_record->warnings() ) { + my $decoded_usmarc_record; + eval { $decoded_usmarc_record = MARC::Record->new_from_usmarc($usmarc_record); }; + if ( $@ || $decoded_usmarc_record->warnings() ) { #NOTE: We override the warnings since they're many and misleading @warnings = ( From 2e0dafcde5436966866ca028a0a4d614285e5488 Mon Sep 17 00:00:00 2001 From: David Cook Date: Mon, 20 Jan 2025 02:55:12 +0000 Subject: [PATCH 3/6] Bug 38913: (QA follow-up) test UTF-8 exceptions in large MARC records MARC records with over 99999 bytes are invalid by spec, and when you use UTF-8 encoded characters in your MARC records, there is the potential to generate fatal errors in MARC::File::USMARC when it runs "marc_to_utf8" from "MARC::File::Encode" during its "decode" operation. That is, if you MARC::File::USMARC->encode a MARC record with over 99999 bytes (including a number of UTF-8 bytes), there is the potential when you run MARC::File:USMARC->decode on that same data that you'll generate a fatal exception. The main patch in bug 38913 wraps the function doing the decode, so that a bad record doesn't crash processing. Without the patch, this unit test will fail. With the patch, this unit test will pass. Signed-off-by: Katrin Fischer (cherry picked from commit 9d41abc1e77c15ee88f66ba7aa0b419524760293) Signed-off-by: Paul Derscheid Signed-off-by: Alex Buckley --- t/db_dependent/Koha/SearchEngine/Elasticsearch.t | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t b/t/db_dependent/Koha/SearchEngine/Elasticsearch.t index 24159799f8..4a8cf9c41a 100755 --- a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t +++ b/t/db_dependent/Koha/SearchEngine/Elasticsearch.t @@ -16,6 +16,7 @@ # along with Koha; if not, see . use Modern::Perl; +use Encode; use Test::More tests => 8; use Test::Exception; @@ -680,7 +681,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' MARC::Field->new('999', '', '', c => '1234567'), ); - my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test'); + my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => Encode::decode('UTF-8','To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.')); my $items_count = 1638; while(--$items_count) { $large_marc_record->append_fields($item_field); From dfd9bc68cd0479a7382c9035e992947fa29bdeae Mon Sep 17 00:00:00 2001 From: Pedro Amorim Date: Thu, 19 Sep 2024 12:36:54 +0000 Subject: [PATCH 4/6] Bug 37810: Add test Test plan: 1) Apply only tests patch, run: prove t/db_dependent/Koha/ERM/EUsage/UsageDataProvider.t 2) Verify tests fail 3) Apply fix patch 4) Run tests again, verify they pass Signed-off-by: David Nind Signed-off-by: Nick Clemens Signed-off-by: Martin Renvoize Signed-off-by: Alex Buckley --- .../Koha/ERM/EUsage/UsageDataProvider.t | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/t/db_dependent/Koha/ERM/EUsage/UsageDataProvider.t b/t/db_dependent/Koha/ERM/EUsage/UsageDataProvider.t index 2513b6f247..3f64e0e1d3 100755 --- a/t/db_dependent/Koha/ERM/EUsage/UsageDataProvider.t +++ b/t/db_dependent/Koha/ERM/EUsage/UsageDataProvider.t @@ -17,10 +17,12 @@ use Modern::Perl; -use Test::More tests => 1; +use Test::More tests => 2; use Koha::ERM::EUsage::UsageDataProvider; use Koha::Database; +use Test::MockModule; +use Test::MockObject; use t::lib::TestBuilder; @@ -108,3 +110,37 @@ subtest '_build_url_query' => sub { $schema->storage->txn_rollback; }; + +subtest 'test_connection() tests' => sub { + + plan tests => 1; + + $schema->storage->txn_begin; + + my $ua = Test::MockModule->new('LWP::UserAgent'); + $ua->mock('simple_request', sub { + my $response = Test::MockObject->new(); + + $response->mock('code', sub { + return 200; + }); + $response->mock('is_error', sub { + return 0; + }); + $response->mock('is_redirect', sub { + return 0; + }); + $response->mock('decoded_content', sub { + return '{"Description":"COUNTER Usage Reports for Test platform.","ServiceActive":true,"RegistryURL":"https://www.whatever.com"}'; + }); + $response->{_rc} = 200; + return $response; + }); + + my $usage_data_provider = $builder->build_object( + { class => 'Koha::ERM::EUsage::UsageDataProviders', value => { name => 'TestProvider' } } ); + + is( $usage_data_provider->test_connection, 1); + + $schema->storage->txn_rollback; +}; From f8fb37b3fab20ecf79ae6fbfe03b684fabc9b9b1 Mon Sep 17 00:00:00 2001 From: Pedro Amorim Date: Mon, 2 Sep 2024 14:51:31 +0000 Subject: [PATCH 5/6] Bug 37810: Consider ServiceActive on status response Some SUSHI providers return ServiceActive instead of the documented Service_Active: https://countermetrics.stoplight.io/docs/counter-sushi-api/f0dd30f814944-server-status This ensures the test connection does not fail regardless of what is used Signed-off-by: David Nind Signed-off-by: Nick Clemens Signed-off-by: Martin Renvoize Signed-off-by: Alex Buckley --- Koha/ERM/EUsage/UsageDataProvider.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Koha/ERM/EUsage/UsageDataProvider.pm b/Koha/ERM/EUsage/UsageDataProvider.pm index ab9f314c80..5e04714bb4 100644 --- a/Koha/ERM/EUsage/UsageDataProvider.pm +++ b/Koha/ERM/EUsage/UsageDataProvider.pm @@ -300,10 +300,10 @@ sub test_connection { my $status; if ( ref($result) eq 'ARRAY' ) { for my $r (@$result) { - $status = $r->{Service_Active}; + $status = $r->{Service_Active} // $r->{ServiceActive}; } } else { - $status = $result->{Service_Active}; + $status = $result->{Service_Active} // $result->{ServiceActive}; } if ($status) { From da96f8cfe50e19258e11ea5a8f6b969090de2f9c Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Tue, 1 Oct 2024 14:05:56 +0000 Subject: [PATCH 6/6] Bug 37810: (QA follow-up) Tidy Signed-off-by: Nick Clemens Signed-off-by: Martin Renvoize Signed-off-by: Alex Buckley --- .../Koha/ERM/EUsage/UsageDataProvider.t | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/t/db_dependent/Koha/ERM/EUsage/UsageDataProvider.t b/t/db_dependent/Koha/ERM/EUsage/UsageDataProvider.t index 3f64e0e1d3..941bfe351d 100755 --- a/t/db_dependent/Koha/ERM/EUsage/UsageDataProvider.t +++ b/t/db_dependent/Koha/ERM/EUsage/UsageDataProvider.t @@ -118,29 +118,45 @@ subtest 'test_connection() tests' => sub { $schema->storage->txn_begin; my $ua = Test::MockModule->new('LWP::UserAgent'); - $ua->mock('simple_request', sub { + $ua->mock( + 'simple_request', + sub { my $response = Test::MockObject->new(); - $response->mock('code', sub { - return 200; - }); - $response->mock('is_error', sub { - return 0; - }); - $response->mock('is_redirect', sub { - return 0; - }); - $response->mock('decoded_content', sub { - return '{"Description":"COUNTER Usage Reports for Test platform.","ServiceActive":true,"RegistryURL":"https://www.whatever.com"}'; - }); + $response->mock( + 'code', + sub { + return 200; + } + ); + $response->mock( + 'is_error', + sub { + return 0; + } + ); + $response->mock( + 'is_redirect', + sub { + return 0; + } + ); + $response->mock( + 'decoded_content', + sub { + return + '{"Description":"COUNTER Usage Reports for Test platform.","ServiceActive":true,"RegistryURL":"https://www.whatever.com"}'; + } + ); $response->{_rc} = 200; return $response; - }); + } + ); my $usage_data_provider = $builder->build_object( { class => 'Koha::ERM::EUsage::UsageDataProviders', value => { name => 'TestProvider' } } ); - is( $usage_data_provider->test_connection, 1); + is( $usage_data_provider->test_connection, 1 ); $schema->storage->txn_rollback; };