diff --git a/Koha/BackgroundJob/PatronImport.pm b/Koha/BackgroundJob/PatronImport.pm index 987edb9fda..5cd6e2fdf8 100644 --- a/Koha/BackgroundJob/PatronImport.pm +++ b/Koha/BackgroundJob/PatronImport.pm @@ -20,6 +20,7 @@ use Try::Tiny qw( catch try ); use Koha::List::Patron qw( AddPatronList AddPatronsToList ); use Koha::Patrons::Import; +use Koha::UploadedFiles; use base 'Koha::BackgroundJob'; @@ -54,8 +55,12 @@ sub process { $self->start; - my $file_content = $args->{file_content}; - open my $fh, '<', \$file_content or die "Cannot create in-memory file handle: $!"; + my $upload = Koha::UploadedFiles->find( $args->{uploaded_file_id} ); + unless ($upload) { + $self->status('failed')->store; + return; + } + my $fh = $upload->file_handle; my $result = try { Koha::Patrons::Import->new->import_patrons( @@ -80,7 +85,6 @@ sub process { }; my $data = $self->decoded_data; - delete $data->{file_content}; # No longer needed; free DB space if ($result) { if ( $args->{createpatronlist} && @{ $result->{imported_borrowers} // [] } ) { @@ -110,9 +114,17 @@ Enqueue the new job. sub enqueue { my ( $self, $args ) = @_; - return unless $args->{file_content}; + return unless $args->{uploaded_file_id}; - my $line_count = scalar grep { /\S/ } split /\n/, $args->{file_content}; + my $upload = Koha::UploadedFiles->find( $args->{uploaded_file_id} ); + return unless $upload; + + my $fh = $upload->file_handle; + my $line_count = 0; + while ( my $line = <$fh> ) { + $line_count++ if $line =~ /\S/; + } + close $fh; $self->SUPER::enqueue( { diff --git a/Koha/Uploader.pm b/Koha/Uploader.pm index 2b96e64937..f4c7b1383a 100644 --- a/Koha/Uploader.pm +++ b/Koha/Uploader.pm @@ -176,6 +176,7 @@ sub allows_add_by { { circulate => 'circulate_remaining_permissions' }, { tools => 'stage_marc_import' }, { tools => 'upload_local_cover_images' }, + { tools => 'import_patrons' }, ]; require C4::Auth; foreach (@$flags) { diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt index 6734b1ef8a..b5efb3d8fe 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt @@ -26,6 +26,11 @@ code { background-color: yellow; } + #fileuploadstatus, + #fileuploadfailed, + #fileuploadcancel { + display: none; + } @@ -61,6 +66,11 @@
[% IF error == 'cannot_enqueue_job' %]
The import job could not be enqueued. Please try again.
+ [% ELSIF error == 'job_failed_to_queue' %] +
+ The import job was created but could not be queued for processing. + View the job for more details, or try again. +
[% ELSE %]

The patron import job has been enqueued!

@@ -81,18 +91,41 @@
  • If a card number exists in the table, you can choose whether to ignore the new one or overwrite the old one.
    • -
    + +
    + Upload a file to import +
      +
    1. +
      + + +
      +
      2. +
    +
    +
    + + +
    + +
    +
    Upload progress: + + 0% +
    +
    +
    +
    + +
    [% INCLUDE 'csrf-token.inc' %] +
    Import into the borrowers table
      -
    1. - - -
      2. -
    2. @@ -237,7 +270,7 @@
    - +
    @@ -295,7 +328,9 @@ [% Asset.js("js/tools-menu.js") | $raw %] [% INCLUDE 'str/members-menu.inc' %] [% Asset.js("js/members-menu.js") | $raw %] + [% Asset.js("js/file-upload.js") | $raw %] [% END %] [% INCLUDE 'intranet-bottom.inc' %] diff --git a/t/db_dependent/Koha/BackgroundJob/PatronImport.t b/t/db_dependent/Koha/BackgroundJob/PatronImport.t index 1704d3f557..b2713fbb0d 100755 --- a/t/db_dependent/Koha/BackgroundJob/PatronImport.t +++ b/t/db_dependent/Koha/BackgroundJob/PatronImport.t @@ -17,6 +17,8 @@ use Modern::Perl; +use File::Temp qw( tempdir ); +use Test::MockModule; use Test::NoWarnings; use Test::More tests => 7; @@ -24,6 +26,7 @@ use Koha::BackgroundJob::PatronImport; use Koha::BackgroundJobs; use Koha::Database; use Koha::Patrons; +use Koha::UploadedFiles; use t::lib::Mocks; use t::lib::TestBuilder; @@ -31,6 +34,34 @@ use t::lib::TestBuilder; my $schema = Koha::Database->new->schema; my $builder = t::lib::TestBuilder->new; +my $tempdir = tempdir( CLEANUP => 1 ); +my $ctx_mock = Test::MockModule->new('C4::Context'); +$ctx_mock->mock( 'temporary_directory' => sub { return $tempdir; } ); + +my $upload_counter = 0; + +sub build_uploaded_file { + my ($csv) = @_; + + $upload_counter++; + my $upload = Koha::UploadedFile->new( + { + hashvalue => sprintf( 'testhash%03d', $upload_counter ), + filename => "patrons_$upload_counter.csv", + dir => '', + uploadcategorycode => 'temp', + filesize => length($csv), + permanent => 0, + } + )->store; + + open my $fh, '>', $upload->full_path or die "Cannot write test upload file: $!"; + print $fh $csv; + close $fh; + + return $upload; +} + subtest 'enqueue() tests' => sub { plan tests => 5; @@ -39,10 +70,11 @@ subtest 'enqueue() tests' => sub { my $csv = "surname,branchcode,categorycode\nSmith,CPL,PT\nJones,CPL,PT\n"; + my $upload = build_uploaded_file($csv); my $job_id = Koha::BackgroundJob::PatronImport->new->enqueue( { - file_content => $csv, - matchpoint => 'cardnumber', + uploaded_file_id => $upload->id, + matchpoint => 'cardnumber', } ); my $job = Koha::BackgroundJobs->find($job_id)->_derived_class; @@ -53,10 +85,11 @@ subtest 'enqueue() tests' => sub { is( $job->type, 'patron_import', 'job_type is patron_import' ); my $csv_no_newline = "surname,branchcode,categorycode\nSmith,CPL,PT\nJones,CPL,PT"; + my $upload2 = build_uploaded_file($csv_no_newline); my $job_id2 = Koha::BackgroundJob::PatronImport->new->enqueue( { - file_content => $csv_no_newline, - matchpoint => 'cardnumber', + uploaded_file_id => $upload2->id, + matchpoint => 'cardnumber', } ); my $job2 = Koha::BackgroundJobs->find($job_id2)->_derived_class; @@ -67,7 +100,7 @@ subtest 'enqueue() tests' => sub { subtest 'process() tests' => sub { - plan tests => 8; + plan tests => 7; $schema->storage->txn_begin; @@ -85,9 +118,10 @@ subtest 'process() tests' => sub { "" ); + my $upload = build_uploaded_file($csv); my $job_id = Koha::BackgroundJob::PatronImport->new->enqueue( { - file_content => $csv, + uploaded_file_id => $upload->id, matchpoint => 'cardnumber', overwrite_cardnumber => 0, defaults => {}, @@ -107,7 +141,6 @@ subtest 'process() tests' => sub { is( $data->{already_in_db}, 0, 'No patrons already in db' ); is( $data->{invalid}, 0, 'No invalid rows' ); is( $data->{total}, 2, 'Total count is correct' ); - ok( !exists $data->{file_content}, 'file_content cleared from stored data' ); is( Koha::Patrons->search( { cardnumber => { '-in' => [ 'TEST001', 'TEST002' ] } } )->count, 2, 'Patrons exist in the database' @@ -136,9 +169,10 @@ subtest 'process() with errors' => sub { "" ); + my $upload = build_uploaded_file($csv); my $job_id = Koha::BackgroundJob::PatronImport->new->enqueue( { - file_content => $csv, + uploaded_file_id => $upload->id, matchpoint => 'cardnumber', overwrite_cardnumber => 0, defaults => {}, @@ -180,11 +214,12 @@ subtest 'process() already_in_db and overwritten' => sub { "" ); + my $upload = build_uploaded_file($csv); my %base_args = ( - file_content => $csv, - matchpoint => 'cardnumber', - defaults => {}, - preserve_fields => [], + uploaded_file_id => $upload->id, + matchpoint => 'cardnumber', + defaults => {}, + preserve_fields => [], ); # First import creates the patrons @@ -238,9 +273,10 @@ subtest 'process() with patron list creation' => sub { "" ); + my $upload = build_uploaded_file($csv); my $job_id = Koha::BackgroundJob::PatronImport->new->enqueue( { - file_content => $csv, + uploaded_file_id => $upload->id, matchpoint => 'cardnumber', overwrite_cardnumber => 0, defaults => {}, @@ -284,9 +320,10 @@ subtest 'process() with cancelled job' => sub { "" ); + my $upload = build_uploaded_file($csv); my $job_id = Koha::BackgroundJob::PatronImport->new->enqueue( { - file_content => $csv, + uploaded_file_id => $upload->id, matchpoint => 'cardnumber', overwrite_cardnumber => 0, defaults => {}, diff --git a/tools/import_borrowers.pl b/tools/import_borrowers.pl index 92c729757b..175ede475a 100755 --- a/tools/import_borrowers.pl +++ b/tools/import_borrowers.pl @@ -46,6 +46,7 @@ use Koha::DateUtils qw( dt_from_string ); use Koha::Patron::Attribute::Types; use Koha::Patron::Categories; use Koha::Patrons; +use Koha::UploadedFiles; use Text::CSV; @@ -88,9 +89,9 @@ if ( $input->param('sample') ) { my @preserve_fields = $input->multi_param('preserve_existing'); -my $uploadborrowers = $input->param('uploadborrowers'); -my $matchpoint = $input->param('matchpoint'); -my $welcome_new = $input->param('welcome_new'); +my $uploadedfileid = $input->param('uploadedfileid'); +my $matchpoint = $input->param('matchpoint'); +my $welcome_new = $input->param('welcome_new'); if ($matchpoint) { $matchpoint =~ s/^patron_attribute_//; } @@ -100,11 +101,10 @@ my $createpatronlist = $input->param('createpatronlist') || 0; my $dt = dt_from_string(); my $timestamp = $dt->ymd('-') . ' ' . $dt->hms(':'); -if ( $op eq 'cud-import' && $uploadborrowers && length($uploadborrowers) > 0 ) { +if ( $op eq 'cud-import' && $uploadedfileid ) { - my $handle = $input->upload('uploadborrowers'); - my $file_content = do { local $/; <$handle> }; - my $patronlistname = $uploadborrowers . ' (' . $timestamp . ')'; + my $upload = Koha::UploadedFiles->find($uploadedfileid); + my $patronlistname = ( $upload ? $upload->filename : $uploadedfileid ) . ' (' . $timestamp . ')'; my %defaults; for my $k (@columnkeys) { @@ -115,10 +115,11 @@ if ( $op eq 'cud-import' && $uploadborrowers && length($uploadborrowers) > 0 ) { my $overwrite_passwords = defined $input->param('overwrite_passwords') ? 1 : 0; my $update_dateexpiry = $input->param('update_dateexpiry') // ""; + my $job = Koha::BackgroundJob::PatronImport->new; try { - my $job_id = Koha::BackgroundJob::PatronImport->new->enqueue( + my $job_id = $job->enqueue( { - file_content => $file_content, + uploaded_file_id => $uploadedfileid, defaults => \%defaults, matchpoint => $matchpoint, overwrite_cardnumber => scalar $input->param('overwrite_cardnumber'), @@ -134,15 +135,30 @@ if ( $op eq 'cud-import' && $uploadborrowers && length($uploadborrowers) > 0 ) { } ); - $template->param( - op => 'enqueued', - job_id => $job_id, - ); + if ($job_id) { + $template->param( + op => 'enqueued', + job_id => $job_id, + ); + } else { + $template->param( + op => 'enqueued', + error => 'cannot_enqueue_job', + ); + } } catch { - $template->param( - op => 'enqueued', - error => 'cannot_enqueue_job', - ); + if ( $job->id ) { + $template->param( + op => 'enqueued', + job_id => $job->id, + error => 'job_failed_to_queue', + ); + } else { + $template->param( + op => 'enqueued', + error => 'cannot_enqueue_job', + ); + } }; } else {