Bug 19049: Fix regression on stage-marc-import with to_marc plugin

Bug 12412 added the use of to_marc plugins allowing arbitrary file formats
in stage-marc-import (as long as the plugins can handle them). The feature
was not very visible in the code, and when bug 10407 added the marcxml
format, it made some changes that broke the use of to_marc.

This patch restores the functionality by:
[1] Adding a sub RecordsFromMarcPlugin to ImportBatch.pm, specifically
    addressing the conversion from arbitrary formats to MARC::Record.
    The original to_marc interface is used: pass it the file contents,
    and it returns a string consisting of a number of MARC blobs separated
    by \x1D.
    Consequently, the call of to_marc is removed from routine
    BatchStageMarcRecords where it did not belong. The to_marc_plugin
    parameter is removed and two calls are adjusted accordingly.
[2] Instead of a separate combo with plugins, the format combo contains
    MARC, MARCXML and optionally some plugin formats.
[3] The code in stage-marc-import.pl now clearly shows the three main
    format types: MARC, MARCXML or plugin based.

Note: This patch restores more or less the situation after bug 12412, but
I would actually recommend to have the to_marc plugins return MARC::Record
objects instead of large text strings. In the second example I added a
to_marc plugin that actually converts MARC record objects to string format,
while RecordsFromMarcPlugin reconverts them to MARC::Records.

Test plan:
See second patch.

Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl>

Signed-off-by: Katrin Fischer <katrin.fischer.83@web.de>

Signed-off-by: Kyle M Hall <kyle@bywatersolutions.com>

Signed-off-by: Jonathan Druart <jonathan.druart@bugs.koha-community.org>

C4/ImportBatch.pm

@@ -352,8 +352,8 @@ sub ModAuthInBatch {
 
 ( $batch_id, $num_records, $num_items, @invalid_records ) =
   BatchStageMarcRecords(
-    $encoding,                   $marc_records,
-    $file_name,                  $to_marc_plugin,
+    $record_type,                $encoding,
+    $marc_records,               $file_name,
     $marc_modification_template, $comments,
     $branch_code,                $parse_items,
     $leave_as_staging,           $progress_interval,
@@ -367,7 +367,6 @@ sub BatchStageMarcRecords {
     my $encoding = shift;
     my $marc_records = shift;
     my $file_name = shift;
-    my $to_marc_plugin = shift;
     my $marc_modification_template = shift;
     my $comments = shift;
     my $branch_code = shift;
@@ -399,13 +398,6 @@ sub BatchStageMarcRecords {
         SetImportBatchItemAction($batch_id, 'ignore');
     }
 
-    $marc_records = Koha::Plugins::Handler->run(
-        {
-            class  => $to_marc_plugin,
-            method => 'to_marc',
-            params => { data => $marc_records }
-        }
-    ) if $to_marc_plugin && @$marc_records;
 
     my $marc_type = C4::Context->preference('marcflavour');
     $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
@@ -1556,6 +1548,44 @@ sub RecordsFromMARCXMLFile {
     return (\@errors, \@marcRecords);
 }
 
+=head2 RecordsFromMarcPlugin
+
+    Converts text of input_file into array of MARC records with to_marc plugin
+
+=cut
+
+sub RecordsFromMarcPlugin {
+    my ($input_file, $plugin_class, $encoding) = @_;
+
+    # Read input file
+    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
+    my ( $text, $marc, @return );
+    $/ = "\035";
+    while (<IN>) {
+        s/^\s+//;
+        s/\s+$//;
+        next unless $_;
+        $text .= $_;
+    }
+    close IN;
+
+    # Convert to large MARC blob with plugin
+    $text = Koha::Plugins::Handler->run({
+        class  => $plugin_class,
+        method => 'to_marc',
+        params => { data => $text },
+    });
+
+    # Convert to array of MARC records
+    my $marc_type = C4::Context->preference('marcflavour');
+    foreach my $blob ( split(/\x1D/, $text) ) {
+        next if $blob =~ /^\s*$/;
+        my ($marcrecord) = MarcToUTF8Record($blob, $marc_type, $encoding);
+        push @return, $marcrecord;
+    }
+    return \@return;
+}
+
 # internal functions
 
 sub _create_import_record {

koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt

@@ -198,27 +198,13 @@ function cbUpload( status, fileid ) {
         <select name='format' id='format'>
             <option value='ISO2709'>MARC</option>
             <option value='MARCXML'>MARCXML</option>
+            [% FOREACH p IN plugins %]
+                <option value="[% p.metadata.class %]">Other formats, handled by plugin [% p.metadata.name %]</option>
+            [% END %]
         </select>
     </li>
 </ol></fieldset>
 
-  [% IF plugins %]
-    <fieldset class="rows">
-      <legend>Transform file to MARC:</legend>
-      <ol>
-        <li>
-          <label for="comments">Convert file to MARC using the following plugin: </label>
-          <select name="to_marc_plugin" id="to_marc_plugin">
-            <option value="">Do not use.</option>
-              [% FOREACH p IN plugins %]
-                <option value="[% p.metadata.class %]">[% p.metadata.name %]</option>
-              [% END %]
-          </select>
-        </li>
-      </ol>
-    </fieldset>
-  [% END %]
-
   [% IF MarcModificationTemplatesLoop %]
     <fieldset class="rows">
       <legend>Use MARC Modification Template:</legend>

misc/stage_file.pl

@@ -116,7 +116,7 @@ sub process_batch {
     print "... staging MARC records -- please wait\n";
     #FIXME: We should really allow the use of marc modification frameworks and to_marc plugins here if possible
     my ($batch_id, $num_valid_records, $num_items, @import_errors) =
-        BatchStageMarcRecords($record_type, $params->{encoding}, $marc_records, $params->{input_file}, undef, undef, $params->{batch_comment}, '', $params->{add_items}, 0,
+        BatchStageMarcRecords($record_type, $params->{encoding}, $marc_records, $params->{input_file}, undef, $params->{batch_comment}, '', $params->{add_items}, 0,
                               100, \&print_progress_and_commit);
     print "... finished staging MARC records\n";
 

tools/stage-marc-import.pl

@@ -57,8 +57,7 @@ my $item_action                = $input->param('item_action');
 my $comments                   = $input->param('comments');
 my $record_type                = $input->param('record_type');
 my $encoding                   = $input->param('encoding') || 'UTF-8';
-my $format                     = $input->param('format') || 'ISO2709';
-my $to_marc_plugin             = $input->param('to_marc_plugin');
+my $format                     = $input->param('format');
 my $marc_modification_template = $input->param('marc_modification_template_id');
 
 my ( $template, $loggedinuser, $cookie ) = get_template_and_user(
@@ -92,8 +91,13 @@ if ($completedJobID) {
     my ( $errors, $marcrecords );
     if( $format eq 'MARCXML' ) {
         ( $errors, $marcrecords ) = C4::ImportBatch::RecordsFromMARCXMLFile( $file, $encoding);
-    } else {
+    } elsif( $format eq 'MARC' ) {
         ( $errors, $marcrecords ) = C4::ImportBatch::RecordsFromISO2709File( $file, $record_type, $encoding );
+    } elsif( $format ) { # plugin
+        $errors = [];
+        $marcrecords = C4::ImportBatch::RecordsFromMarcPlugin( $file, $format, $encoding );
+    } else {
+        die "No format specified";
     }
     warn "$filename: " . ( join ',', @$errors ) if @$errors;
         # no need to exit if we have no records (or only errors) here
@@ -141,7 +145,7 @@ if ($completedJobID) {
       BatchStageMarcRecords(
         $record_type,    $encoding,
         $marcrecords,    $filename,
-        $to_marc_plugin, $marc_modification_template,
+        $marc_modification_template,
         $comments,       '',
         $parse_items,    0,
         50, staging_progress_callback( $job, $dbh )