Bug 34587: Improve SUSHI COUNTER error handling

This is doing a few things, checking for sushi errors:
1) Abort background job and provide error message if SUSHI returned
   {Severity} in response, this means SUSHI error, docs here:
https://app.swaggerhub.com/apis/COUNTER/counter-sushi_5_0_api/5.0.2#/SUSHI_error_model

2) Abort background job and provide error message if SUSHI provided
   proper response, but contains Exceptions in Report_Header

3) Abort background job and provide error message if SUSHI returned 0
   report items (e.g. everything is fine but there are no items for the
   harvest period provided

4) Abort background job and provide error message if COUNTER file
   created is larger than allowed by the database.

5) Provide error message if manual upload file size exceeds the max
   allowed by the database

Signed-off-by: Jessica Zairo <jzairo@bywatersolutions.com>
Signed-off-by: Michaela Sieber <michaela.sieber@kit.edu>
Signed-off-by: Nick Clemens <nick@bywatersolutions.com>
Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>

Koha/BackgroundJob/ErmSushiHarvester.pm

@@ -155,6 +155,7 @@ sub set_job_size {
             type => 'success', # success, warning or error
             code => 'object_added', # object_added or object_already_exists
             title => $row->{Title},
+            message => 'message',
         }
     );
 

Koha/ERM/EUsage/UsageDataProvider.pm

@@ -215,11 +215,17 @@ sub harvest_sushi {
         return;
     }
 
+    my $decoded_response = decode_json( $response->decoded_content );
+
+    return if $self->_sushi_errors($decoded_response);
+
     # Parse the SUSHI response
     my $sushi_counter =
-        Koha::ERM::EUsage::SushiCounter->new( { response => decode_json( $response->decoded_content ) } );
+        Koha::ERM::EUsage::SushiCounter->new( { response => $decoded_response } );
     my $counter_file = $sushi_counter->get_COUNTER_from_SUSHI;
 
+    return if $self->_counter_file_size_too_large($counter_file);
+
     $self->counter_files( 
         [
             { 
@@ -440,6 +446,77 @@ sub _check_trailing_character {
     return $url;
 }
 
+=head3 _sushi_errors
+
+Checks and handles possible errors in the SUSHI response
+Additionally, adds background job report message(s) if that is the case
+
+=cut
+
+sub _sushi_errors {
+    my ( $self, $decoded_response ) = @_;
+
+    if ( $decoded_response->{Severity} ) {
+        $self->{job_callbacks}->{add_message_callback}->(
+            {
+                type    => 'error',
+                code    => $decoded_response->{Code},
+                message => $decoded_response->{Severity} . ' - ' . $decoded_response->{Message},
+            }
+        ) if $self->{job_callbacks};
+        return 1;
+    }
+
+    if ( $decoded_response->{Report_Header}->{Exceptions} ) {
+        foreach my $exception ( @{ $decoded_response->{Report_Header}->{Exceptions} } ) {
+            $self->{job_callbacks}->{add_message_callback}->(
+                {
+                    type    => 'error',
+                    code    => $exception->{Code},
+                    message => $exception->{Message} . ' - ' . $exception->{Data},
+                }
+            ) if $self->{job_callbacks};
+        }
+        return 1;
+    }
+
+    if ( scalar @{ $decoded_response->{Report_Items} } == 0 ) {
+        $self->{job_callbacks}->{add_message_callback}->(
+            {
+                type => 'error',
+                code => 'no_items',
+            }
+        ) if $self->{job_callbacks};
+        return 1;
+    }
+
+    return 0;
+}
+
+=head3 _counter_file_size_too_large
+
+Checks whether a counter file size exceeds the size allowed by the database or not
+Additionally, adds a background job report message if that is the case
+
+=cut
+
+sub _counter_file_size_too_large {
+    my ( $self, $counter_file ) = @_;
+
+    my $max_allowed_packet = C4::Context->dbh->selectrow_array(q{SELECT @@max_allowed_packet});
+    if ( length($counter_file) > $max_allowed_packet ) {
+        $self->{job_callbacks}->{add_message_callback}->(
+            {
+                type    => 'error',
+                code    => 'payload_too_large',
+                message => $max_allowed_packet / 1024 / 1024,
+            }
+        ) if $self->{job_callbacks};
+        return 1;
+    }
+    return 0;
+}
+
 =head3 _type
 
 =cut

Koha/REST/V1/ERM/EUsage/UsageDataProviders.pm

@@ -313,6 +313,10 @@ sub process_COUNTER_file {
                     ? decode_base64( $body->{file_content} )
                     : "";
 
+                my $max_allowed_packet = C4::Context->dbh->selectrow_array(q{SELECT @@max_allowed_packet});
+                Koha::Exceptions::PayloadTooLarge->throw("File size exceeds limit defined by server")
+                    if length($file_content) > $max_allowed_packet;
+
                 # Validate the file_content without storing, it'll throw an exception if fail
                 my $counter_file_validation = Koha::ERM::EUsage::CounterFile->new( { file_content => $file_content } );
                 $counter_file_validation->validate;
@@ -356,7 +360,7 @@ sub process_COUNTER_file {
             } elsif ( $_->isa('Koha::Exceptions::PayloadTooLarge') ) {
                 return $c->render(
                     status  => 413,
-                    openapi => { error => $_->error }
+                    openapi => { error => $_->description }
                 );
             } elsif ( $_->isa('Koha::Exceptions::ERM::EUsage::CounterFile::UnsupportedRelease') ) {
                 return $c->render(

api/v1/swagger/paths/erm_usage_data_providers.yaml

@@ -437,6 +437,10 @@
         description: Access forbidden
         schema:
           $ref: "../swagger.yaml#/definitions/error"
+      413:
+        description: Payload too large
+        schema:
+          $ref: "../swagger.yaml#/definitions/error"
       500:
         description: |-
           Internal server error. Possible `error_code` attribute values:

koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc

@@ -84,12 +84,20 @@
                 <i class="fa fa-exclamation error"></i>
             [% END %]
             [% SWITCH m.code %]
-            [% CASE 'object_already_exists' %]
-                <span><strong>[% m.title %]</strong> already exists in this data provider and was not created. New usage statistics were verified anyway</span>
-            [% CASE 'object_added' %]
-                <span><strong>[% m.title %]</strong> successfully added</span>
-            [% CASE 'object_could_not_be_added' %]
-                <span><strong>[% m.title %]</strong> could not be processed - please check the logs</span>
+                [% CASE 'object_already_exists' %]
+                    <span><strong>[% m.title %]</strong> already exists in this data provider and was not created. New usage statistics were verified anyway</span>
+                [% CASE 'object_added' %]
+                    <span><strong>[% m.title %]</strong> successfully added</span>
+                [% CASE 'object_could_not_be_added' %]
+                    <span><strong>[% m.title %]</strong> could not be processed - please check the logs</span>
+                [% CASE 'payload_too_large' %]
+                    <span>COUNTER report file size exceeds limit defined by the database. Limit is [% m.message %] MB.</span><br>
+                    <span>Please increase this limit or harvest a shorter period.</span>
+                [% CASE 'no_items' %]
+                    <span>SUSHI response returned no items.</span><br>
+                    <span>Please try a different harvest period.</span>
+                [% CASE %]
+                    <span>SUSHI returned: [% m.message %] [% m.code %]</span>
             [% END %]
         </div>
     [% END %]