Bug 34587: File upload - Background job

ErmSushiHarvester background job will now be initiated by either enqueue_counter_file_processing_job or by enqueue_sushi_harvest_jobs from Koha/ERM/UsageDataProvider.pm, the former if triggered by a manual file upload, the latter if by the 'run now' button or by the cron script. This commit also includes some rewording/refactoring, namely: - COUNTER file validation now happens in the API, before enqueuing the job. - Removal of no longer used POST /erm/counter_files endpoint - Koha/ERM/UsageDataProvider.pm: -- run method is now enqueue_sushi_harvest_jobs -- new enqueue_counter_file_processing_job method -- harvest method is now harvest_sushi -- new set_background_job_callbacks method to set the background job callbacks - REST/V1/ERM/UsageDataProviders -- run method is now process_SUSHI_response -- new process_COUNTER_file endpoint Signed-off-by: Jessica Zairo <jzairo@bywatersolutions.com> Signed-off-by: Michaela Sieber <michaela.sieber@kit.edu> Signed-off-by: Nick Clemens <nick@bywatersolutions.com> Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>
2023-08-21 14:50:45 +00:00 · 2023-08-21 14:50:45 +00:00 · cf87082682
parent e5ec3df36e
commit cf87082682
12 changed files with 291 additions and 119 deletions
--- a/Koha/BackgroundJob/ErmSushiHarvester.pm
+++ b/Koha/BackgroundJob/ErmSushiHarvester.pm
@ -75,10 +75,7 @@ sub process {
    my $ud_provider =
      Koha::ERM::UsageDataProviders->find( $args->{ud_provider_id} );

-    $ud_provider->harvest(
-        $args->{begin_date},
-        $args->{end_date},
-        $args->{report_type},
+    $ud_provider->set_background_job_callbacks(
        {
            report_info_callback => sub { $self->report_info(@_); },
            step_callback        => sub { $self->step; },
@ -87,9 +84,34 @@ sub process {
        }
    );

+    my $counter_file;
+    if ( $args->{file_content} ) {
+        my $counter_files_rs = $ud_provider->counter_files(
+            [
+                {
+                    usage_data_provider_id => $ud_provider->erm_usage_data_provider_id,
+                    file_content           => $args->{file_content},
+                    date_uploaded          => POSIX::strftime( "%Y%m%d%H%M%S", localtime ),
+                    filename               => $ud_provider->name . "_" . $ud_provider->{report_type},
+                }
+            ]
+        );
+        $counter_file = $counter_files_rs->last;
+    } else {
+        $ud_provider->harvest_sushi(
+            {
+                begin_date  => $args->{begin_date},
+                end_date    => $args->{end_date},
+                report_type => $args->{report_type}
+            }
+        );
+    }
+
+    my $job_report_report_type = $ud_provider->{report_type} || $counter_file->type;
+
    # Prepare job report
    my $report = {
-        report_type      => $ud_provider->{report_type},
+        report_type      => $job_report_report_type,
        total_records    => $ud_provider->{total_records},
        us_report_info   => $self->{us_report_info},
        ud_provider_id   => $ud_provider->erm_usage_data_provider_id,
--- a/Koha/ERM/CounterFile.pm
+++ b/Koha/ERM/CounterFile.pm
@ -81,7 +81,6 @@ Receive background_job_callbacks to be able to update job progress
 sub store {
    my ( $self, $background_job_callbacks ) = @_;

-    $self->_validate;
    $self->_set_report_type_from_file;

    my $result = $self->SUPER::store;
@ -275,7 +274,7 @@ sub _add_yearly_usage_entries {
    }
 }

-=head3 _validate
+=head3 validate

 Verifies if the given file_content is a valid COUNTER file or not

@ -284,7 +283,7 @@ A I <Koha::Exceptions::ERM::CounterFile> exception is thrown

 =cut

-sub _validate {
+sub validate {
    my ($self) = @_;

    open my $fh, "<", \$self->file_content or die;
--- a/Koha/ERM/UsageDataProvider.pm
+++ b/Koha/ERM/UsageDataProvider.pm
@ -126,17 +126,17 @@ sub enqueue_sushi_harvest_jobs {
    return \@jobs;
 }

-=head3 harvest
+=head3 harvest_sushi

-    $ud_provider->harvest(
+    $ud_provider->harvest_sushi(
        {
-            step_callback        => sub { $self->step; },
-            set_size_callback    => sub { $self->set_job_size(@_); },
-            add_message_callback => sub { $self->add_message(@_); },
+            begin_date  => $args->{begin_date},
+            end_date    => $args->{end_date},
+            report_type => $args->{report_type}
        }
    );

-Run the SUSHI harvester of this usage data provider
+Runs this usage data provider's SUSHI harvester
 Builds the URL query and requests the COUNTER 5 SUSHI service

 COUNTER SUSHI api spec:
@ -144,30 +144,37 @@ https://app.swaggerhub.com/apis/COUNTER/counter-sushi_5_0_api/5.0.2

 =over

+=item begin_date
+
+Begin date of the SUSHI harvest
+
+=back
+
+=over
+
+=item end_date
+
+End date of the SUSHI harvest
+
+=back
+
+=over
+
 =item report_type

 Report type to run this harvest on

 =back

-=over
-
-=item background_job_callbacks
-
-Receive background_job_callbacks to be able to update job
-
-=back
-
 =cut

-sub harvest {
-    my ( $self, $begin_date, $end_date, $report_type, $background_job_callbacks ) = @_;
+sub harvest_sushi {
+    my ( $self, $args ) = @_;

    # Set class wide vars
-    $self->{job_callbacks} = $background_job_callbacks;
-    $self->{report_type}   = $report_type;
-    $self->{begin_date}    = $begin_date;
-    $self->{end_date}      = $end_date;
+    $self->{report_type} = $args->{report_type};
+    $self->{begin_date}  = $args->{begin_date};
+    $self->{end_date}    = $args->{end_date};

    my $url      = $self->_build_url_query;
    my $request  = HTTP::Request->new( 'GET' => $url );
@ -217,6 +224,28 @@ sub harvest {
    $self->parse_SUSHI_response( decode_json( $response->decoded_content ) );
 }

+=head3 set_background_job_callbacks
+
+    $self->set_background_job_callbacks($background_job_callbacks);
+
+Sets the background job callbacks
+
+=over
+
+=item background_job_callbacks
+
+Background job callbacks
+
+=back
+
+=cut
+
+sub set_background_job_callbacks {
+    my ( $self, $background_job_callbacks ) = @_;
+
+    $self->{job_callbacks} = $background_job_callbacks;
+}
+
 =head3 parse_SUSHI_response

    $self->parse_SUSHI_response( decode_json( $response->decoded_content ) );
--- a/Koha/REST/V1/ERM/CounterFiles.pm
+++ b/Koha/REST/V1/ERM/CounterFiles.pm
@ -19,7 +19,6 @@ package Koha::REST::V1::ERM::CounterFiles;

 use Modern::Perl;

-use MIME::Base64 qw( decode_base64 );
 use Mojo::Base 'Mojolicious::Controller';

 use Koha::ERM::CounterFiles;
--- a/Koha/REST/V1/ERM/UsageDataProviders.pm
+++ b/Koha/REST/V1/ERM/UsageDataProviders.pm
@ -19,6 +19,7 @@ package Koha::REST::V1::ERM::UsageDataProviders;

 use Modern::Perl;

+use MIME::Base64 qw( decode_base64 );
 use Mojo::Base 'Mojolicious::Controller';

 use Koha::ERM::UsageDataProviders;
@ -307,11 +308,113 @@ sub delete {
    };
 }

-=head3 run
+=head3 process_COUNTER_file
+
+Controller function that handles processing of the COUNTER file
+It will->enqueue_counter_file_processing_job for its respective usage data provider

 =cut

-sub run {
+sub process_COUNTER_file {
+    my $c = shift->openapi->valid_input or return;
+
+    return try {
+        Koha::Database->new->schema->txn_do(
+            sub {
+
+                my $body = $c->validation->param('body');
+
+                my $file_content =
+                  defined( $body->{file_content} )
+                  ? decode_base64( $body->{file_content} )
+                  : "";
+
+   # Validate the file_content without storing, it'll throw an exception if fail
+                my $counter_file_validation =
+                  Koha::ERM::CounterFile->new(
+                    { file_content => $file_content } );
+                $counter_file_validation->validate;
+
+                # Validation was successful, enqueue the job
+                my $udprovider =
+                  Koha::ERM::UsageDataProviders->find(
+                    $c->validation->param('erm_usage_data_provider_id') );
+
+                my $jobs = $udprovider->enqueue_counter_file_processing_job(
+                    {
+                        file_content => $file_content,
+                    }
+                );
+
+                return $c->render(
+                    status  => 200,
+                    openapi => { jobs => [ @{$jobs} ] }
+                );
+            }
+        );
+    }
+    catch {
+
+        my $to_api_mapping = Koha::ERM::CounterFile->new->to_api_mapping;
+
+        if ( blessed $_ ) {
+            if ( $_->isa('Koha::Exceptions::Object::DuplicateID') ) {
+                return $c->render(
+                    status  => 409,
+                    openapi =>
+                      { error => $_->error, conflict => $_->duplicate_id }
+                );
+            }
+            elsif ( $_->isa('Koha::Exceptions::Object::FKConstraint') ) {
+                return $c->render(
+                    status  => 400,
+                    openapi => {
+                            error => "Given "
+                          . $to_api_mapping->{ $_->broken_fk }
+                          . " does not exist"
+                    }
+                );
+            }
+            elsif ( $_->isa('Koha::Exceptions::BadParameter') ) {
+                return $c->render(
+                    status  => 400,
+                    openapi => {
+                            error => "Given "
+                          . $to_api_mapping->{ $_->parameter }
+                          . " does not exist"
+                    }
+                );
+            }
+            elsif ( $_->isa('Koha::Exceptions::PayloadTooLarge') ) {
+                return $c->render(
+                    status  => 413,
+                    openapi => { error => $_->error }
+                );
+            }
+            elsif (
+                $_->isa(
+                    'Koha::Exceptions::ERM::CounterFile::UnsupportedRelease')
+              )
+            {
+                return $c->render(
+                    status  => 400,
+                    openapi => { error => $_->description }
+                );
+            }
+        }
+
+        $c->unhandled_exception($_);
+    };
+}
+
+=head3 process_SUSHI_response
+
+Controller function that handles processing of the SUSHI response
+It will ->enqueue_sushi_harvest_jobs for this usage data provider
+
+=cut
+
+sub process_SUSHI_response {
    my $c = shift->openapi->valid_input or return;

    my $body       = $c->validation->param('body');
@ -325,7 +428,8 @@ sub run {
        );
    }

-    my $udprovider = Koha::ERM::UsageDataProviders->find( $c->validation->param('erm_usage_data_provider_id') );
+    my $udprovider = Koha::ERM::UsageDataProviders->find(
+        $c->validation->param('erm_usage_data_provider_id') );

    unless ($udprovider) {
        return $c->render(
@ -335,7 +439,12 @@ sub run {
    }

    return try {
-        my $jobs = $udprovider->run( { begin_date => $begin_date, end_date => $end_date } );
+        my $jobs = $udprovider->enqueue_sushi_harvest_jobs(
+            {
+                begin_date => $begin_date,
+                end_date   => $end_date
+            }
+        );

        return $c->render(
            status  => 200,
@ -354,7 +463,8 @@ sub run {
 sub test_connection {
    my $c = shift->openapi->valid_input or return;

-    my $udprovider = Koha::ERM::UsageDataProviders->find( $c->validation->param('erm_usage_data_provider_id') );
+    my $udprovider = Koha::ERM::UsageDataProviders->find(
+        $c->validation->param('erm_usage_data_provider_id') );

    unless ($udprovider) {
        return $c->render(
--- a/api/v1/swagger/paths/erm_counter_files.yaml
+++ b/api/v1/swagger/paths/erm_counter_files.yaml
@ -129,66 +129,6 @@
    x-koha-authorization:
      permissions:
        erm: 1
-  post:
-    x-mojo-to: ERM::CounterFiles#add
-    operationId: addErmCounterFiles
-    tags:
-      - counter_file
-    summary: Add counter_file
-    consumes:
-      - application/json
-    produces:
-      - application/json
-    parameters:
-      - description: A JSON object containing information about the new counter_file
-        in: body
-        name: body
-        required: true
-        schema:
-            $ref: "../swagger.yaml#/definitions/erm_counter_file"
-    responses:
-      201:
-        description: A successfully created counter_file
-        schema:
-          items:
-            $ref: "../swagger.yaml#/definitions/erm_counter_file"
-      400:
-        description: Bad parameter
-        schema:
-          $ref: "../swagger.yaml#/definitions/error"
-      401:
-        description: Authentication required
-        schema:
-          $ref: "../swagger.yaml#/definitions/error"
-      403:
-        description: Access forbidden
-        schema:
-          $ref: "../swagger.yaml#/definitions/error"
-      404:
-        description: Ressource not found
-        schema:
-          $ref: "../swagger.yaml#/definitions/error"
-      409:
-        description: Conflict in creating resource
-        schema:
-          $ref: "../swagger.yaml#/definitions/error"
-      413:
-        description: Payload too large
-        schema:
-          $ref: "../swagger.yaml#/definitions/error"
-      500:
-        description: |-
-          Internal server error. Possible `error_code` attribute values:
-          * `internal_server_error`
-        schema:
-          $ref: "../swagger.yaml#/definitions/error"
-      503:
-        description: Under maintenance
-        schema:
-          $ref: "../swagger.yaml#/definitions/error"
-    x-koha-authorization:
-      permissions:
-        erm: 1
 "/erm/counter_files/{erm_counter_files_id}":
  delete:
    x-mojo-to: ERM::CounterFiles#delete
--- a/api/v1/swagger/paths/erm_usage_data_providers.yaml
+++ b/api/v1/swagger/paths/erm_usage_data_providers.yaml
@ -353,13 +353,13 @@
    x-koha-authorization:
      permissions:
        erm: 1
-"/erm/usage_data_providers/{erm_usage_data_provider_id}/run":
+"/erm/usage_data_providers/{erm_usage_data_provider_id}/process_SUSHI_response":
  post:
-    x-mojo-to: ERM::UsageDataProviders#run
-    operationId: runUsageDataProviderHarvester
+    x-mojo-to: ERM::UsageDataProviders#process_SUSHI_response
+    operationId: processSUSHICounterUsageDataProviderHarvester
    tags:
      - usage data provider harvester
-    summary: Run this data provider's harvester
+    summary: Process SUSHI COUNTER for this data provider's harvester
    produces:
      - application/json
    parameters:
@ -382,7 +382,51 @@
          additionalProperties: false
    responses:
      200:
-        description: Successful harvest run
+        description: Successful SUSHI COUNTER processing
+        schema:
+          items:
+            $ref: "../swagger.yaml#/definitions/erm_usage_data_provider"
+      400:
+        description: Bad request
+        schema:
+          $ref: "../swagger.yaml#/definitions/error"
+      403:
+        description: Access forbidden
+        schema:
+          $ref: "../swagger.yaml#/definitions/error"
+      500:
+        description: |-
+          Internal server error. Possible `error_code` attribute values:
+          * `internal_server_error`
+        schema:
+          $ref: "../swagger.yaml#/definitions/error"
+      503:
+        description: Under maintenance
+        schema:
+          $ref: "../swagger.yaml#/definitions/error"
+    x-koha-authorization:
+      permissions:
+        erm: 1
+"/erm/usage_data_providers/{erm_usage_data_provider_id}/process_COUNTER_file":
+  post:
+    x-mojo-to: ERM::UsageDataProviders#process_COUNTER_file
+    operationId: processCOUNTERFileUsageDataProviderHarvester
+    tags:
+      - usage data provider harvester
+    summary: Process COUNTER file upload for this data provider's harvester
+    produces:
+      - application/json
+    parameters:
+      - $ref: "../swagger.yaml#/parameters/erm_usage_data_provider_id_pp"
+      - description: A JSON object containing information about the new counter_file
+        in: body
+        name: body
+        required: true
+        schema:
+            $ref: "../swagger.yaml#/definitions/erm_counter_file"
+    responses:
+      200:
+        description: Successful COUNTER file processing
        schema:
          items:
            $ref: "../swagger.yaml#/definitions/erm_usage_data_provider"
--- a/api/v1/swagger/swagger.yaml
+++ b/api/v1/swagger/swagger.yaml
@ -309,8 +309,10 @@ paths:
    $ref: ./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers
  "/erm/usage_data_providers/{erm_usage_data_provider_id}":
    $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}"
-  "/erm/usage_data_providers/{erm_usage_data_provider_id}/run":
-    $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}~1run"
+  "/erm/usage_data_providers/{erm_usage_data_provider_id}/process_SUSHI_response":
+    $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}~1process_SUSHI_response"
+  "/erm/usage_data_providers/{erm_usage_data_provider_id}/process_COUNTER_file":
+    $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}~1process_COUNTER_file"
  "/erm/usage_data_providers/{erm_usage_data_provider_id}/test_connection":
    $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}~1test_connection"
  /erm/usage_databases:
--- a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc
+++ b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc
@ -18,18 +18,20 @@
                        <td>Report rows processed</td>
                        <td>[% job.progress %]</td>
                    </tr>
-                    <tr>
-                    [% IF report.report_type.match( '^PR' ) %]
-                        <td>Platforms processed</td>
-                    [% ELSIF report.report_type.match( '^DR' ) %]
-                        <td>Databases processed</td>
-                    [% ELSIF report.report_type.match( '^IR' ) %]
-                        <td>Items processed</td>
-                    [% ELSIF report.report_type.match( '^TR' ) %]
-                        <td>Titles processed</td>
+                    [% IF report.total_records %]
+                        <tr>
+                        [% IF report.report_type.match( '^PR' ) %]
+                            <td>SUSHI Platforms processed</td>
+                        [% ELSIF report.report_type.match( '^DR' ) %]
+                            <td>SUSHI Databases processed</td>
+                        [% ELSIF report.report_type.match( '^IR' ) %]
+                            <td>SUSHI Items processed</td>
+                        [% ELSIF report.report_type.match( '^TR' ) %]
+                            <td>SUSHI Titles processed</td>
+                        [% END %]
+                            <td>[% report.total_records | html %]</td>
+                        </tr>
                    [% END %]
-                        <td>[% report.total_records | html %]</td>
-                    </tr>
                    <tr>
                        <td>Monthly usages skipped</td>
                        <td>[% report.us_report_info.skipped_mus | html %]</td>
--- a/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersFileImport.vue
+++ b/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersFileImport.vue
@ -35,11 +35,21 @@
 </template>

 <script>
+import { inject } from "vue"
 import ButtonSubmit from "../ButtonSubmit.vue"
-import { setMessage } from "../../messages"
 import { APIClient } from "../../fetch/api-client.js"

 export default {
+    setup() {
+        const AVStore = inject("AVStore") // Left in for future permissions fixes
+        const { get_lib_from_av, map_av_dt_filter } = AVStore
+
+        const { setMessage } = inject("mainStore")
+
+        return {
+            setMessage,
+        }
+    },
    data() {
        return {
            file: {
@ -69,15 +79,21 @@ export default {
        addDocument(e) {
            e.preventDefault()
            const client = APIClient.erm
-            client.counter_files
-                .create({
+            client.usage_data_providers
+                .process_COUNTER_file(this.file.usage_data_provider_id, {
                    usage_data_provider_id: this.file.usage_data_provider_id,
                    filename: this.file.filename,
                    file_content: this.file.file_content,
                })
                .then(
                    success => {
-                        setMessage(this.$__("COUNTER file uploaded"))
+                        let message = ""
+                        success.jobs.forEach((job, i) => {
+                            message += this.$__(
+                                '<li>Job for uploaded file has been queued, <a href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id=%s" target="_blank">click here</a> to check its progress.</li>'
+                            ).format(job.job_id)
+                        })
+                        this.setMessage(message, true)
                    },
                    error => {}
                )
--- a/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersList.vue
+++ b/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersList.vue
@ -223,7 +223,7 @@ export default {
                    callback_result => {
                        const client = APIClient.erm
                        client.usage_data_providers
-                            .run(id, {
+                            .process_SUSHI_response(id, {
                                begin_date: callback_result.inputs.find(
                                    input => {
                                        return input.id == "begin_date"
--- a/koha-tmpl/intranet-tmpl/prog/js/vue/fetch/erm-api-client.js
+++ b/koha-tmpl/intranet-tmpl/prog/js/vue/fetch/erm-api-client.js
@ -277,9 +277,18 @@ export class ERMAPIClient extends HttpClient {
                    endpoint: "usage_data_providers/" + id,
                    body: usage_data_provider,
                }),
-            run: (id, body) =>
+            process_SUSHI_response: (id, body) =>
                this.post({
-                    endpoint: "usage_data_providers/" + id + "/run",
+                    endpoint:
+                        "usage_data_providers/" +
+                        id +
+                        "/process_SUSHI_response",
+                    body: body,
+                }),
+            process_COUNTER_file: (id, body) =>
+                this.post({
+                    endpoint:
+                        "usage_data_providers/" + id + "/process_COUNTER_file",
                    body: body,
                }),
            test: id =>