gitea
/
Koha-clms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Koha-clms/misc/export_records.pl

594 lines
20 KiB
Perl
Executable File
Raw Blame History

#!/usr/bin/perl
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <https://www.gnu.org/licenses>.
use Modern::Perl;
use MARC::File::XML;
use List::MoreUtils qw( uniq );
use Getopt::Long qw( GetOptions );
use Pod::Usage qw( pod2usage );
use File::Basename qw( fileparse );
use Koha::Script;
use C4::Auth;
use C4::Context;
use C4::Record;
use C4::Reports::Guided qw( execute_query );
use Koha::Biblioitems;
use Koha::Database;
use Koha::CsvProfiles;
use Koha::Exporter::Record;
use Koha::DateUtils qw( dt_from_string output_pref );
use Koha::Reports;
use Koha::File::Transports;
my (
$output_format,
$timestamp,
$dont_export_items,
$csv_profile_id,
$deleted_barcodes,
$clean,
$filename,
$record_type,
$id_list_file,
$starting_authid,
$ending_authid,
$authtype,
$starting_biblionumber,
$ending_biblionumber,
$itemtype,
$starting_callnumber,
$ending_callnumber,
$start_accession,
$end_accession,
$marc_conditions,
$embed_see_from_headings,
$report_id,
@report_params,
$report,
$sql,
$params_needed,
$destination_server_id,
$delete_local_after_run,
$help,
);
GetOptions(
'format=s' => \$output_format,
'date=s' => \$timestamp,
'dont_export_items' => \$dont_export_items,
'csv_profile_id=s' => \$csv_profile_id,
'deleted_barcodes' => \$deleted_barcodes,
'clean' => \$clean,
'filename=s' => \$filename,
'record-type=s' => \$record_type,
'id_list_file=s' => \$id_list_file,
'starting_authid=s' => \$starting_authid,
'ending_authid=s' => \$ending_authid,
'authtype=s' => \$authtype,
'starting_biblionumber=s' => \$starting_biblionumber,
'ending_biblionumber=s' => \$ending_biblionumber,
'itemtype=s' => \$itemtype,
'starting_callnumber=s' => \$starting_callnumber,
'ending_callnumber=s' => \$ending_callnumber,
'start_accession=s' => \$start_accession,
'end_accession=s' => \$end_accession,
'marc_conditions=s' => \$marc_conditions,
'embed_see_from_headings' => \$embed_see_from_headings,
'report_id=s' => \$report_id,
'report_param=s' => \@report_params,
'destination_server_id=s' => \$destination_server_id,
'delete_local_after_run' => \$delete_local_after_run,
'h|help|?' => \$help,
) || pod2usage(1);
if ($help) {
pod2usage(1);
}
$filename ||= 'koha.mrc';
$output_format ||= 'iso2709';
$record_type ||= 'bibs';
# Retrocompatibility for the format parameter
$output_format = 'iso2709' if $output_format eq 'marc';
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
pod2usage(q|CSV output is only available for biblio records|);
}
if ( $output_format eq 'csv' and not $csv_profile_id ) {
pod2usage(q|Define a csv profile to export in CSV|);
}
if ( $record_type ne 'bibs' and $record_type ne 'auths' ) {
pod2usage(q|--record_type is not valid|);
}
if ( $deleted_barcodes and $record_type ne 'bibs' ) {
pod2usage(q|--deleted_barcodes can only be used with biblios|);
}
my $file_transport;
if ($destination_server_id) {
$file_transport = Koha::File::Transports->find($destination_server_id);
pod2usage( sprintf( "No file transport server (%s) found", $destination_server_id ) )
unless $file_transport;
}
# Validate flag combinations
if ( $delete_local_after_run && !$destination_server_id ) {
pod2usage("--delete_local_after_run requires --destination_server_id to be specified");
}
if ($report_id) {
# Check report exists
$report = Koha::Reports->find($report_id);
unless ($report) {
pod2usage( sprintf( "No saved report (%s) found", $report_id ) );
}
$sql = $report->savedsql;
# Check defined report can be used to export the record_type
if ( $sql !~ /biblionumber/ && $record_type eq 'bibs' ) {
pod2usage(q|The --report_id you specified does not fetch a biblionumber|);
} elsif ( $sql !~ /authid/ && $record_type eq 'auths' ) {
pod2usage(q|The --report_id you specified does not fetch an authid|);
}
# convert SQL parameters to placeholders
my $params_needed = ( $sql =~ s/(<<[^>]+>>)/\?/g );
die( "You supplied " . scalar @report_params . " parameter(s) and $params_needed are required by the report" )
if scalar @report_params != $params_needed;
}
$start_accession = dt_from_string($start_accession) if $start_accession;
$end_accession = dt_from_string($end_accession) if $end_accession;
# Parse marc conditions
my @marc_conditions;
if ($marc_conditions) {
foreach my $condition ( split( /,\s*/, $marc_conditions ) ) {
if ( $condition =~ /^(\d{3})([\w\d]?)(=|(?:!=)|>|<)([^,]+)$/ ) {
push @marc_conditions, [ $1, $2, $3, $4 ];
} elsif ( $condition =~ /^(exists|not_exists)\((\d{3})([\w\d]?)\)$/ ) {
push @marc_conditions, [ $2, $3, $1 eq 'exists' ? '?' : '!?' ];
} else {
die("Invalid condititon: $condition");
}
}
}
my $dbh = C4::Context->dbh;
# Redirect stdout
open STDOUT, '>', $filename if $filename;
my @record_ids;
$timestamp =
($timestamp) ? output_pref( { dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, } ) : '';
if ( $record_type eq 'bibs' ) {
if ($report) {
# Run the report and fetch biblionumbers
my ($sth) = execute_query(
{
sql => $sql,
sql_params => \@report_params,
report_id => $report_id,
}
);
while ( my $row = $sth->fetchrow_hashref() ) {
if ( $row->{biblionumber} ) {
push @record_ids, $row->{biblionumber};
} else {
pod2usage(q|The --report_id you specified returned no biblionumbers|);
}
}
} elsif ($timestamp) {
if ( !$dont_export_items ) {
push @record_ids, $_->{biblionumber} for @{
$dbh->selectall_arrayref(
q| (
SELECT biblio_metadata.biblionumber
FROM biblio_metadata
LEFT JOIN items USING(biblionumber)
WHERE biblio_metadata.timestamp >= ?
OR items.timestamp >= ?
) UNION (
SELECT biblio_metadata.biblionumber
FROM biblio_metadata
LEFT JOIN deleteditems USING(biblionumber)
WHERE biblio_metadata.timestamp >= ?
OR deleteditems.timestamp >= ?
) |, { Slice => {} }, ($timestamp) x 4
);
};
} else {
push @record_ids, $_->{biblionumber} for @{
$dbh->selectall_arrayref(
q| (
SELECT biblio_metadata.biblionumber
FROM biblio_metadata
WHERE biblio_metadata.timestamp >= ?
) |, { Slice => {} }, $timestamp
);
};
}
} else {
my $conditions = {
( $starting_biblionumber or $ending_biblionumber )
? (
"me.biblionumber" => {
( $starting_biblionumber ? ( '>=' => $starting_biblionumber ) : () ),
( $ending_biblionumber ? ( '<=' => $ending_biblionumber ) : () ),
}
)
: (),
( $starting_callnumber or $ending_callnumber )
? (
callnumber => {
( $starting_callnumber ? ( '>=' => $starting_callnumber ) : () ),
( $ending_callnumber ? ( '<=' => $ending_callnumber ) : () ),
}
)
: (),
( $start_accession or $end_accession )
? (
dateaccessioned => {
( $start_accession ? ( '>=' => $start_accession ) : () ),
( $end_accession ? ( '<=' => $end_accession ) : () ),
}
)
: (),
(
$itemtype
? C4::Context->preference('item-level_itypes')
? ( 'items.itype' => $itemtype )
: ( 'me.itemtype' => $itemtype )
: ()
),
};
my $biblioitems = Koha::Biblioitems->search( $conditions, { join => 'items' } );
while ( my $biblioitem = $biblioitems->next ) {
push @record_ids, $biblioitem->biblionumber;
}
}
} elsif ( $record_type eq 'auths' ) {
if ($report) {
# Run the report and fetch authids
my ($sth) = execute_query(
{
sql => $sql,
sql_params => \@report_params,
report_id => $report_id,
}
);
while ( my $row = $sth->fetchrow_hashref() ) {
if ( $row->{authid} ) {
push @record_ids, $row->{authid};
} else {
pod2usage(q|The --report_id you specified returned no authids|);
}
}
} elsif ($timestamp) {
push @record_ids, $_->{authid} for @{
$dbh->selectall_arrayref(
q| (
SELECT authid
FROM auth_header
WHERE modification_time >= ?
) |, { Slice => {} }, $timestamp
);
};
} else {
my $conditions = {
( $starting_authid or $ending_authid )
? (
authid => {
( $starting_authid ? ( '>=' => $starting_authid ) : () ),
( $ending_authid ? ( '<=' => $ending_authid ) : () ),
}
)
: (),
( $authtype ? ( authtypecode => $authtype ) : () ),
};
# Koha::MetadataRecord::Authority is not a Koha::Object...
my $authorities = Koha::Database->new->schema->resultset('AuthHeader')->search($conditions);
@record_ids = map { $_->authid } $authorities->all;
}
}
@record_ids = uniq @record_ids;
if ( @record_ids and $id_list_file ) {
open my $fh, '<', $id_list_file or die "Cannot open file $id_list_file ($!)";
my @filter_record_ids = <$fh>;
@filter_record_ids = map { my $id = $_; $id =~ s/[\r\n]*$//; $id } @filter_record_ids;
# intersection
my %record_ids = map { $_ => 1 } @record_ids;
@record_ids = grep $record_ids{$_}, @filter_record_ids;
}
if ($deleted_barcodes) {
for my $record_id (@record_ids) {
my $barcode = $dbh->selectall_arrayref(
q|
SELECT DISTINCT barcode
FROM deleteditems
WHERE deleteditems.biblionumber = ?
AND barcode IS NOT NULL AND barcode != ''
|, { Slice => {} }, $record_id
);
say $_->{barcode} for @$barcode;
}
} else {
Koha::Exporter::Record::export(
{
record_type => $record_type,
record_ids => \@record_ids,
record_conditions => @marc_conditions ? \@marc_conditions : undef,
format => $output_format,
csv_profile_id => $csv_profile_id,
export_items => ( not $dont_export_items ),
clean => $clean || 0,
embed_see_from_headings => $embed_see_from_headings || 0,
}
);
}
if ($file_transport) {
# Verify the file was created successfully before attempting upload
unless ( -f $filename ) {
die "Error: Output file '$filename' was not created successfully\n";
}
# Connect to the transport
unless ( $file_transport->connect ) {
die sprintf( "Error: Unable to connect to file transport server (ID: %s)\n", $destination_server_id );
}
# Change to upload directory if specified
my $upload_dir = $file_transport->upload_directory;
if ($upload_dir) {
unless ( $file_transport->change_directory($upload_dir) ) {
$file_transport->disconnect;
die sprintf(
"Error: Unable to change to upload directory '%s' on server (ID: %s)\n", $upload_dir,
$destination_server_id
);
}
}
# Upload the file
# Extract just the filename from the path for remote upload
my ($remote_filename) = fileparse($filename);
unless ( $file_transport->upload_file( $filename, $remote_filename ) ) {
$file_transport->disconnect;
die sprintf( "Error: Unable to upload file '%s' to server (ID: %s)\n", $filename, $destination_server_id );
}
# Always disconnect when done
$file_transport->disconnect;
print STDERR "Successfully uploaded '$filename' to file transport server (ID: $destination_server_id)\n";
}
if ($delete_local_after_run) {
if ( -f $filename ) {
unless ( unlink $filename ) {
die sprintf( "Error: Unable to delete local file '%s': %s\n", $filename, $! );
}
print STDERR "Successfully deleted local file '$filename'\n";
}
}
exit;
=head1 NAME
export records - This script exports record (biblios or authorities)
=head1 SYNOPSIS
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
=head1 OPTIONS
=over
=item B<-h|--help>
Print a brief help message.
=item B<--format>
--format=FORMAT FORMAT is either 'xml', 'csv' (biblio records only) or 'marc' (default).
=item B<--date>
--date=DATETIME DATETIME should be entered as the 'dateformat' syspref is
set (dd/mm/yyyy[ hh:mm:ss] for metric, yyyy-mm-dd[ hh:mm:ss] for iso,
mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
have been modified since DATETIME.
=item B<--record-type>
--record-type=TYPE TYPE is 'bibs' or 'auths'.
=item B<--dont_export_items>
--dont_export_items If enabled, the item infos won't be exported.
=item B<--csv_profile_id>
--csv_profile_id=ID Generate a CSV file with the given CSV profile id (see tools/csv-profiles.pl)
This can only be used to export biblio records.
=item B<--deleted_barcodes>
--deleted_barcodes If used, a list of barcodes of items deleted since DATE
is produced (or from all deleted items if no date is
specified). Used only if TYPE is 'bibs'.
=item B<--clean>
--clean removes NSE/NSB.
=item B<--id_list_file>
--id_list_file=PATH PATH is a path to a file containing a list of
IDs (biblionumber or authid) with one ID per line.
This list works as a filter; it is compatible with
other parameters for selecting records.
=item B<--filename>
--filename=FILENAME FILENAME used to export the data.
=item B<--starting_authid>
--starting_authid=ID Export authorities with authid >= ID
=item B<--ending_authid>
--ending_authid=ID Export authorities with authid <= ID
=item B<--authtype>
--authtype=AUTHTYPE Export authorities from the given AUTHTYPE
=item B<--starting_biblionumber>
--starting_biblionumber=ID Export biblio with biblionumber >= ID
=item B<--ending_biblionumber>
--ending_biblionumber=ID Export biblio with biblionumber <= ID
=item B<--itemtype>
--itemtype=ITEMTYPE Export biblio from the given ITEMTYPE
=item B<--starting_callnumber>
--starting_callnumber=CALLNUMBER Export biblio with callnumber >=CALLNUMBER
=item B<--ending_callnumber>
--ending_callnumber=CALLNUMBER Export biblio with callnumber <=CALLNUMBER
=item B<--start_accession>
--starting_accession=DATE Export biblio with an item accessionned after DATE
=item B<--end_accession>
--end_accession=DATE Export biblio with an item accessionned after DATE
=item B<--marc_conditions>
--marc_conditions=CONDITIONS Only include biblios with MARC data matching CONDITIONS.
CONDITIONS is on the format: <marc_target><binary_operator><value>,
or <unary_operation>(<marc_target>).
with multiple conditions separated by commas (,).
For example: --marc_conditions="035a!=(EXAMPLE)123,041a=swe".
Multiple conditions are all required to match.
If <marc_target> has multiple values all values
are also required to match.
Valid operators are: = (equal to), != (not equal to),
> (great than) and < (less than).
Two unary operations are also supported:
exists(<marc_target>) and not_exists(<marc_target>).
For example: --marc_conditions="exists(035a)".
"exists(<marc_target)" will include marc records where
<marc_target> exists regardless of target value, and
"exists(<marc_target>)" will include marc records where
no <marc_target> exists.
=item B<--embed_see_from_headings>
--embed_see_from_headings Embed see from (non-preferred form) headings in bibliographic record.
=item B<--report_id>
--report_id=ID Export biblionumbers or authids from a given saved report output.
If you want to export authority records then your report must
select authid and you must define --record-type=auths when
running this script.
=item B<--report_param>
--report_param=PARAM Repeatable, should provide one param per param requested for the
report.
Report params are not combined as on the staff side, so you may
need to repeat params.
=item B<--destination_server_id>
--destination_server_id=ID Provide this option, along with the destination server ID, to
upload the resultant mrc file to the selected file transport server.
You can create file transport servers using the Koha staff interface,
under Administration.
=item B<--delete_local_after_run>
--delete_local_after_run Deletes the local file at the end of the script run. Can be
useful if, for example, you are uploading the file to a
file transport server.
=back
=head1 AUTHOR
Koha Development Team
=head1 COPYRIGHT
Copyright Koha Team
=head1 LICENSE
This file is part of Koha.
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <https://www.gnu.org/licenses>.
=cut
Reference in New Issue View Git Blame Copy Permalink