Bug 21156: Add plural translation capabilities to JS files

It adds Javascript equivalent of Koha::I18N's exported subroutines, and
they are used the same way.

String extraction is done only on *.js files and require gettext 0.19
(available in Debian jessie, and also in wheezy-backports)

It adds Javascript library Gettext.js for handling translation and a
Perl script po2json to transform PO file into JSON.

Gettext.js and po2json both come from Locale::Simple.
There are several tools named po2json. It's simpler to integrate this
one into Koha than to check if the good one is installed on the system.
Locale::Simple is not needed.

To avoid polluting the global namespace too much, this patch also
introduce a global JS object named Koha and add some stuff in Koha.i18n

Test plan:
1. Add a translatable string in a JS file. For example, add this:
     alert(__nx("There is one item", "There are {count} items", 3,
     {count: 3}));
   to staff-global.js
2. cd misc/translator && ./translate update fr-FR
3. Open misc/translator/po/fr-FR-messages-js.po, verify that your
   string is present, and translate it
4. cd misc/translator && ./translate install fr-FR
5. (Optional) Verify that
   koha-tmpl/intranet-tmpl/prog/fr-FR/js/locale_data.js exists and
   contains your translation
6. Open your browser on the staff main page, change language and verify
   that the message is translated
7. Repeat 1-6 on OPAC side

Signed-off-by: Martin Renvoize <martin.renvoize@ptfs-europe.com>
Signed-off-by: Bernardo Gonzalez Kriegel <bgkriegel@gmail.com>
Works well, translation is OK and test message is displayed correctly.
Current qa-tool error is a false positive.
Signed-off-by: Martin Renvoize <martin.renvoize@ptfs-europe.com>

C4/Installer/PerlDependencies.pm

@@ -142,7 +142,7 @@ our $PERL_DEPS = {
     'Locale::PO' => {
         'usage'    => 'Core',
         'required' => '1',
-        'min_ver'  => '0.17'
+        'min_ver'  => '0.24'
     },
     'LWP::Simple' => {
         'usage'    => 'Core',

koha-tmpl/intranet-tmpl/js/i18n.js

@@ -0,0 +1,51 @@
+(function() {
+  var params = {
+    "domain": "Koha"
+  };
+  if (typeof json_locale_data !== 'undefined') {
+    params.locale_data = json_locale_data;
+  }
+
+  Koha.i18n = {
+    gt: new Gettext(params),
+
+    expand: function(text, vars) {
+      var replace_callback = function(match, name) {
+        return name in vars ? vars[name] : match;
+      };
+      return text.replace(/\{(.*?)\}/g, replace_callback);
+    }
+  };
+})();
+
+function __(msgid) {
+  return Koha.i18n.gt.gettext(msgid);
+}
+
+function __x(msgid, vars) {
+  return Koha.i18n.expand(__(msgid), vars);
+}
+
+function __n(msgid, msgid_plural, count) {
+  return Koha.i18n.gt.ngettext(msgid, msgid_plural, count);
+}
+
+function __nx(msgid, msgid_plural, count, vars) {
+  return Koha.i18n.expand(__n(msgid, msgid_plural, count), vars);
+}
+
+function __p(msgctxt, msgid) {
+  return Koha.i18n.gt.pgettext(msgctxt, msgid);
+}
+
+function __px(msgctxt, msgid, vars) {
+  return Koha.i18n.expand(__p(msgctxt, msgid), vars);
+}
+
+function __np(msgctxt, msgid, msgid_plural, count) {
+  return Koha.i18n.gt.npgettext(msgctxt, msgid, msgid_plural, count);
+}
+
+function __npx(msgctxt, msgid, msgid_plural, count, vars) {
+  return Koha.i18n.expand(__np(msgctxt, msgid, msgid_plural, count), vars);
+}

koha-tmpl/intranet-tmpl/prog/en/includes/doc-head-close.inc

@@ -22,6 +22,14 @@
 [% INCLUDE intranetstylesheet.inc %]
 [% IF ( bidi ) %][% Asset.css("css/right-to-left.css") | $raw %][% END %]
 
+<script>
+var Koha = {};
+</script>
+
+<script src="[% themelang %]/js/locale_data.js"></script>
+<script src="[% interface %]/js/Gettext.js"></script>
+<script src="[% interface %]/js/i18n.js"></script>
+
 [% IF ( login ) %]
     [% Asset.css("css/login.css") | $raw %]
 [% END %]

koha-tmpl/opac-tmpl/bootstrap/js/i18n.js

@@ -0,0 +1,51 @@
+(function() {
+  var params = {
+    "domain": "Koha"
+  };
+  if (typeof json_locale_data !== 'undefined') {
+    params.locale_data = json_locale_data;
+  }
+
+  Koha.i18n = {
+    gt: new Gettext(params),
+
+    expand: function(text, vars) {
+      var replace_callback = function(match, name) {
+        return name in vars ? vars[name] : match;
+      };
+      return text.replace(/\{(.*?)\}/g, replace_callback);
+    }
+  };
+})();
+
+function __(msgid) {
+  return Koha.i18n.gt.gettext(msgid);
+}
+
+function __x(msgid, vars) {
+  return Koha.i18n.expand(__(msgid), vars);
+}
+
+function __n(msgid, msgid_plural, count) {
+  return Koha.i18n.gt.ngettext(msgid, msgid_plural, count);
+}
+
+function __nx(msgid, msgid_plural, count, vars) {
+  return Koha.i18n.expand(__n(msgid, msgid_plural, count), vars);
+}
+
+function __p(msgctxt, msgid) {
+  return Koha.i18n.gt.pgettext(msgctxt, msgid);
+}
+
+function __px(msgctxt, msgid, vars) {
+  return Koha.i18n.expand(__p(msgctxt, msgid), vars);
+}
+
+function __np(msgctxt, msgid, msgid_plural, count) {
+  return Koha.i18n.gt.npgettext(msgctxt, msgid, msgid_plural, count);
+}
+
+function __npx(msgctxt, msgid, msgid_plural, count, vars) {
+  return Koha.i18n.expand(__np(msgctxt, msgid, msgid_plural, count), vars);
+}

misc/translator/LangInstaller.pm

@@ -81,6 +81,7 @@ sub new {
     $self->{msginit}         = `which msginit`;
     $self->{xgettext}        = `which xgettext`;
     $self->{sed}             = `which sed`;
+    $self->{po2json}         = "$Bin/po2json";
     chomp $self->{cp};
     chomp $self->{msgmerge};
     chomp $self->{msgfmt};
@@ -486,8 +487,10 @@ sub create_messages {
 
     my $pot = "$Bin/$self->{domain}.pot";
     my $po = "$self->{path_po}/$self->{lang}-messages.po";
+    my $js_pot = "$self->{domain}-js.pot";
+    my $js_po = "$self->{path_po}/$self->{lang}-messages-js.po";
 
-    unless ( -f $pot ) {
+    unless ( -f $pot && -f $js_pot ) {
         $self->extract_messages();
     }
 
@@ -495,10 +498,13 @@ sub create_messages {
     my $locale = $self->locale_name();
     system "$self->{msginit} -i $pot -o $po -l $locale --no-translator 2> /dev/null";
     warn "Problems creating $pot ".$? if ( $? == -1 );
+    system "$self->{msginit} -i $js_pot -o $js_po -l $locale --no-translator 2> /dev/null";
+    warn "Problems creating $js_pot ".$? if ( $? == -1 );
 
     # If msginit failed to correctly set Plural-Forms, set a default one
-    system "$self->{sed} --in-place $po "
-        . "--expression='s/Plural-Forms: nplurals=INTEGER; plural=EXPRESSION/Plural-Forms: nplurals=2; plural=(n != 1)/'";
+    system "$self->{sed} --in-place "
+        . "--expression='s/Plural-Forms: nplurals=INTEGER; plural=EXPRESSION/Plural-Forms: nplurals=2; plural=(n != 1)/' "
+        . "$po $js_po";
 }
 
 sub update_messages {
@@ -506,14 +512,17 @@ sub update_messages {
 
     my $pot = "$Bin/$self->{domain}.pot";
     my $po = "$self->{path_po}/$self->{lang}-messages.po";
+    my $js_pot = "$self->{domain}-js.pot";
+    my $js_po = "$self->{path_po}/$self->{lang}-messages-js.po";
 
-    unless ( -f $pot ) {
+    unless ( -f $pot && -f $js_pot ) {
         $self->extract_messages();
     }
 
-    if ( -f $po ) {
+    if ( -f $po && -f $js_pot ) {
         say "Update messages ($self->{lang})" if $self->{verbose};
         system "$self->{msgmerge} --backup=off --quiet -U $po $pot";
+        system "$self->{msgmerge} --backup=off --quiet -U $js_po $js_pot";
     } else {
         $self->create_messages();
     }
@@ -674,11 +683,12 @@ sub extract_messages {
 
     push @files_to_scan, @tt_files;
 
-    my $xgettext_cmd = "$self->{xgettext} --force-po -L Perl --from-code=UTF-8 "
+    my $xgettext_common_args = "--force-po --from-code=UTF-8 "
         . "--package-name=Koha --package-version='' "
         . "-k -k__ -k__x -k__n:1,2 -k__nx:1,2 -k__xn:1,2 -k__p:1c,2 "
         . "-k__px:1c,2 -k__np:1c,2,3 -k__npx:1c,2,3 -kN__ -kN__n:1,2 "
-        . "-kN__p:1c,2 -kN__np:1c,2,3 "
+        . "-kN__p:1c,2 -kN__np:1c,2,3 ";
+    my $xgettext_cmd = "$self->{xgettext} -L Perl $xgettext_common_args "
         . "-o $Bin/$self->{domain}.pot -D $tempdir -D $basedir";
     $xgettext_cmd .= " $_" foreach (@files_to_scan);
 
@@ -686,9 +696,31 @@ sub extract_messages {
         die "system call failed: $xgettext_cmd";
     }
 
+    my @js_dirs = (
+        "$intranetdir/koha-tmpl/intranet-tmpl/prog/js",
+        "$intranetdir/koha-tmpl/opac-tmpl/bootstrap/js",
+    );
+
+    my @js_files;
+    find(sub {
+        if ($_ =~ m/\.js$/) {
+            my $filename = $File::Find::name;
+            $filename =~ s|^$intranetdir/||;
+            push @js_files, $filename;
+        }
+    }, @js_dirs);
+
+    $xgettext_cmd = "$self->{xgettext} -L JavaScript $xgettext_common_args "
+        . "-o $Bin/$self->{domain}-js.pot -D $intranetdir";
+    $xgettext_cmd .= " $_" foreach (@js_files);
+
+    if (system($xgettext_cmd) != 0) {
+        die "system call failed: $xgettext_cmd";
+    }
+
     my $replace_charset_cmd = "$self->{sed} --in-place " .
-        "$Bin/$self->{domain}.pot " .
-        "--expression='s/charset=CHARSET/charset=UTF-8/'";
+        "--expression='s/charset=CHARSET/charset=UTF-8/' " .
+        "$Bin/$self->{domain}.pot $Bin/$self->{domain}-js.pot";
     if (system($replace_charset_cmd) != 0) {
         die "system call failed: $replace_charset_cmd";
     }
@@ -701,19 +733,37 @@ sub install_messages {
     my $modir = "$self->{path_po}/$locale/LC_MESSAGES";
     my $pofile = "$self->{path_po}/$self->{lang}-messages.po";
     my $mofile = "$modir/$self->{domain}.mo";
+    my $js_pofile = "$self->{path_po}/$self->{lang}-messages-js.po";
 
-    if ( not -f $pofile ) {
+    unless ( -f $pofile && -f $js_pofile ) {
         $self->create_messages();
     }
     say "Install messages ($locale)" if $self->{verbose};
     make_path($modir);
     system "$self->{msgfmt} -o $mofile $pofile";
+
+    my $js_locale_data = 'var json_locale_data = {"Koha":' . `$self->{po2json} $js_pofile` . '};';
+    my $progdir = $self->{context}->config('intrahtdocs') . '/prog';
+    mkdir "$progdir/$self->{lang}/js";
+    open my $fh, '>', "$progdir/$self->{lang}/js/locale_data.js";
+    print $fh $js_locale_data;
+    close $fh;
+
+    my $opachtdocs = $self->{context}->config('opachtdocs');
+    opendir(my $dh, $opachtdocs);
+    for my $theme ( grep { not /^\.|lib|xslt/ } readdir($dh) ) {
+        mkdir "$opachtdocs/$theme/$self->{lang}/js";
+        open my $fh, '>', "$opachtdocs/$theme/$self->{lang}/js/locale_data.js";
+        print $fh $js_locale_data;
+        close $fh;
+    }
 }
 
 sub remove_pot {
     my $self = shift;
 
     unlink "$Bin/$self->{domain}.pot";
+    unlink "$Bin/$self->{domain}-js.pot";
 }
 
 sub install {

misc/translator/po2json

@@ -0,0 +1,249 @@
+#!/usr/bin/env perl
+# PODNAME: po2json
+# ABSTRACT: Command line tool for converting a po file into a Gettext.js compatible json dataset
+
+# Copyright (C) 2008, Joshua I. Miller E<lt>unrtst@cpan.orgE<gt>, all
+# rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Library General Public License as published
+# by the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Library General Public License for more details.
+#
+# You should have received a copy of the GNU Library General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+# USA.
+
+
+use strict;
+use JSON 2.53;
+use Locale::PO 0.24;
+use File::Basename qw(basename);
+
+my $gettext_context_glue = "\004";
+
+sub usage {
+    return "$0 {-p} {file.po} > {outputfile.json}
+    -p  : do pretty-printing of json data\n";
+}
+
+&main;
+
+sub main
+{
+    my ($src_fh, $src);
+
+    my $pretty = 0;
+    if ($ARGV[0] =~ /^--?p$/) {
+        shift @ARGV;
+        $pretty = 1;
+    }
+
+    if (length($ARGV[0]))
+    {
+        if ($ARGV[0] =~ /^-h/) {
+            print &usage;
+            exit 1;
+        }
+
+        unless (-r $ARGV[0]) {
+            print "ERROR: Unable to read file [$ARGV[0]]\n";
+            die &usage;
+        }
+
+        $src = $ARGV[0];
+    } else {
+        die &usage;
+    }
+
+    # we'll be building this data struct
+    my $json = {};
+
+    my $plural_form_count;
+    # get po object stack
+    my $pos = Locale::PO->load_file_asarray($src) or die "Can't parse po file [$src].";
+
+
+    foreach my $po (@$pos)
+    {
+        my $qmsgid1 = $po->msgid;
+        my $msgid1 = $po->dequote( $qmsgid1 );
+
+        # on the header
+        if (length($msgid1) == 0)
+        {
+            my $qmsgstr = $po->msgstr;
+            my $cur = $po->dequote( $qmsgstr );
+            my %cur;
+            foreach my $h (split(/\n/, $cur))
+            {
+                next unless length($h);
+                my @h = split(':', $h, 2);
+
+                if (length($cur{$h[0]})) {
+                    warn "SKIPPING DUPLICATE HEADER LINE: $h\n";
+                } elsif ($h[0] =~ /#-#-#-#-#/) {
+                    warn "SKIPPING ERROR MARKER IN HEADER: $h\n";
+                } elsif (@h == 2) {
+                    $cur{$h[0]} = $h[1];
+                } else {
+                    warn "PROBLEM LINE IN HEADER: $h\n";
+                    $cur{$h} = '';
+                }
+            }
+
+            # init header ref
+            $$json{''} ||= {};
+
+            # populate header ref
+            foreach my $key (keys %cur) {
+                $$json{''}{$key} = length($cur{$key}) ? $cur{$key} : '';
+            }
+
+            # save plural form count
+            if ($$json{''}{'Plural-Forms'}) {
+                my $t = $$json{''}{'Plural-Forms'};
+                $t =~ s/^\s*//;
+                if ($t =~ /nplurals=(\d+)/) {
+                    $plural_form_count = $1;
+                } else {
+                    die "ERROR parsing plural forms header [$t]\n";
+                }
+            } else {
+                warn "NO PLURAL FORM HEADER FOUND - DEFAULTING TO 2\n";
+                # just default to 2
+                $plural_form_count = 2;
+            }
+
+        # on a normal msgid
+        } else {
+            my $qmsgctxt = $po->msgctxt;
+            my $msgctxt = $po->dequote($qmsgctxt) if $qmsgctxt;
+
+            # build the new msgid key
+            my $msg_ctxt_id = defined($msgctxt) ? join($gettext_context_glue, ($msgctxt, $msgid1)) : $msgid1;
+
+            # build translation side
+            my @trans;
+
+            # msgid plural side
+            my $qmsgid_plural = $po->msgid_plural;
+            my $msgid2 = $po->dequote( $qmsgid_plural ) if $qmsgid_plural;
+            push(@trans, $msgid2);
+
+            # translated string
+            # this shows up different if we're plural
+            if (defined($msgid2) && length($msgid2))
+            {
+                my $plurals = $po->msgstr_n;
+                for (my $i=0; $i<$plural_form_count; $i++)
+                {
+                    my $qstr = ref($plurals) ? $$plurals{$i} : undef;
+                    my $str  = $po->dequote( $qstr ) if $qstr;
+                    push(@trans, $str);
+                }
+
+            # singular
+            } else {
+                my $qmsgstr = $po->msgstr;
+                my $msgstr = $po->dequote( $qmsgstr ) if $qmsgstr;
+                push(@trans, $msgstr);
+            }
+
+            $$json{$msg_ctxt_id} = \@trans;
+        }
+    }
+
+
+    my $jsonobj = new JSON;
+    my $basename = basename($src);
+    $basename =~ s/\.pot?$//;
+    if ($pretty)
+    {
+        print $jsonobj->pretty->encode( { $basename => $json });
+    } else {
+        print $jsonobj->encode($json);
+    }
+}
+
+__END__
+
+=pod
+
+=head1 NAME
+
+po2json - Command line tool for converting a po file into a Gettext.js compatible json dataset
+
+=head1 VERSION
+
+version 0.019
+
+=head1 SYNOPSIS
+
+ po2json /path/to/domain.po > domain.json
+
+=head1 DESCRIPTION
+
+This takes a PO file, as is created from GNU Gettext's xgettext, and converts it into a JSON file.
+
+The output is an annonymous associative array. So, if you plan to load this via a <script> tag, more processing will be require (the output from this program must be assigned to a named javascript variable). For example:
+
+    echo -n "var json_locale_data = " > domain.json
+    po2json /path/to/domain.po >> domain.json
+    echo ";" >> domain.json
+
+=head1 NAME
+
+po2json - Convert a Uniforum format portable object file to javascript object notation.
+
+=head1 OPTIONS
+
+ -p : pretty-print the output. Makes the output more human-readable.
+
+=head1 BUGS
+
+Locale::PO has a potential bug (I don't know if this actually causes a problem or not). Given a .po file with an entry like:
+
+    msgid ""
+    "some string"
+    msgstr ""
+
+When $po->dump is run on that entry, it will output:
+
+    msgid "some string"
+    msgstr ""
+
+The above is removing the first linebreak. I don't know if that is significant. If so, we'll have to rewrite using a different parser (or include our own parser).
+
+=head1 REQUIRES
+
+ Locale::PO
+ JSON
+
+=head1 SEE ALSO
+
+ Locale::PO
+ Gettext.js
+
+=head1 AUTHOR
+
+Copyright (C) 2008, Joshua I. Miller E<lt>unrtst@cpan.orgE<gt>, all rights reserved. See the source code for details.
+
+=head1 AUTHOR
+
+Torsten Raudssus <torsten@raudss.us>
+
+=head1 COPYRIGHT AND LICENSE
+
+This software is copyright (c) 2012 by DuckDuckGo, Inc. L<http://duckduckgo.com/>, Torsten Raudssus <torsten@raudss.us>.
+
+This is free software; you can redistribute it and/or modify it under
+the same terms as the Perl 5 programming language system itself.
+
+=cut