Allow translating from javascript files

Dockerfile

@@ -28,7 +28,9 @@ FROM static-deps AS static
 # small amount of copying when only `webpack.config.js` is modified.
 COPY warehouse/static/ /opt/warehouse/src/warehouse/static/
 COPY warehouse/admin/static/ /opt/warehouse/src/warehouse/admin/static/
+COPY warehouse/locale/ /opt/warehouse/src/warehouse/locale/
 COPY webpack.config.js /opt/warehouse/src/
+COPY webpack.plugin.localize.js /opt/warehouse/src/
 
 RUN NODE_ENV=production npm run build
 

Makefile

@@ -104,7 +104,7 @@ licenses: .state/docker-build-base
 deps: .state/docker-build-base
 	docker compose run --rm base bin/deps
 
-translations: .state/docker-build-base
+translations: .state/docker-build-base .state/docker-build-static
 	docker compose run --rm base bin/translations
 
 requirements/%.txt: requirements/%.in

babel.cfg

@@ -3,3 +3,6 @@
 encoding = utf-8
 extensions=warehouse.utils.html:ClientSideIncludeExtension,warehouse.i18n.extensions.TrimmedTranslatableTagsExtension
 silent=False
+[javascript: **.js]
+encoding=utf-8
+silent=False

bin/translations

@@ -11,3 +11,4 @@ export LANG="${ENCODING:-en_US.UTF-8}"
 set -x
 
 make -C warehouse/locale/ translations
+python bin/translations_json.py

bin/translations_json.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pathlib
+import json
+import polib
+
+from warehouse.i18n import KNOWN_LOCALES
+
+"""
+Extracts the translations required in javascript to per-locale messages.json files.
+"""
+
+domain = "messages"
+localedir = "warehouse/locale"
+languages = [locale for locale in KNOWN_LOCALES]
+cwd = pathlib.Path().cwd()
+print("\nCreating messages.json files\n")
+
+# look in each language file that is used by the app
+for lang in languages:
+    # read the .po file to find any .js file messages
+    entries = []
+    include_next = False
+
+    po_path = cwd.joinpath(localedir, lang, 'LC_MESSAGES', 'messages.po')
+    if not po_path.exists():
+        continue
+    po = polib.pofile(po_path)
+    for entry in po.translated_entries():
+        occurs_in_js = any(o.endswith('.js') for o, _ in entry.occurrences)
+        if occurs_in_js:
+            entries.append(entry)
+
+    # if one or more translation messages from javascript files were found,
+    # then write the json file to the same folder.
+    result = {
+        "": {
+            "language": lang,
+            "plural-forms": po.metadata['Plural-Forms'],
+        }
+    }
+    for e in entries:
+        if e.msgid_plural:
+            result[e.msgid] = list(e.msgstr_plural.values())
+        elif e.msgstr:
+            result[e.msgid] = e.msgstr
+        else:
+            raise ValueError(f"No value available for ${e}")
+
+    json_path = po_path.with_suffix('.json')
+    with json_path.open('w') as f:
+        print(f"Writing messages to {json_path}")
+        json.dump(result, f)

docker-compose.yml

@@ -219,6 +219,7 @@ services:
     volumes:
       - ./warehouse:/opt/warehouse/src/warehouse:z
       - ./webpack.config.js:/opt/warehouse/src/webpack.config.js:z
+      - ./webpack.plugin.localize.js:/opt/warehouse/src/webpack.plugin.localize.js:z
       - ./.babelrc:/opt/warehouse/src/.babelrc:z
       - ./.stylelintrc.json:/opt/warehouse/src/.stylelintrc.json:z
       - ./tests/frontend:/opt/warehouse/src/tests/frontend:z

docs/dev/development/frontend.rst

@@ -151,3 +151,25 @@ One of these blocks provides code syntax highlighting, which can be tested with
 reference project provided at `<http://localhost/project/pypi-code-highlighting-demo/>`_
 when using development database. Source reStructuredText file is available
 `here <https://github.com/evemorgen/pypi-code-highlighting-demo>`_.
+
+
+Javascript localization support
+-------------------------------
+
+Strings in JS can be translated, see the see the :doc:`../translations` docs.
+
+As part of the webpack build,
+the translation data for each locale in ``KNOWN_LOCALES``
+is placed in |warehouse/static/js/warehouse/utils/messages-access.js|_.
+
+A separate js bundle is generated for each locale,
+named like this: ``warehouse.[locale].[contenthash].js``.
+
+The JS bundle to include is selected in |warehouse/templates/base.html|_
+using the current :code:`request.localizer.locale_name`.
+
+.. |warehouse/static/js/warehouse/utils/messages-access.js| replace:: ``warehouse/static/js/warehouse/utils/messages-access.js``
+.. _warehouse/static/js/warehouse/utils/messages-access.js: https://github.com/pypi/warehouse/blob/main/warehouse/static/js/warehouse/utils/messages-access.js
+
+.. |warehouse/templates/base.html| replace:: ``warehouse/templates/base.html``
+.. _warehouse/templates/base.html: https://github.com/pypi/warehouse/blob/main/warehouse/templates/base.html

docs/dev/development/reviewing-patches.rst

@@ -148,8 +148,8 @@ Merge requirements
   backwards incompatible release of a dependency) no pull requests may be
   merged until this is rectified.
 * All merged patches must have 100% test coverage.
-* All user facing strings must be marked for translation and the ``.pot`` and
-  ``.po`` files must be updated.
+* All user facing strings must be marked for translation and the ``.pot``,
+  ``.po``, and ``.json`` files must be updated.
 
 .. _`excellent to one another`: https://speakerdeck.com/ohrite/better-code-review
 

docs/dev/translations.rst

@@ -62,11 +62,23 @@ In Python, given a request context, call :code:`request._(message)` to mark
    from warehouse.i18n import localize as _
    message = _("Your message here.")
 
+In javascript, use :code:`gettext("singular", ...placeholder_values)` and
+:code:`ngettext("singular", "plural", count, ...placeholder_values)`.
+The function names are important because they need to be recognised by pybabel.
+
+.. code-block:: javascript
+
+   import { gettext, ngettext } from "../utils/messages-access";
+   gettext("Get some fruit");
+   // -> (en) "Get some fruit"
+   ngettext("Yesterday", "In the past", numDays);
+   // -> (en) numDays is 1: "Yesterday"; numDays is 3: "In the past"
+
 
 Passing non-translatable values to translated strings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To pass values you don't want to be translated into
+In html, to pass values you don't want to be translated into
 translated strings, define them inside the :code:`{% trans %}` tag.
 For example, to pass a non-translatable link
 :code:`request.route_path('classifiers')` into a string, instead of
@@ -86,6 +98,15 @@ Instead, define it inside the :code:`{% trans %}` tag:
       Filter by <a href="{{ href }}">classifier</a>
       {% endtrans %}
 
+In javascript, use :code:`%1`, :code:`%2`, etc as
+placeholders and provide the placeholder values:
+
+.. code-block:: javascript
+
+   import { ngettext } from "../utils/messages-access";
+   ngettext("Yesterday", "About %1 days ago", numDays, numDays);
+   // -> (en) numDays is 1: "Yesterday"; numDays is 3: "About 3 days ago"
+
 
 Marking new strings for pluralization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -105,6 +126,8 @@ variants of a string, for example:
 
 This is not yet directly possible in Python for Warehouse.
 
+In javascript, use :code:`ngettext()` as described above.
+
 Marking views as translatable
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

requirements/dev.txt

@@ -3,4 +3,5 @@ hupper>=1.9
 pip-tools>=1.0
 pyramid_debugtoolbar>=2.5
 pip-api
-watchdog
+polib
+watchdog

tests/frontend/messages_access_test.js

@@ -0,0 +1,109 @@
+/* Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* global expect, describe, it */
+
+import {gettext, ngettext, ngettextCustom} from "../../warehouse/static/js/warehouse/utils/messages-access";
+
+
+describe("messages access util", () => {
+
+  describe("gettext with defaults", () => {
+    it("uses default singular when no translation is available", async () => {
+      const singular = "My default message.";
+      const result = gettext(singular);
+      expect(result).toEqual(singular);
+    });
+    it("inserts placeholders into the default singular", async () => {
+      const singular = "My default message: %1";
+      const extras = ["more message here"];
+      const result = gettext(singular, ...extras);
+      expect(result).toEqual("My default message: more message here");
+    });
+  });
+
+  describe("ngettext with defaults", () => {
+    it("uses default singular when no translation is available", async () => {
+      const singular = "My default message.";
+      const plural = "My default messages.";
+      const num = 1;
+      const result = ngettext(singular, plural, num);
+      expect(result).toEqual(singular);
+    });
+    it("inserts placeholders into the default singular", async () => {
+      const singular = "My %2 default %1 message.";
+      const plural = "My default messages.";
+      const num = 1;
+      const extras = ["more message here", "something else"];
+      const result = ngettext(singular, plural, num, ...extras);
+      expect(result).toEqual("My something else default more message here message.");
+    });
+    it("uses default plural when no translation is available", async () => {
+      const singular = "My default message.";
+      const plural = "My default messages.";
+      const num = 2;
+      const result = ngettext(singular, plural, num);
+      expect(result).toEqual(plural);
+    });
+    it("inserts placeholders into the default plural", async () => {
+      const singular = "My %2 default %1 message.";
+      const plural = "My default plural messages %1 %2.";
+      const num = 2;
+      const extras = ["more message here", "something else"];
+      const result = ngettext(singular, plural, num, ...extras);
+      expect(result).toEqual("My default plural messages more message here something else.");
+    });
+  });
+
+  describe("with translation data", () => {
+    const data = {
+      "": {"language": "fr", "plural-forms": "nplurals=2; plural=n > 1;"},
+      "My default message.": "My translated message.",
+      "My %2 message with placeholders %1.": "My translated %1 message with placeholders %2",
+      "My message with plurals": ["My translated message 1.", "My translated messages 2."],
+      "My message with plurals %1 again": ["My translated message 1 %1.", "My translated message 2 %1"],
+    };
+    const pluralForms = function (n) {
+      let nplurals, plural;
+      nplurals = 2; plural = n > 1;
+      return {total: nplurals, index: ((nplurals > 1 && plural === true) ? 1 : (plural ? plural : 0))};
+    };
+    it("uses singular when translation is available", async () => {
+      const singular = "My default message.";
+      const result = ngettextCustom(singular, null, 1, [], data, pluralForms);
+      expect(result).toEqual("My translated message.");
+    });
+    it("inserts placeholders into the singular translation", async () => {
+      const singular = "My %2 message with placeholders %1.";
+      const extras = ["more message here", "another"];
+      const result = ngettextCustom(singular, null, 1, extras, data, pluralForms);
+      expect(result).toEqual("My translated more message here message with placeholders another");
+    });
+    it("uses plural when translation is available", async () => {
+      const singular = "My message with plurals";
+      const plural = "My messages with plurals";
+      const num = 2;
+      const extras = ["not used"];
+      const result = ngettextCustom(singular, plural, num, extras, data, pluralForms);
+      expect(result).toEqual("My translated messages 2.");
+    });
+    it("inserts placeholders into the plural translation", async () => {
+      const singular = "My message with plurals %1 again";
+      const plural = "My messages with plurals %1 again";
+      const num = 2;
+      const extras = ["more message here"];
+      const result = ngettextCustom(singular, plural, num, extras, data, pluralForms);
+      expect(result).toEqual("My translated message 2 more message here");
+    });
+  });
+});

tests/frontend/password_strength_gauge_controller_test.js

@@ -37,10 +37,13 @@ describe("Password strength gauge controller", () => {
     application.register("password-strength-gauge", PasswordStrengthGaugeController);
   });
 
-
   describe("initial state", () => {
     describe("the password strength gauge and screen reader text", () => {
-      it("are at 0 level and reading a password empty text", () => {
+      it("are at 0 level and reading a password empty text", async () => {
+
+        const passwordTarget = getByPlaceholderText(document.body, "Your password");
+        fireEvent.input(passwordTarget, { target: { value: "" } });
+
         const gauge = document.getElementById("gauge");
         const ZXCVBN_LEVELS = [0, 1, 2, 3, 4];
         ZXCVBN_LEVELS.forEach(i =>
@@ -74,15 +77,16 @@ describe("Password strength gauge controller", () => {
     });
 
     describe("that are strong", () => {
-      it("show high score and suggestions on screen reader", () => {
-        window.zxcvbn = jest.fn(() => {
+      it("show high score and suggestions on screen reader", async () => {
+        window.zxcvbn = jest.fn( () => {
           return {
             score: 5,
             feedback: {
               suggestions: [],
             },
           };
         });
+
         const passwordTarget = getByPlaceholderText(document.body, "Your password");
         fireEvent.input(passwordTarget, { target: { value: "the strongest password ever" } });