feat: adds custom prefix support for gettext po (#2004)

Author: garikkh

packages/format-po-gettext/README.md

@@ -13,7 +13,7 @@
 
 > **Warning**
 > This formatter is made for compatibility with translation management systems, which do not support ICU expressions in PO files.
-> 
+>
 > It does not support all features of LinguiJS and should be carefully considered over other formats.
 >
 > Not supported features (native gettext doesn't support this):
@@ -72,10 +72,17 @@ export type PoGettextFormatterOptions = {
 
   /**
    * Disable warning about unsupported `Select` feature encountered in catalogs
-   * 
+   *
    * @default false
    */
   disableSelectWarning?: boolean
+
+  /**
+   * Overrides the default prefix for icu and plural comments in the final PO catalog.
+   *
+   * @default "js-lingui:"
+   */
+  customICUPrefix?: string
 }
 ```
 

packages/format-po-gettext/src/__snapshots__/po-gettext.test.ts.snap

@@ -180,3 +180,110 @@ exports[`po-gettext format should convert gettext plurals to ICU plural messages
   },
 }
 `;
+
+exports[`po-gettext format using custom prefix handles custom prefix 1`] = `
+msgid ""
+msgstr ""
+"POT-Creation-Date: 2018-08-27 10:00+0000\\n"
+"MIME-Version: 1.0\\n"
+"Content-Type: text/plain; charset=utf-8\\n"
+"Content-Transfer-Encoding: 8bit\\n"
+"X-Generator: @lingui/cli\\n"
+"Language: en\\n"
+"Project-Id-Version: \\n"
+"Report-Msgid-Bugs-To: \\n"
+"PO-Revision-Date: \\n"
+"Last-Translator: \\n"
+"Language-Team: \\n"
+"Plural-Forms: \\n"
+
+#. This is a comment by the developers about how the content must be localized.
+#. js-lingui-explicit-id
+#. custom-prefix:pluralize_on=someCount
+msgid "message_with_id"
+msgid_plural "message_with_id_plural"
+msgstr[0] "Singular case with id"
+msgstr[1] "Case number {someCount} with id"
+
+#. custom-prefix:icu=%7BanotherCount%2C+plural%2C+one+%7BSingular+case%7D+other+%7BCase+number+%7BanotherCount%7D%7D%7D&pluralize_on=anotherCount
+msgid "Singular case"
+msgid_plural "Case number {anotherCount}"
+msgstr[0] "Singular case"
+msgstr[1] "Case number {anotherCount}"
+
+`;
+
+exports[`po-gettext format using custom prefix warns and falls back to using count if prefix is not found 1`] = `
+{
+  lO3l+X: {
+    comments: [
+      js-lingui:icu=%7BanotherCount%2C+plural%2C+one+%7BSingular+case%7D+other+%7BCase+number+%7BanotherCount%7D%7D%7D&pluralize_on=anotherCount,
+    ],
+    context: null,
+    extra: {
+      flags: [],
+      translatorComments: [],
+    },
+    message: Singular case,
+    obsolete: false,
+    origin: [],
+    translation: {count, plural, one {Singular case} other {Case number {anotherCount}}},
+  },
+  maCaRp: {
+    comments: [
+      js-lingui:icu=%7Bcount%2C+plural%2C+one+%7BSingular%7D+other+%7BPlural%7D%7D&pluralize_on=count,
+    ],
+    context: null,
+    extra: {
+      flags: [],
+      translatorComments: [],
+    },
+    message: Singular,
+    obsolete: false,
+    origin: [],
+    translation: ,
+  },
+  message_with_id: {
+    comments: [
+      js-lingui:pluralize_on=someCount,
+      js-lingui-explicit-id,
+    ],
+    context: null,
+    extra: {
+      flags: [],
+      translatorComments: [],
+    },
+    obsolete: false,
+    origin: [],
+    translation: {count, plural, one {Singular case} other {Case number {someCount}}},
+  },
+  message_with_id_but_without_translation: {
+    comments: [
+      Comment made by the developers.,
+      js-lingui:pluralize_on=count,
+      js-lingui-explicit-id,
+    ],
+    context: null,
+    extra: {
+      flags: [],
+      translatorComments: [],
+    },
+    obsolete: false,
+    origin: [],
+    translation: ,
+  },
+  static: {
+    comments: [
+      js-lingui-explicit-id,
+    ],
+    context: null,
+    extra: {
+      flags: [],
+      translatorComments: [],
+    },
+    obsolete: false,
+    origin: [],
+    translation: Static message,
+  },
+}
+`;

packages/format-po-gettext/src/po-gettext.test.ts

@@ -230,4 +230,78 @@ msgstr[2] "# dní"
 
     expect(catalog).toMatchSnapshot()
   })
+
+  describe("using custom prefix", () => {
+    it("parses plurals correctly", () => {
+      const defaultProfile = fs
+        .readFileSync(path.join(__dirname, "fixtures/messages_plural.po"))
+        .toString()
+      const customProfile = defaultProfile.replace(
+        /js-lingui:/g,
+        "custom-prefix:"
+      )
+
+      const defaultPrefix = createFormat()
+      const customPrefix = createFormat({ customICUPrefix: "custom-prefix:" })
+
+      const defaultCatalog = defaultPrefix.parse(
+        defaultProfile,
+        defaultParseCtx
+      )
+      const customCatalog = customPrefix.parse(customProfile, defaultParseCtx)
+
+      expect(defaultCatalog).toEqual(customCatalog)
+    })
+
+    it("warns and falls back to using count if prefix is not found", () => {
+      const defaultProfile = fs
+        .readFileSync(path.join(__dirname, "fixtures/messages_plural.po"))
+        .toString()
+
+      const usingInvalidPrefix = createFormat({
+        customICUPrefix: "invalid-prefix:",
+      })
+      mockConsole((console) => {
+        const catalog = usingInvalidPrefix.parse(
+          defaultProfile,
+          defaultParseCtx
+        )
+        expect(console.warn).toHaveBeenCalledWith(
+          expect.stringContaining(
+            "should be stored in a comment starting with"
+          ),
+          expect.anything()
+        )
+        expect(catalog).toMatchSnapshot()
+      })
+    })
+
+    it("handles custom prefix", () => {
+      const format = createFormat({ customICUPrefix: "custom-prefix:" })
+
+      const catalog: CatalogType = {
+        message_with_id: {
+          message:
+            "{someCount, plural, one {Singular case with id\
+            and linebreak} other {Case number {someCount} with id}}",
+          translation:
+            "{someCount, plural, one {Singular case with id} other {Case number {someCount} with id}}",
+          comments: [
+            "This is a comment by the developers about how the content must be localized.",
+            "js-lingui-explicit-id",
+          ],
+        },
+        WGI12K: {
+          message:
+            "{anotherCount, plural, one {Singular case} other {Case number {anotherCount}}}",
+          translation:
+            "{anotherCount, plural, one {Singular case} other {Case number {anotherCount}}}",
+        },
+      }
+
+      const pofile = format.serialize(catalog, defaultSerializeCtx)
+
+      expect(pofile).toMatchSnapshot()
+    })
+  })
 })

packages/format-po-gettext/src/po-gettext.ts

@@ -13,6 +13,7 @@ type POItem = InstanceType<typeof PO.Item>
 
 export type PoGettextFormatterOptions = PoFormatterOptions & {
   disableSelectWarning?: boolean
+  customICUPrefix?: string
 }
 
 // Attempts to turn a single tokenized ICU plural case back into a string.
@@ -40,7 +41,7 @@ const ICU_SELECT_REGEX = /^{.*, select(Ordinal)?, .*}$/
 const LINE_ENDINGS = /\r?\n/g
 
 // Prefix that is used to identitify context information used by this module in PO's "extracted comments".
-const CTX_PREFIX = "js-lingui:"
+const DEFAULT_CTX_PREFIX = "js-lingui:"
 
 function serializePlurals(
   item: POItem,
@@ -52,6 +53,7 @@ function serializePlurals(
   // Depending on whether custom ids are used by the developer, the (potential plural) "original", untranslated ICU
   // message can be found in `message.message` or in the item's `key` itself.
   const icuMessage = message.message
+  const ctxPrefix = options.customICUPrefix || DEFAULT_CTX_PREFIX
 
   if (!icuMessage) {
     return item
@@ -99,7 +101,7 @@ function serializePlurals(
       }
 
       ctx.sort()
-      item.extractedComments.push(CTX_PREFIX + ctx.toString())
+      item.extractedComments.push(ctxPrefix + ctx.toString())
 
       // If there is a translated value, parse that instead of the original message to prevent overriding localized
       // content with the original message. If there is no translated value, don't touch msgstr, since marking item as
@@ -161,7 +163,8 @@ const getPluralCases = (lang: string): string[] | undefined => {
 const convertPluralsToICU = (
   item: POItem,
   pluralForms: string[],
-  lang: string
+  lang: string,
+  ctxPrefix: string = DEFAULT_CTX_PREFIX
 ) => {
   const translationCount = item.msgstr.length
   const messageKey = item.msgid
@@ -181,13 +184,13 @@ const convertPluralsToICU = (
   }
 
   const contextComment = item.extractedComments
-    .find((comment) => comment.startsWith(CTX_PREFIX))
-    ?.substr(CTX_PREFIX.length)
+    .find((comment) => comment.startsWith(ctxPrefix))
+    ?.substring(ctxPrefix.length)
   const ctx = new URLSearchParams(contextComment)
 
   if (contextComment != null) {
     item.extractedComments = item.extractedComments.filter(
-      (comment) => !comment.startsWith(CTX_PREFIX)
+      (comment) => !comment.startsWith(ctxPrefix)
     )
   }
 
@@ -229,7 +232,7 @@ const convertPluralsToICU = (
   let pluralizeOn = ctx.get("pluralize_on")
   if (!pluralizeOn) {
     console.warn(
-      `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${CTX_PREFIX}"), assuming "count".`,
+      `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${ctxPrefix}"), assuming "count".`,
       messageKey
     )
     pluralizeOn = "count"
@@ -262,7 +265,12 @@ export function formatter(
       let pluralForms = getPluralCases(po.headers.Language)
 
       po.items.forEach((item) => {
-        convertPluralsToICU(item, pluralForms, po.headers.Language)
+        convertPluralsToICU(
+          item,
+          pluralForms,
+          po.headers.Language,
+          options.customICUPrefix
+        )
       })
 
       return formatter.parse(po.toString(), ctx) as CatalogType

website/docs/ref/catalog-formats.md

@@ -146,7 +146,7 @@ export default {
 
 ### Configuration {#po-gettext-configuration}
 
-PO Gettext formatter accepts the following options:
+The PO Gettext formatter accepts the following options:
 
 ```ts
 export type PoGettextFormatterOptions = {
@@ -170,6 +170,13 @@ export type PoGettextFormatterOptions = {
    * @default false
    */
   disableSelectWarning?: boolean;
+
+  /**
+   * Overrides the default prefix for icu and plural comments in the final PO catalog.
+   *
+   * @default "js-lingui:"
+   */
+  customICUPrefix?: string;
 };
 ```
 
@@ -203,6 +210,19 @@ With this format, plural messages are exported in the following ways, depending
 
   Note how `msgid` and `msgid_plural` were extracted from the original message.
 
+- Message **with a custom comment prefix**.
+
+  Some TMS might modify the ICU comment by attempting to split lines to be 80 characters or less, or have trouble reading lingui comments because of the `js-lingui:` prefix. To change the prefix, set `customICUPrefix` to modify the prefix for ICU comments.
+
+  ```po
+  # with default prefix
+  #. js-
+  #. lingui:icu=%7BanotherCount%2C+plural%2C+one+%7BSingular+case%7D+other+%7BCase+number+%7BanotherCount%7D%7D%7D&pluralize_on=anotherCount
+
+  # customICUPrefix = jsi18n:
+  #. jsi18n:icu=%7BanotherCount%2C+plural%2C+one+%7BSingular+case%7D+other+%7BCase+number+%7BanotherCount%7D%7D%7D&pluralize_on=anotherCount
+  ```
+
 ### Limitations {#po-gettext-limitations}
 
 This format comes with several caveats and should only be used when using ICU plurals in PO files is not an option: