Skip to content

Commit 8c36fbf

Browse files
Anty0Anty0
authored
feat: import and export support for i18next (tolgee#2463)
Supported features in current implementation: - [x] Interpolation - [x] Replaceable keys - [ ] Keep unescaped flag - Ignored during import - [x] Format - [x] Number - Only without options, otherwise ignored - [ ] Currency - Ignored - [ ] Datetime - Ignored - [ ] Relativetime - Ignored - [ ] List - Ignored - [ ] Custom format functions - Ignored - [ ] Legacy format function - Ignored - [x] Plurals - [x] Ordinal plurals - [ ] Interval plurals - Imported as normal keys with the text itself containing whole structure describing the interval plural - [ ] Nesting - Ignored - [x] Context - Imported as normal keys with context as part of the key name
1 parent ca20fcd commit 8c36fbf

36 files changed

+1216
-626
lines changed

‎backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ class ExportInfoControllerTest : AbstractControllerTest() {
2626
node("defaultFileStructureTemplate")
2727
.isString.isEqualTo("{namespace}/{languageTag}.{extension}")
2828
}
29-
node("[4]") {
29+
node("[5]") {
3030
node("extension").isEqualTo("")
3131
node("mediaType").isEqualTo("")
3232
node("defaultFileStructureTemplate")

‎backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ enum class ExportFormat(
99
) {
1010
JSON("json", "application/json"),
1111
JSON_TOLGEE("json", "application/json"),
12+
JSON_I18NEXT("json", "application/json"),
1213
XLIFF("xliff", "application/x-xliff+xml"),
1314
PO("po", "text/x-gettext-translation"),
1415
APPLE_STRINGS_STRINGSDICT(

‎backend/data/src/main/kotlin/io/tolgee/formats/ExportMessageFormat.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package io.tolgee.formats
22

33
import io.tolgee.formats.paramConvertors.out.IcuToApplePlaceholderConvertor
44
import io.tolgee.formats.paramConvertors.out.IcuToCPlaceholderConvertor
5+
import io.tolgee.formats.paramConvertors.out.IcuToI18nextPlaceholderConvertor
56
import io.tolgee.formats.paramConvertors.out.IcuToJavaPlaceholderConvertor
67
import io.tolgee.formats.paramConvertors.out.IcuToPhpPlaceholderConvertor
78
import io.tolgee.formats.paramConvertors.out.IcuToRubyPlaceholderConvertor
@@ -13,6 +14,7 @@ enum class ExportMessageFormat(val paramConvertorFactory: () -> FromIcuPlacehold
1314
JAVA_STRING_FORMAT(paramConvertorFactory = { IcuToJavaPlaceholderConvertor() }),
1415
APPLE_SPRINTF(paramConvertorFactory = { IcuToApplePlaceholderConvertor() }),
1516
RUBY_SPRINTF(paramConvertorFactory = { IcuToRubyPlaceholderConvertor() }),
17+
I18NEXT(paramConvertorFactory = { IcuToI18nextPlaceholderConvertor() }),
1618
ICU(paramConvertorFactory = { IcuToIcuPlaceholderConvertor() }),
1719
// PYTHON_SPRINTF,
1820
}

‎backend/data/src/main/kotlin/io/tolgee/formats/FormsToIcuPluralConvertor.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package io.tolgee.formats
22

3+
import com.ibm.icu.text.PluralRules
4+
35
class FormsToIcuPluralConvertor(
46
val forms: Map<String, String>,
57
val argName: String = DEFAULT_PLURAL_ARGUMENT_NAME,
@@ -10,6 +12,11 @@ class FormsToIcuPluralConvertor(
1012
val newLineStringInit = if (addNewLines) "\n" else " "
1113
val icuMsg = StringBuffer("{$argName, plural,$newLineStringInit")
1214
forms.let {
15+
if (PluralRules.KEYWORD_OTHER !in it) {
16+
return@let it + (PluralRules.KEYWORD_OTHER to "")
17+
}
18+
return@let it
19+
}.let {
1320
if (optimize) {
1421
return@let optimizePluralForms(it)
1522
}

‎backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt‎

Lines changed: 35 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -13,23 +13,23 @@ class GenericStructuredProcessor(
1313
private val format: ImportFormat,
1414
) : ImportFileProcessor() {
1515
override fun process() {
16-
data.import("")
16+
var processedData = data
17+
if (format.pluralsViaSuffixesParser != null) {
18+
processedData =
19+
GenericSuffixedPluralsPreprocessor(
20+
context = context,
21+
data = data,
22+
pluralsViaSuffixesParser = format.pluralsViaSuffixesParser,
23+
).preprocess()
24+
}
25+
processedData.import("")
1726
}
1827

1928
private fun Any?.import(key: String) {
2029
// Convertor handles strings and possible nested plurals, if convertor returns null,
2130
// it means that it's not a string or nested plurals, so we need to parse it further
22-
convert(this)?.let { result ->
23-
result.forEach {
24-
context.addTranslation(
25-
key,
26-
languageTagOrGuess,
27-
it.message,
28-
rawData = this@import,
29-
convertedBy = format,
30-
pluralArgName = it.pluralArgName,
31-
)
32-
}
31+
convert(this)?.let {
32+
it.applyAll(key, this@import)
3333
return
3434
}
3535

@@ -42,17 +42,6 @@ class GenericStructuredProcessor(
4242
it.parseMap(key)
4343
return
4444
}
45-
46-
convert(this)?.firstOrNull()?.let {
47-
context.addTranslation(
48-
keyName = key,
49-
languageName = languageTagOrGuess,
50-
value = it.message,
51-
pluralArgName = it.pluralArgName,
52-
rawData = this@import,
53-
convertedBy = format,
54-
)
55-
}
5645
}
5746

5847
private fun convert(data: Any?): List<MessageConvertorResult>? {
@@ -86,6 +75,29 @@ class GenericStructuredProcessor(
8675
}
8776
}
8877

78+
private fun MessageConvertorResult.apply(
79+
key: String,
80+
rawData: Any?,
81+
) {
82+
context.addTranslation(
83+
keyName = key,
84+
languageName = languageTagOrGuess,
85+
value = message,
86+
rawData = rawData,
87+
convertedBy = format,
88+
pluralArgName = pluralArgName,
89+
)
90+
}
91+
92+
private fun List<MessageConvertorResult>.applyAll(
93+
key: String,
94+
rawData: Any?,
95+
) {
96+
forEach {
97+
it.apply(key, rawData)
98+
}
99+
}
100+
89101
private val languageTagOrGuess: String by lazy {
90102
languageTag ?: firstLanguageTagGuessOrUnknown
91103
}

‎backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,14 @@
11
package io.tolgee.formats.genericStructuredFile.`in`
22

3-
import com.ibm.icu.text.PluralRules
43
import io.tolgee.formats.MessageConvertorResult
4+
import io.tolgee.formats.allPluralKeywords
55
import io.tolgee.formats.importCommon.ImportFormat
66
import io.tolgee.formats.importCommon.unwrapString
7-
import java.util.*
87

98
class GenericStructuredRawDataToTextConvertor(
109
private val format: ImportFormat,
1110
private val languageTag: String,
1211
) : StructuredRawDataConvertor {
13-
private val availablePluralKeywords by lazy {
14-
val locale = Locale.forLanguageTag(languageTag)
15-
PluralRules.forLocale(locale).keywords.toSet()
16-
}
17-
1812
override fun convert(
1913
rawData: Any?,
2014
projectIcuPlaceholdersEnabled: Boolean,
@@ -77,11 +71,15 @@ class GenericStructuredRawDataToTextConvertor(
7771
): List<MessageConvertorResult>? {
7872
val map = rawData as? Map<*, *> ?: return null
7973

80-
if (!format.pluralsViaNesting) {
74+
if (!format.pluralsViaNesting && format.pluralsViaSuffixesParser == null) {
75+
return null
76+
}
77+
78+
if (!map.keys.all { it in allPluralKeywords }) {
8179
return null
8280
}
8381

84-
if (!map.keys.all { it in availablePluralKeywords }) {
82+
if (map.size < 2) {
8583
return null
8684
}
8785

‎backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package io.tolgee.formats.genericStructuredFile.`in`
2+
3+
import io.tolgee.formats.allPluralKeywords
4+
import io.tolgee.formats.importCommon.ParsedPluralsKey
5+
import io.tolgee.formats.importCommon.PluralsKeyParser
6+
import io.tolgee.service.dataImport.processors.FileProcessorContext
7+
8+
class GenericSuffixedPluralsPreprocessor(
9+
val context: FileProcessorContext,
10+
private val data: Any?,
11+
private val pluralsViaSuffixesParser: PluralsKeyParser,
12+
) {
13+
fun preprocess(): Any? {
14+
return data.preprocess()
15+
}
16+
17+
private fun Any?.preprocess(): Any? {
18+
if (this == null) {
19+
return null
20+
}
21+
22+
(this as? List<*>)?.let {
23+
return it.preprocessList()
24+
}
25+
26+
(this as? Map<*, *>)?.let {
27+
return it.preprocessMap()
28+
}
29+
30+
return this
31+
}
32+
33+
private fun List<*>.preprocessList(): List<*> {
34+
return this.map { it.preprocess() }
35+
}
36+
37+
private fun Any?.parsePluralsKey(keyParser: PluralsKeyParser): ParsedPluralsKey? {
38+
val key = this as? String ?: return null
39+
return keyParser.parse(key).takeIf {
40+
it.key != null && it.plural in allPluralKeywords
41+
} ?: ParsedPluralsKey(null, null, key)
42+
}
43+
44+
private fun Map<*, *>.groupByPlurals(keyParser: PluralsKeyParser): Map<String?, List<Pair<ParsedPluralsKey, Any?>>> {
45+
return this.entries.mapIndexedNotNull { idx, (key, value) ->
46+
key.parsePluralsKey(keyParser)?.let { it to value }.also {
47+
if (it == null) {
48+
context.fileEntity.addKeyIsNotStringIssue(key.toString(), idx)
49+
}
50+
}
51+
}.groupBy { (parsedKey, _) -> parsedKey.key }.toMap()
52+
}
53+
54+
private fun List<Pair<ParsedPluralsKey, Any?>>.useOriginalKey(): List<Pair<String, Any?>> {
55+
return map { (parsedKey, value) ->
56+
parsedKey.originalKey to value.preprocess()
57+
}
58+
}
59+
60+
private fun List<Pair<ParsedPluralsKey, Any?>>.usePluralsKey(commonKey: String): List<Pair<String, Any?>> {
61+
return listOf(
62+
commonKey to
63+
this.associate { (parsedKey, value) ->
64+
parsedKey.plural to value
65+
},
66+
)
67+
}
68+
69+
private fun Map<*, *>.preprocessMap(): Map<*, *> {
70+
return this.groupByPlurals(pluralsViaSuffixesParser).flatMap { (commonKey, values) ->
71+
if (commonKey == null || values.size < 2) {
72+
return@flatMap values.useOriginalKey()
73+
}
74+
return@flatMap values.usePluralsKey(commonKey)
75+
}.toMap()
76+
}
77+
}

‎backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,11 @@ class GenericStructuredFileExporter(
5555
)
5656
}
5757

58+
private val pluralsViaSuffixes
59+
get() = messageFormat == ExportMessageFormat.I18NEXT
60+
5861
private val pluralsViaNesting
59-
get() = messageFormat != ExportMessageFormat.ICU
62+
get() = !pluralsViaSuffixes && messageFormat != ExportMessageFormat.ICU
6063

6164
private val placeholderConvertorFactory
6265
get() = messageFormat.paramConvertorFactory
@@ -65,6 +68,9 @@ class GenericStructuredFileExporter(
6568
if (pluralsViaNesting) {
6669
return addNestedPlural(translation)
6770
}
71+
if (pluralsViaSuffixes) {
72+
return addSuffixedPlural(translation)
73+
}
6874
return addSingularTranslation(translation)
6975
}
7076

@@ -84,6 +90,24 @@ class GenericStructuredFileExporter(
8490
)
8591
}
8692

93+
private fun addSuffixedPlural(translation: ExportTranslationView) {
94+
val pluralForms =
95+
convertMessageForNestedPlural(translation.text) ?: let {
96+
// this should never happen, but if it does, it's better to add a null key then crash or ignore it
97+
addNullValue(translation)
98+
return
99+
}
100+
101+
val builder = getFileContentResultBuilder(translation)
102+
pluralForms.forEach { (keyword, form) ->
103+
builder.addValue(
104+
translation.languageTag,
105+
"${translation.key.name}_$keyword",
106+
form,
107+
)
108+
}
109+
}
110+
87111
private fun addNullValue(translation: ExportTranslationView) {
88112
val builder = getFileContentResultBuilder(translation)
89113
builder.addValue(

‎backend/data/src/main/kotlin/io/tolgee/formats/getGroupOrNull.kt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package io.tolgee.formats
2+
3+
fun MatchGroupCollection.getGroupOrNull(name: String): MatchGroup? {
4+
try {
5+
return this[name]
6+
} catch (e: IllegalArgumentException) {
7+
if (e.message?.contains("No group with name") != true) {
8+
throw e
9+
}
10+
return null
11+
}
12+
}

‎backend/data/src/main/kotlin/io/tolgee/formats/i18next/constants.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
package io.tolgee.formats.i18next
2+
3+
const val I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY = "_i18nextUnescapedPlaceholders"

0 commit comments

Comments
 (0)