Skip to content

Commit 4f27075

Browse files
santysisisantysisi
committed
[Translation][Lokalise] fix "Project too big for sync export"
1 parent 4433ffc commit 4f27075

File tree

3 files changed

+203
-0
lines changed

3 files changed

+203
-0
lines changed

‎src/Symfony/Component/Translation/Bridge/Lokalise/LokaliseProvider.php‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace Symfony\Component\Translation\Bridge\Lokalise;
1313

1414
use Psr\Log\LoggerInterface;
15+
use Symfony\Component\Translation\Exception\LogicException;
1516
use Symfony\Component\Translation\Exception\ProviderException;
1617
use Symfony\Component\Translation\Loader\LoaderInterface;
1718
use Symfony\Component\Translation\MessageCatalogueInterface;
@@ -31,6 +32,10 @@
3132
final class LokaliseProvider implements ProviderInterface
3233
{
3334
private const LOKALISE_GET_KEYS_LIMIT = 5000;
35+
private const PROJECT_TOO_BIG_STATUS_CODE = 413;
36+
/** @var list<string> */
37+
private const FAILED_PROCESS_STATUS = ['cancelled', 'failed'];
38+
private const SUCESS_PROCESS_STATUS = 'finished';
3439

3540
private HttpClientInterface $client;
3641
private LoaderInterface $loader;
@@ -165,6 +170,13 @@ private function exportFiles(array $locales, array $domains): array
165170
}
166171

167172
if (200 !== $response->getStatusCode()) {
173+
if (($errorCode = $responseContent['error']['code'] ?? null) && self::PROJECT_TOO_BIG_STATUS_CODE === $errorCode) {
174+
if (!\extension_loaded('zip')) {
175+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s". There are too many translations. You need to enable the "zip" extension to use asynchronous export.', $response->getContent(false)), $response);
176+
}
177+
178+
return $this->exportFilesAsync($locales, $domains);
179+
}
168180
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
169181
}
170182

@@ -176,6 +188,109 @@ private function exportFiles(array $locales, array $domains): array
176188
return array_combine($reformattedLanguages, $responseContent['files']);
177189
}
178190

191+
/**
192+
* @see https://developers.lokalise.com/reference/download-files-async
193+
*/
194+
private function exportFilesAsync(array $locales, array $domains): array
195+
{
196+
$response = $this->client->request('POST', 'files/async-download', [
197+
'json' => [
198+
'format' => 'symfony_xliff',
199+
'original_filenames' => true,
200+
'filter_langs' => array_values($locales),
201+
'filter_filenames' => array_map($this->getLokaliseFilenameFromDomain(...), $domains),
202+
'export_empty_as' => 'skip',
203+
'replace_breaks' => false,
204+
],
205+
]);
206+
207+
if (200 !== $response->getStatusCode()) {
208+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
209+
}
210+
211+
$processId = $response->toArray()['process_id'];
212+
$attempt = 0;
213+
while (true) {
214+
$response = $this->client->request('GET', \sprintf('processes/%s', $processId));
215+
$process = $response->toArray()['process'];
216+
if (\in_array($process['status'], self::FAILED_PROCESS_STATUS, true)) {
217+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
218+
}
219+
if (self::SUCESS_PROCESS_STATUS === $process['status']) {
220+
$downloadUrl = $process['details']['download_url'];
221+
break;
222+
}
223+
++$attempt;
224+
usleep(500000 * $attempt);
225+
}
226+
227+
$newfile = \sprintf('%s%s%s.zip', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
228+
$extractPath = \sprintf('%s%s%s', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
229+
if (!copy($downloadUrl, $newfile)) {
230+
throw new LogicException(\sprintf('failed to copy "%s".', $downloadUrl));
231+
}
232+
233+
try {
234+
$zip = new \ZipArchive();
235+
if (!$zip->open($newfile, \ZipArchive::CREATE)) {
236+
throw new LogicException(\sprintf('failed to open zip file "%s".', $newfile));
237+
}
238+
239+
try {
240+
if (!$zip->extractTo($extractPath)) {
241+
throw new LogicException(\sprintf('failed to extract content from zip file "%s".', $newfile));
242+
}
243+
} finally {
244+
$zip->close();
245+
}
246+
247+
$fileContents = $this->getFileContents($extractPath);
248+
249+
// Lokalise returns languages with "-" separator, we need to reformat them to "_" separator.
250+
/** @var array<array-key, array-key> $reformattedLanguages */
251+
$reformattedLanguages = array_map(function ($language) {
252+
return str_replace('-', '_', $language);
253+
}, array_keys($fileContents));
254+
255+
return array_combine($reformattedLanguages, $fileContents);
256+
} finally {
257+
unlink($newfile);
258+
$this->removeDir($extractPath);
259+
}
260+
}
261+
262+
private function getFileContents(string $dir, string $baseDir = ''): array
263+
{
264+
$fileContents = [];
265+
foreach (scandir($dir) as $filename) {
266+
if (\in_array($filename, ['.', '..'])) {
267+
continue;
268+
}
269+
$path = \sprintf('%s%s%s', $dir, \DIRECTORY_SEPARATOR, $filename);
270+
if (is_dir($path)) {
271+
$fileContents = array_merge($fileContents, $this->getFileContents($path, $filename));
272+
continue;
273+
}
274+
$fileContents[$baseDir][$filename]['content'] = file_get_contents($path);
275+
}
276+
277+
return $fileContents;
278+
}
279+
280+
private function removeDir(string $dir): void
281+
{
282+
$it = new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS);
283+
$files = new \RecursiveIteratorIterator($it, \RecursiveIteratorIterator::CHILD_FIRST);
284+
foreach ($files as $file) {
285+
if ($file->isDir()) {
286+
rmdir($file->getPathname());
287+
} else {
288+
unlink($file->getPathname());
289+
}
290+
}
291+
rmdir($dir);
292+
}
293+
179294
private function createKeys(array $keys, string $domain): array
180295
{
181296
$keysToCreate = [];

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/Fixtures/Symfony-locale.zip‎

1.19 KB
Binary file not shown.

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/LokaliseProviderTest.php‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,94 @@ public function testReadForManyLocalesAndManyDomains(array $locales, array $doma
697697
}
698698
}
699699

700+
/**
701+
* @requires extension zip
702+
*/
703+
public function testReadWithExportAsync()
704+
{
705+
$firstResponse = function (): ResponseInterface {
706+
return new JsonMockResponse(
707+
['error' => ['code' => 413, 'message' => 'test']],
708+
['http_code' => 406],
709+
);
710+
};
711+
$secondResponse = function (): ResponseInterface {
712+
return new JsonMockResponse(
713+
['process_id' => 123],
714+
);
715+
};
716+
$thirdResponse = function (): ResponseInterface {
717+
$zipLocation = __DIR__.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'Symfony-locale.zip';
718+
719+
return new JsonMockResponse(
720+
['process' => ['status' => 'finished', 'details' => ['download_url' => $zipLocation]]],
721+
);
722+
};
723+
724+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse]))->withOptions([
725+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
726+
'headers' => ['X-Api-Token' => 'API_KEY'],
727+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
728+
$translatorBag = $provider->read(['foo'], ['baz']);
729+
730+
// We don't want to assert equality of metadata here, due to the ArrayLoader usage.
731+
/** @var MessageCatalogue $catalogue */
732+
foreach ($translatorBag->getCatalogues() as $catalogue) {
733+
$catalogue->deleteMetadata('', '');
734+
}
735+
736+
$arrayLoader = new ArrayLoader();
737+
$expectedTranslatorBag = new TranslatorBag();
738+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
739+
[
740+
'how are you' => 'How are you?',
741+
'welcome_header' => 'Hello world!',
742+
],
743+
'en',
744+
'no_filename'
745+
));
746+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
747+
[
748+
'how are you' => 'Como estas?',
749+
'welcome_header' => 'Hola mundo!',
750+
],
751+
'es',
752+
'no_filename'
753+
));
754+
$this->assertEquals($expectedTranslatorBag->getCatalogues(), $translatorBag->getCatalogues());
755+
}
756+
757+
/**
758+
* @requires extension zip
759+
*/
760+
public function testReadWithExportAsyncFailedProcess()
761+
{
762+
$firstResponse = function (): ResponseInterface {
763+
return new JsonMockResponse(
764+
['error' => ['code' => 413, 'message' => 'test']],
765+
['http_code' => 406],
766+
);
767+
};
768+
$secondResponse = function (): ResponseInterface {
769+
return new JsonMockResponse(
770+
['process_id' => 123],
771+
);
772+
};
773+
$thirdResponse = function (): ResponseInterface {
774+
return new JsonMockResponse(
775+
['process' => ['status' => 'failed']],
776+
);
777+
};
778+
779+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse]))->withOptions([
780+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
781+
'headers' => ['X-Api-Token' => 'API_KEY'],
782+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
783+
784+
$this->expectException(ProviderException::class);
785+
$provider->read(['foo'], ['baz']);
786+
}
787+
700788
public function testDeleteProcess()
701789
{
702790
$getKeysIdsForMessagesDomainResponse = function (string $method, string $url, array $options = []): ResponseInterface {

0 commit comments

Comments
 (0)