Skip to content

Commit 565c8c2

Browse files
santysisisantysisi
committed
[Translation][Lokalise] fix "Project too big for sync export"
1 parent 4433ffc commit 565c8c2

File tree

3 files changed

+207
-0
lines changed

3 files changed

+207
-0
lines changed

‎src/Symfony/Component/Translation/Bridge/Lokalise/LokaliseProvider.php‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace Symfony\Component\Translation\Bridge\Lokalise;
1313

1414
use Psr\Log\LoggerInterface;
15+
use Symfony\Component\Translation\Exception\LogicException;
1516
use Symfony\Component\Translation\Exception\ProviderException;
1617
use Symfony\Component\Translation\Loader\LoaderInterface;
1718
use Symfony\Component\Translation\MessageCatalogueInterface;
@@ -31,6 +32,9 @@
3132
final class LokaliseProvider implements ProviderInterface
3233
{
3334
private const LOKALISE_GET_KEYS_LIMIT = 5000;
35+
private const PROJECT_TOO_BIG_STATUS_CODE = 413;
36+
private const FAILED_PROCESS_STATUS = ['cancelled', 'failed'];
37+
private const SUCESS_PROCESS_STATUS = 'finished';
3438

3539
private HttpClientInterface $client;
3640
private LoaderInterface $loader;
@@ -165,6 +169,13 @@ private function exportFiles(array $locales, array $domains): array
165169
}
166170

167171
if (200 !== $response->getStatusCode()) {
172+
if (self::PROJECT_TOO_BIG_STATUS_CODE === ($responseContent['error']['code'] ?? null)) {
173+
if (!\extension_loaded('zip')) {
174+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s". There are too many translations. You need to enable the "zip" extension to use asynchronous export.', $response->getContent(false)), $response);
175+
}
176+
177+
return $this->exportFilesAsync($locales, $domains);
178+
}
168179
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
169180
}
170181

@@ -176,6 +187,108 @@ private function exportFiles(array $locales, array $domains): array
176187
return array_combine($reformattedLanguages, $responseContent['files']);
177188
}
178189

190+
/**
191+
* @see https://developers.lokalise.com/reference/download-files-async
192+
*/
193+
private function exportFilesAsync(array $locales, array $domains): array
194+
{
195+
$response = $this->client->request('POST', 'files/async-download', [
196+
'json' => [
197+
'format' => 'symfony_xliff',
198+
'original_filenames' => true,
199+
'filter_langs' => array_values($locales),
200+
'filter_filenames' => array_map($this->getLokaliseFilenameFromDomain(...), $domains),
201+
'export_empty_as' => 'skip',
202+
'replace_breaks' => false,
203+
],
204+
]);
205+
206+
if (200 !== $response->getStatusCode()) {
207+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
208+
}
209+
210+
$processId = $response->toArray()['process_id'];
211+
while (true) {
212+
$response = $this->client->request('GET', \sprintf('processes/%s', $processId));
213+
$process = $response->toArray()['process'];
214+
if (\in_array($process['status'], self::FAILED_PROCESS_STATUS, true)) {
215+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
216+
}
217+
if (self::SUCESS_PROCESS_STATUS === $process['status']) {
218+
$downloadUrl = $process['details']['download_url'];
219+
break;
220+
}
221+
usleep(500000);
222+
}
223+
224+
$response = $this->client->request('GET', $downloadUrl, ['buffer' => false]);
225+
if (200 !== $response->getStatusCode()) {
226+
throw new ProviderException(\sprintf('Unable to download translations file from Lokalise: "%s".', $response->getContent(false)), $response);
227+
}
228+
$zipFile = tempnam(sys_get_temp_dir(), 'lokalise');
229+
$extractPath = $zipFile.'.dir';
230+
try {
231+
$fileHandler = fopen($zipFile, 'w');
232+
foreach ($this->client->stream($response) as $chunk) {
233+
fwrite($fileHandler, $chunk->getContent());
234+
}
235+
fclose($fileHandler);
236+
237+
$zip = new \ZipArchive();
238+
if (!$zip->open($zipFile)) {
239+
throw new LogicException(\sprintf('Failed to open zip file "%s".', $zipFile));
240+
}
241+
242+
try {
243+
if (!$zip->extractTo($extractPath)) {
244+
throw new LogicException(\sprintf('Failed to extract content from zip file "%s".', $zipFile));
245+
}
246+
} finally {
247+
$zip->close();
248+
}
249+
250+
return $this->getFileContents($extractPath);
251+
} finally {
252+
unlink($zipFile);
253+
$this->removeDir($extractPath);
254+
}
255+
}
256+
257+
private function getFileContents(string $dir): array
258+
{
259+
$fileContents = [];
260+
foreach (scandir($dir) as $dirName) {
261+
if (\in_array($dirName, ['.', '..'], true)) {
262+
continue;
263+
}
264+
$path = $dir.\DIRECTORY_SEPARATOR.$dirName;
265+
// Lokalise returns languages with "-" separator, we need to reformat them to "_" separator.
266+
$language = str_replace('-', '_', $dirName);
267+
foreach (scandir($path) as $filename) {
268+
if (\in_array($filename, ['.', '..'], true)) {
269+
continue;
270+
}
271+
$fileContents[$language][$filename]['content'] = file_get_contents($path.\DIRECTORY_SEPARATOR.$filename);
272+
}
273+
}
274+
275+
return $fileContents;
276+
}
277+
278+
private function removeDir(string $dir): void
279+
{
280+
$it = new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS);
281+
$files = new \RecursiveIteratorIterator($it, \RecursiveIteratorIterator::CHILD_FIRST);
282+
foreach ($files as $file) {
283+
if ($file->isDir()) {
284+
rmdir($file->getPathname());
285+
} else {
286+
unlink($file->getPathname());
287+
}
288+
}
289+
rmdir($dir);
290+
}
291+
179292
private function createKeys(array $keys, string $domain): array
180293
{
181294
$keysToCreate = [];

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/Fixtures/Symfony-locale.zip‎

1.19 KB
Binary file not shown.

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/LokaliseProviderTest.php‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,100 @@ public function testReadForManyLocalesAndManyDomains(array $locales, array $doma
697697
}
698698
}
699699

700+
/**
701+
* @requires extension zip
702+
*/
703+
public function testReadWithExportAsync()
704+
{
705+
$zipLocation = __DIR__.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'Symfony-locale.zip';
706+
$firstResponse = function (): ResponseInterface {
707+
return new JsonMockResponse(
708+
['error' => ['code' => 413, 'message' => 'test']],
709+
['http_code' => 406],
710+
);
711+
};
712+
$secondResponse = function (): ResponseInterface {
713+
return new JsonMockResponse(
714+
['process_id' => 123],
715+
);
716+
};
717+
$thirdResponse = function (): ResponseInterface {
718+
return new JsonMockResponse(
719+
['process' => ['status' => 'finished', 'details' => ['download_url' => 'https://api.lokalise.com/Symfony-locale.zip']]],
720+
);
721+
};
722+
$fourResponse = function (string $method, string $url, array $options = []) use ($zipLocation): ResponseInterface {
723+
$this->assertSame('GET', $method);
724+
$this->assertSame('https://api.lokalise.com/Symfony-locale.zip', $url);
725+
$this->assertFalse($options['buffer']);
726+
727+
return new MockResponse(file_get_contents($zipLocation));
728+
};
729+
730+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse, $fourResponse]))->withOptions([
731+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
732+
'headers' => ['X-Api-Token' => 'API_KEY'],
733+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
734+
$translatorBag = $provider->read(['foo'], ['baz']);
735+
736+
// We don't want to assert equality of metadata here, due to the ArrayLoader usage.
737+
/** @var MessageCatalogue $catalogue */
738+
foreach ($translatorBag->getCatalogues() as $catalogue) {
739+
$catalogue->deleteMetadata('', '');
740+
}
741+
742+
$arrayLoader = new ArrayLoader();
743+
$expectedTranslatorBag = new TranslatorBag();
744+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
745+
[
746+
'how are you' => 'How are you?',
747+
'welcome_header' => 'Hello world!',
748+
],
749+
'en',
750+
'no_filename'
751+
));
752+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
753+
[
754+
'how are you' => 'Como estas?',
755+
'welcome_header' => 'Hola mundo!',
756+
],
757+
'es',
758+
'no_filename'
759+
));
760+
$this->assertEquals($expectedTranslatorBag->getCatalogues(), $translatorBag->getCatalogues());
761+
}
762+
763+
/**
764+
* @requires extension zip
765+
*/
766+
public function testReadWithExportAsyncFailedProcess()
767+
{
768+
$firstResponse = function (): ResponseInterface {
769+
return new JsonMockResponse(
770+
['error' => ['code' => 413, 'message' => 'test']],
771+
['http_code' => 406],
772+
);
773+
};
774+
$secondResponse = function (): ResponseInterface {
775+
return new JsonMockResponse(
776+
['process_id' => 123],
777+
);
778+
};
779+
$thirdResponse = function (): ResponseInterface {
780+
return new JsonMockResponse(
781+
['process' => ['status' => 'failed']],
782+
);
783+
};
784+
785+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse]))->withOptions([
786+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
787+
'headers' => ['X-Api-Token' => 'API_KEY'],
788+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
789+
790+
$this->expectException(ProviderException::class);
791+
$provider->read(['foo'], ['baz']);
792+
}
793+
700794
public function testDeleteProcess()
701795
{
702796
$getKeysIdsForMessagesDomainResponse = function (string $method, string $url, array $options = []): ResponseInterface {

0 commit comments

Comments
 (0)