Skip to content

Commit de16442

Browse files
santysisisantysisi
committed
[Translation][Lokalise] fix "Project too big for sync export"
1 parent 4433ffc commit de16442

File tree

3 files changed

+213
-0
lines changed

3 files changed

+213
-0
lines changed

‎src/Symfony/Component/Translation/Bridge/Lokalise/LokaliseProvider.php‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace Symfony\Component\Translation\Bridge\Lokalise;
1313

1414
use Psr\Log\LoggerInterface;
15+
use Symfony\Component\Translation\Exception\LogicException;
1516
use Symfony\Component\Translation\Exception\ProviderException;
1617
use Symfony\Component\Translation\Loader\LoaderInterface;
1718
use Symfony\Component\Translation\MessageCatalogueInterface;
@@ -31,6 +32,10 @@
3132
final class LokaliseProvider implements ProviderInterface
3233
{
3334
private const LOKALISE_GET_KEYS_LIMIT = 5000;
35+
private const PROJECT_TOO_BIG_STATUS_CODE = 413;
36+
/** @var list<string> */
37+
private const FAILED_PROCESS_STATUS = ['cancelled', 'failed'];
38+
private const SUCESS_PROCESS_STATUS = 'finished';
3439

3540
private HttpClientInterface $client;
3641
private LoaderInterface $loader;
@@ -165,6 +170,13 @@ private function exportFiles(array $locales, array $domains): array
165170
}
166171

167172
if (200 !== $response->getStatusCode()) {
173+
if (($errorCode = $responseContent['error']['code'] ?? null) && self::PROJECT_TOO_BIG_STATUS_CODE === $errorCode) {
174+
if (!\extension_loaded('zip')) {
175+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s". There are too many translations. You need to enable the "zip" extension to use asynchronous export.', $response->getContent(false)), $response);
176+
}
177+
178+
return $this->exportFilesAsync($locales, $domains);
179+
}
168180
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
169181
}
170182

@@ -176,6 +188,113 @@ private function exportFiles(array $locales, array $domains): array
176188
return array_combine($reformattedLanguages, $responseContent['files']);
177189
}
178190

191+
/**
192+
* @see https://developers.lokalise.com/reference/download-files-async
193+
*/
194+
private function exportFilesAsync(array $locales, array $domains): array
195+
{
196+
$response = $this->client->request('POST', 'files/async-download', [
197+
'json' => [
198+
'format' => 'symfony_xliff',
199+
'original_filenames' => true,
200+
'filter_langs' => array_values($locales),
201+
'filter_filenames' => array_map($this->getLokaliseFilenameFromDomain(...), $domains),
202+
'export_empty_as' => 'skip',
203+
'replace_breaks' => false,
204+
],
205+
]);
206+
207+
if (200 !== $response->getStatusCode()) {
208+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
209+
}
210+
211+
$processId = $response->toArray()['process_id'];
212+
$attempt = 0;
213+
while (true) {
214+
$response = $this->client->request('GET', \sprintf('processes/%s', $processId));
215+
$process = $response->toArray()['process'];
216+
if (\in_array($process['status'], self::FAILED_PROCESS_STATUS, true)) {
217+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
218+
}
219+
if (self::SUCESS_PROCESS_STATUS === $process['status']) {
220+
$downloadUrl = $process['details']['download_url'];
221+
break;
222+
}
223+
++$attempt;
224+
usleep(500000 * $attempt);
225+
}
226+
227+
$response = $this->client->request('GET', $downloadUrl, ['buffer' => true]);
228+
if (200 !== $response->getStatusCode()) {
229+
throw new ProviderException(\sprintf('Unable to download translations file from Lokalise: "%s".', $response->getContent(false)), $response);
230+
}
231+
$newfile = \sprintf('%s%s%s.zip', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
232+
$extractPath = \sprintf('%s%s%s', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
233+
try {
234+
$fileHandler = fopen($newfile, 'w');
235+
foreach ($this->client->stream($response) as $chunk) {
236+
fwrite($fileHandler, $chunk->getContent());
237+
}
238+
fclose($fileHandler);
239+
240+
$zip = new \ZipArchive();
241+
if (!$zip->open($newfile, \ZipArchive::CREATE)) {
242+
throw new LogicException(\sprintf('failed to open zip file "%s".', $newfile));
243+
}
244+
245+
try {
246+
if (!$zip->extractTo($extractPath)) {
247+
throw new LogicException(\sprintf('failed to extract content from zip file "%s".', $newfile));
248+
}
249+
} finally {
250+
$zip->close();
251+
}
252+
253+
$fileContents = $this->getFileContents($extractPath);
254+
255+
// Lokalise returns languages with "-" separator, we need to reformat them to "_" separator.
256+
/** @var array<array-key, array-key> $reformattedLanguages */
257+
$reformattedLanguages = array_map(static fn ($language) => str_replace('-', '_', $language), array_keys($fileContents));
258+
259+
return array_combine($reformattedLanguages, $fileContents);
260+
} finally {
261+
unlink($newfile);
262+
$this->removeDir($extractPath);
263+
}
264+
}
265+
266+
private function getFileContents(string $dir, string $baseDir = ''): array
267+
{
268+
$fileContents = [];
269+
foreach (scandir($dir) as $filename) {
270+
if (\in_array($filename, ['.', '..'])) {
271+
continue;
272+
}
273+
$path = \sprintf('%s%s%s', $dir, \DIRECTORY_SEPARATOR, $filename);
274+
if (is_dir($path)) {
275+
$fileContents = array_merge($fileContents, $this->getFileContents($path, $filename));
276+
continue;
277+
}
278+
$fileContents[$baseDir][$filename]['content'] = file_get_contents($path);
279+
}
280+
281+
return $fileContents;
282+
}
283+
284+
private function removeDir(string $dir): void
285+
{
286+
$it = new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS);
287+
$files = new \RecursiveIteratorIterator($it, \RecursiveIteratorIterator::CHILD_FIRST);
288+
foreach ($files as $file) {
289+
if ($file->isDir()) {
290+
rmdir($file->getPathname());
291+
} else {
292+
unlink($file->getPathname());
293+
}
294+
}
295+
rmdir($dir);
296+
}
297+
179298
private function createKeys(array $keys, string $domain): array
180299
{
181300
$keysToCreate = [];

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/Fixtures/Symfony-locale.zip‎

1.19 KB
Binary file not shown.

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/LokaliseProviderTest.php‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,100 @@ public function testReadForManyLocalesAndManyDomains(array $locales, array $doma
697697
}
698698
}
699699

700+
/**
701+
* @requires extension zip
702+
*/
703+
public function testReadWithExportAsync()
704+
{
705+
$zipLocation = __DIR__.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'Symfony-locale.zip';
706+
$firstResponse = function (): ResponseInterface {
707+
return new JsonMockResponse(
708+
['error' => ['code' => 413, 'message' => 'test']],
709+
['http_code' => 406],
710+
);
711+
};
712+
$secondResponse = function (): ResponseInterface {
713+
return new JsonMockResponse(
714+
['process_id' => 123],
715+
);
716+
};
717+
$thirdResponse = function () use ($zipLocation): ResponseInterface {
718+
return new JsonMockResponse(
719+
['process' => ['status' => 'finished', 'details' => ['download_url' => $zipLocation]]],
720+
);
721+
};
722+
$fourResponse = function (string $method, string $url, array $options = []) use ($zipLocation): ResponseInterface {
723+
$this->assertSame('GET', $method);
724+
$this->assertSame('https://api.lokalise.com'.$zipLocation, $url);
725+
$this->assertTrue($options['buffer']);
726+
727+
return new MockResponse(file_get_contents($zipLocation));
728+
};
729+
730+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse, $fourResponse]))->withOptions([
731+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
732+
'headers' => ['X-Api-Token' => 'API_KEY'],
733+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
734+
$translatorBag = $provider->read(['foo'], ['baz']);
735+
736+
// We don't want to assert equality of metadata here, due to the ArrayLoader usage.
737+
/** @var MessageCatalogue $catalogue */
738+
foreach ($translatorBag->getCatalogues() as $catalogue) {
739+
$catalogue->deleteMetadata('', '');
740+
}
741+
742+
$arrayLoader = new ArrayLoader();
743+
$expectedTranslatorBag = new TranslatorBag();
744+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
745+
[
746+
'how are you' => 'How are you?',
747+
'welcome_header' => 'Hello world!',
748+
],
749+
'en',
750+
'no_filename'
751+
));
752+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
753+
[
754+
'how are you' => 'Como estas?',
755+
'welcome_header' => 'Hola mundo!',
756+
],
757+
'es',
758+
'no_filename'
759+
));
760+
$this->assertEquals($expectedTranslatorBag->getCatalogues(), $translatorBag->getCatalogues());
761+
}
762+
763+
/**
764+
* @requires extension zip
765+
*/
766+
public function testReadWithExportAsyncFailedProcess()
767+
{
768+
$firstResponse = function (): ResponseInterface {
769+
return new JsonMockResponse(
770+
['error' => ['code' => 413, 'message' => 'test']],
771+
['http_code' => 406],
772+
);
773+
};
774+
$secondResponse = function (): ResponseInterface {
775+
return new JsonMockResponse(
776+
['process_id' => 123],
777+
);
778+
};
779+
$thirdResponse = function (): ResponseInterface {
780+
return new JsonMockResponse(
781+
['process' => ['status' => 'failed']],
782+
);
783+
};
784+
785+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse]))->withOptions([
786+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
787+
'headers' => ['X-Api-Token' => 'API_KEY'],
788+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
789+
790+
$this->expectException(ProviderException::class);
791+
$provider->read(['foo'], ['baz']);
792+
}
793+
700794
public function testDeleteProcess()
701795
{
702796
$getKeysIdsForMessagesDomainResponse = function (string $method, string $url, array $options = []): ResponseInterface {

0 commit comments

Comments
 (0)