Skip to content

Commit a79695c

Browse files
santysisinicolas-grekas
santysisi
authored and
nicolas-grekas
committed
[Translation][Lokalise] fix "Project too big for sync export"
1 parent 4433ffc commit a79695c

File tree

3 files changed

+215
-1
lines changed

3 files changed

+215
-1
lines changed

‎src/Symfony/Component/Translation/Bridge/Lokalise/LokaliseProvider.php‎

Lines changed: 121 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
use Psr\Log\LoggerInterface;
1515
use Symfony\Component\Translation\Exception\ProviderException;
16+
use Symfony\Component\Translation\Exception\RuntimeException;
1617
use Symfony\Component\Translation\Loader\LoaderInterface;
1718
use Symfony\Component\Translation\MessageCatalogueInterface;
1819
use Symfony\Component\Translation\Provider\ProviderInterface;
@@ -31,6 +32,9 @@
3132
final class LokaliseProvider implements ProviderInterface
3233
{
3334
private const LOKALISE_GET_KEYS_LIMIT = 5000;
35+
private const PROJECT_TOO_BIG_STATUS_CODE = 413;
36+
private const FAILED_PROCESS_STATUS = ['cancelled', 'failed'];
37+
private const SUCESS_PROCESS_STATUS = 'finished';
3438

3539
private HttpClientInterface $client;
3640
private LoaderInterface $loader;
@@ -165,7 +169,14 @@ private function exportFiles(array $locales, array $domains): array
165169
}
166170

167171
if (200 !== $response->getStatusCode()) {
168-
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
172+
if (self::PROJECT_TOO_BIG_STATUS_CODE !== ($responseContent['error']['code'] ?? null)) {
173+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
174+
}
175+
if (!\extension_loaded('zip')) {
176+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s". Make sure that the "zip" extension is enabled.', $response->getContent(false)), $response);
177+
}
178+
179+
return $this->exportFilesAsync($locales, $domains);
169180
}
170181

171182
// Lokalise returns languages with "-" separator, we need to reformat them to "_" separator.
@@ -176,6 +187,115 @@ private function exportFiles(array $locales, array $domains): array
176187
return array_combine($reformattedLanguages, $responseContent['files']);
177188
}
178189

190+
/**
191+
* @see https://developers.lokalise.com/reference/download-files-async
192+
*/
193+
private function exportFilesAsync(array $locales, array $domains): array
194+
{
195+
$response = $this->client->request('POST', 'files/async-download', [
196+
'json' => [
197+
'format' => 'symfony_xliff',
198+
'original_filenames' => true,
199+
'filter_langs' => array_values($locales),
200+
'filter_filenames' => array_map($this->getLokaliseFilenameFromDomain(...), $domains),
201+
'export_empty_as' => 'skip',
202+
'replace_breaks' => false,
203+
],
204+
]);
205+
206+
if (200 !== $response->getStatusCode()) {
207+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
208+
}
209+
210+
$processId = $response->toArray()['process_id'];
211+
while (true) {
212+
$response = $this->client->request('GET', \sprintf('processes/%s', $processId));
213+
$process = $response->toArray()['process'];
214+
if (\in_array($process['status'], self::FAILED_PROCESS_STATUS, true)) {
215+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
216+
}
217+
if (self::SUCESS_PROCESS_STATUS === $process['status']) {
218+
$downloadUrl = $process['details']['download_url'];
219+
break;
220+
}
221+
usleep(500000);
222+
}
223+
224+
$response = $this->client->request('GET', $downloadUrl, ['buffer' => false]);
225+
if (200 !== $response->getStatusCode()) {
226+
throw new ProviderException(\sprintf('Unable to download translations file from Lokalise: "%s".', $response->getContent(false)), $response);
227+
}
228+
$zipFile = tempnam(sys_get_temp_dir(), 'lokalise');
229+
$extractPath = $zipFile.'.dir';
230+
try {
231+
if (!$h = @fopen($zipFile, 'w')) {
232+
throw new RuntimeException(error_get_last()['message'] ?? 'Failed to create temporary file.');
233+
}
234+
foreach ($this->client->stream($response) as $chunk) {
235+
fwrite($h, $chunk->getContent());
236+
}
237+
fclose($h);
238+
239+
$zip = new \ZipArchive();
240+
if (!$zip->open($zipFile)) {
241+
throw new RuntimeException('Failed to open zipped translations from Lokalise.');
242+
}
243+
244+
try {
245+
if (!$zip->extractTo($extractPath)) {
246+
throw new RuntimeException('Failed to unzip translations from Lokalize.');
247+
}
248+
} finally {
249+
$zip->close();
250+
}
251+
252+
return $this->getZipContents($extractPath);
253+
} finally {
254+
if (is_resource($h)) {
255+
fclose($h);
256+
}
257+
@unlink($zipFile);
258+
$this->removeDir($extractPath);
259+
}
260+
}
261+
262+
private function getZipContents(string $dir): array
263+
{
264+
$contents = [];
265+
foreach (scandir($dir) as $lang) {
266+
if (\in_array($lang, ['.', '..'], true)) {
267+
continue;
268+
}
269+
$path = $dir.'/'.$lang;
270+
// Lokalise returns languages with "-" separator, we need to reformat them to "_" separator.
271+
$lang = str_replace('-', '_', $lang);
272+
foreach (scandir($path) as $name) {
273+
if (!\in_array($name, ['.', '..'], true)) {
274+
$contents[$lang][$name]['content'] = file_get_contents($path.'/'.$name);
275+
}
276+
}
277+
}
278+
279+
return $contents;
280+
}
281+
282+
private function removeDir(string $dir): void
283+
{
284+
if (!is_dir($dir)) {
285+
return;
286+
}
287+
$it = new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS);
288+
$files = new \RecursiveIteratorIterator($it, \RecursiveIteratorIterator::CHILD_FIRST);
289+
foreach ($files as $file) {
290+
if ($file->isDir()) {
291+
rmdir($file->getPathname());
292+
} else {
293+
unlink($file->getPathname());
294+
}
295+
}
296+
rmdir($dir);
297+
}
298+
179299
private function createKeys(array $keys, string $domain): array
180300
{
181301
$keysToCreate = [];

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/Fixtures/Symfony-locale.zip‎

1.19 KB
Binary file not shown.

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/LokaliseProviderTest.php‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,100 @@ public function testReadForManyLocalesAndManyDomains(array $locales, array $doma
697697
}
698698
}
699699

700+
/**
701+
* @requires extension zip
702+
*/
703+
public function testReadWithExportAsync()
704+
{
705+
$zipLocation = __DIR__.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'Symfony-locale.zip';
706+
$firstResponse = function (): ResponseInterface {
707+
return new JsonMockResponse(
708+
['error' => ['code' => 413, 'message' => 'test']],
709+
['http_code' => 406],
710+
);
711+
};
712+
$secondResponse = function (): ResponseInterface {
713+
return new JsonMockResponse(
714+
['process_id' => 123],
715+
);
716+
};
717+
$thirdResponse = function (): ResponseInterface {
718+
return new JsonMockResponse(
719+
['process' => ['status' => 'finished', 'details' => ['download_url' => 'https://api.lokalise.com/Symfony-locale.zip']]],
720+
);
721+
};
722+
$fourResponse = function (string $method, string $url, array $options = []) use ($zipLocation): ResponseInterface {
723+
$this->assertSame('GET', $method);
724+
$this->assertSame('https://api.lokalise.com/Symfony-locale.zip', $url);
725+
$this->assertFalse($options['buffer']);
726+
727+
return new MockResponse(file_get_contents($zipLocation));
728+
};
729+
730+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse, $fourResponse]))->withOptions([
731+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
732+
'headers' => ['X-Api-Token' => 'API_KEY'],
733+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
734+
$translatorBag = $provider->read(['foo'], ['baz']);
735+
736+
// We don't want to assert equality of metadata here, due to the ArrayLoader usage.
737+
/** @var MessageCatalogue $catalogue */
738+
foreach ($translatorBag->getCatalogues() as $catalogue) {
739+
$catalogue->deleteMetadata('', '');
740+
}
741+
742+
$arrayLoader = new ArrayLoader();
743+
$expectedTranslatorBag = new TranslatorBag();
744+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
745+
[
746+
'how are you' => 'How are you?',
747+
'welcome_header' => 'Hello world!',
748+
],
749+
'en',
750+
'no_filename'
751+
));
752+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
753+
[
754+
'how are you' => 'Como estas?',
755+
'welcome_header' => 'Hola mundo!',
756+
],
757+
'es',
758+
'no_filename'
759+
));
760+
$this->assertEquals($expectedTranslatorBag->getCatalogues(), $translatorBag->getCatalogues());
761+
}
762+
763+
/**
764+
* @requires extension zip
765+
*/
766+
public function testReadWithExportAsyncFailedProcess()
767+
{
768+
$firstResponse = function (): ResponseInterface {
769+
return new JsonMockResponse(
770+
['error' => ['code' => 413, 'message' => 'test']],
771+
['http_code' => 406],
772+
);
773+
};
774+
$secondResponse = function (): ResponseInterface {
775+
return new JsonMockResponse(
776+
['process_id' => 123],
777+
);
778+
};
779+
$thirdResponse = function (): ResponseInterface {
780+
return new JsonMockResponse(
781+
['process' => ['status' => 'failed']],
782+
);
783+
};
784+
785+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse]))->withOptions([
786+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
787+
'headers' => ['X-Api-Token' => 'API_KEY'],
788+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
789+
790+
$this->expectException(ProviderException::class);
791+
$provider->read(['foo'], ['baz']);
792+
}
793+
700794
public function testDeleteProcess()
701795
{
702796
$getKeysIdsForMessagesDomainResponse = function (string $method, string $url, array $options = []): ResponseInterface {

0 commit comments

Comments
 (0)