Skip to content

Commit 414d6da

Browse files
santysisisantysisi
committed
[Translation][Lokalise] fix "Project too big for sync export"
1 parent 4433ffc commit 414d6da

File tree

3 files changed

+190
-0
lines changed

3 files changed

+190
-0
lines changed

‎src/Symfony/Component/Translation/Bridge/Lokalise/LokaliseProvider.php‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace Symfony\Component\Translation\Bridge\Lokalise;
1313

1414
use Psr\Log\LoggerInterface;
15+
use Symfony\Component\Translation\Exception\LogicException;
1516
use Symfony\Component\Translation\Exception\ProviderException;
1617
use Symfony\Component\Translation\Loader\LoaderInterface;
1718
use Symfony\Component\Translation\MessageCatalogueInterface;
@@ -31,6 +32,7 @@
3132
final class LokaliseProvider implements ProviderInterface
3233
{
3334
private const LOKALISE_GET_KEYS_LIMIT = 5000;
35+
private const PROJECT_TOO_BIG_STATUS_CODE = 413;
3436

3537
private HttpClientInterface $client;
3638
private LoaderInterface $loader;
@@ -165,6 +167,9 @@ private function exportFiles(array $locales, array $domains): array
165167
}
166168

167169
if (200 !== $response->getStatusCode()) {
170+
if (($errorCode = $responseContent['error']['code'] ?? null) && self::PROJECT_TOO_BIG_STATUS_CODE === $errorCode && \extension_loaded('zip')) {
171+
return $this->exportFilesAsync($locales, $domains);
172+
}
168173
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
169174
}
170175

@@ -176,6 +181,109 @@ private function exportFiles(array $locales, array $domains): array
176181
return array_combine($reformattedLanguages, $responseContent['files']);
177182
}
178183

184+
/**
185+
* @see https://developers.lokalise.com/reference/download-files-async
186+
*/
187+
private function exportFilesAsync(array $locales, array $domains): array
188+
{
189+
$response = $this->client->request('POST', 'files/async-download', [
190+
'json' => [
191+
'format' => 'symfony_xliff',
192+
'original_filenames' => true,
193+
'filter_langs' => array_values($locales),
194+
'filter_filenames' => array_map($this->getLokaliseFilenameFromDomain(...), $domains),
195+
'export_empty_as' => 'skip',
196+
'replace_breaks' => false,
197+
],
198+
]);
199+
200+
if (200 !== $response->getStatusCode()) {
201+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
202+
}
203+
204+
$processId = $response->toArray()['process_id'];
205+
$attempt = 0;
206+
while (true) {
207+
$response = $this->client->request('GET', \sprintf('processes/%s', $processId));
208+
$process = $response->toArray()['process'];
209+
if ('failed' === $process['status']) {
210+
throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
211+
}
212+
if ('finished' === $process['status']) {
213+
$downloadUrl = $process['details']['download_url'];
214+
break;
215+
}
216+
++$attempt;
217+
usleep(500000 * $attempt);
218+
}
219+
220+
$newfile = \sprintf('%s%s%s.zip', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
221+
$extractPath = \sprintf('%s%s%s', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
222+
if (!copy($downloadUrl, $newfile)) {
223+
throw new LogicException(\sprintf('failed to copy "%s".', $downloadUrl));
224+
}
225+
226+
try {
227+
$zip = new \ZipArchive();
228+
if (!$zip->open($newfile, \ZipArchive::CREATE)) {
229+
throw new LogicException(\sprintf('failed to open zip file "%s".', $newfile));
230+
}
231+
232+
try {
233+
if (!$zip->extractTo($extractPath)) {
234+
throw new LogicException(\sprintf('failed to extract content from zip file "%s".', $newfile));
235+
}
236+
} finally {
237+
$zip->close();
238+
}
239+
240+
$fileContents = $this->getFileContents($extractPath);
241+
242+
// Lokalise returns languages with "-" separator, we need to reformat them to "_" separator.
243+
/** @var array<array-key, array-key> $reformattedLanguages */
244+
$reformattedLanguages = array_map(function ($language) {
245+
return str_replace('-', '_', $language);
246+
}, array_keys($fileContents));
247+
248+
return array_combine($reformattedLanguages, $fileContents);
249+
} finally {
250+
unlink($newfile);
251+
$this->removeDir($extractPath);
252+
}
253+
}
254+
255+
private function getFileContents(string $dir, string $baseDir = ''): array
256+
{
257+
$fileContents = [];
258+
foreach (scandir($dir) as $filename) {
259+
if (\in_array($filename, ['.', '..'])) {
260+
continue;
261+
}
262+
$path = \sprintf('%s%s%s', $dir, \DIRECTORY_SEPARATOR, $filename);
263+
if (is_dir($path)) {
264+
$fileContents = array_merge($fileContents, $this->getFileContents($path, $filename));
265+
continue;
266+
}
267+
$fileContents[$baseDir][$filename]['content'] = file_get_contents($path);
268+
}
269+
270+
return $fileContents;
271+
}
272+
273+
private function removeDir(string $dir): void
274+
{
275+
$it = new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS);
276+
$files = new \RecursiveIteratorIterator($it, \RecursiveIteratorIterator::CHILD_FIRST);
277+
foreach ($files as $file) {
278+
if ($file->isDir()) {
279+
rmdir($file->getPathname());
280+
} else {
281+
unlink($file->getPathname());
282+
}
283+
}
284+
rmdir($dir);
285+
}
286+
179287
private function createKeys(array $keys, string $domain): array
180288
{
181289
$keysToCreate = [];

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/Fixtures/Symfony-locale.zip‎

1.19 KB
Binary file not shown.

‎src/Symfony/Component/Translation/Bridge/Lokalise/Tests/LokaliseProviderTest.php‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,88 @@ public function testReadForManyLocalesAndManyDomains(array $locales, array $doma
697697
}
698698
}
699699

700+
public function testReadWithExportAsync()
701+
{
702+
$firstResponse = function (): ResponseInterface {
703+
return new JsonMockResponse(
704+
['error' => ['code' => 413, 'message' => 'test']],
705+
['http_code' => 406],
706+
);
707+
};
708+
$secondResponse = function (): ResponseInterface {
709+
return new JsonMockResponse(
710+
['process_id' => 123],
711+
);
712+
};
713+
$thirdResponse = function (): ResponseInterface {
714+
$zipLocation = __DIR__.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'Symfony-locale.zip';
715+
716+
return new JsonMockResponse(
717+
['process' => ['status' => 'finished', 'details' => ['download_url' => $zipLocation]]],
718+
);
719+
};
720+
721+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse]))->withOptions([
722+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
723+
'headers' => ['X-Api-Token' => 'API_KEY'],
724+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
725+
$translatorBag = $provider->read(['foo'], ['baz']);
726+
727+
// We don't want to assert equality of metadata here, due to the ArrayLoader usage.
728+
/** @var MessageCatalogue $catalogue */
729+
foreach ($translatorBag->getCatalogues() as $catalogue) {
730+
$catalogue->deleteMetadata('', '');
731+
}
732+
733+
$arrayLoader = new ArrayLoader();
734+
$expectedTranslatorBag = new TranslatorBag();
735+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
736+
[
737+
'how are you' => 'How are you?',
738+
'welcome_header' => 'Hello world!',
739+
],
740+
'en',
741+
'no_filename'
742+
));
743+
$expectedTranslatorBag->addCatalogue($arrayLoader->load(
744+
[
745+
'how are you' => 'Como estas?',
746+
'welcome_header' => 'Hola mundo!',
747+
],
748+
'es',
749+
'no_filename'
750+
));
751+
$this->assertEquals($expectedTranslatorBag->getCatalogues(), $translatorBag->getCatalogues());
752+
}
753+
754+
public function testReadWithExportAsyncFailedProcess()
755+
{
756+
$firstResponse = function (): ResponseInterface {
757+
return new JsonMockResponse(
758+
['error' => ['code' => 413, 'message' => 'test']],
759+
['http_code' => 406],
760+
);
761+
};
762+
$secondResponse = function (): ResponseInterface {
763+
return new JsonMockResponse(
764+
['process_id' => 123],
765+
);
766+
};
767+
$thirdResponse = function (): ResponseInterface {
768+
return new JsonMockResponse(
769+
['process' => ['status' => 'failed']],
770+
);
771+
};
772+
773+
$provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse]))->withOptions([
774+
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
775+
'headers' => ['X-Api-Token' => 'API_KEY'],
776+
]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
777+
778+
$this->expectException(ProviderException::class);
779+
$provider->read(['foo'], ['baz']);
780+
}
781+
700782
public function testDeleteProcess()
701783
{
702784
$getKeysIdsForMessagesDomainResponse = function (string $method, string $url, array $options = []): ResponseInterface {

0 commit comments

Comments
 (0)