[Translation][Lokalise] fix "Project too big for sync export"

santysisi · santysisi · commit e984324fd4fa · 2025-11-08T21:12:21.000-03:00
diff --git a/src/Symfony/Component/Translation/Bridge/Lokalise/LokaliseProvider.php b/src/Symfony/Component/Translation/Bridge/Lokalise/LokaliseProvider.php
@@ -12,6 +12,7 @@
 namespace Symfony\Component\Translation\Bridge\Lokalise;
 
 use Psr\Log\LoggerInterface;
+use Symfony\Component\Translation\Exception\LogicException;
 use Symfony\Component\Translation\Exception\ProviderException;
 use Symfony\Component\Translation\Loader\LoaderInterface;
 use Symfony\Component\Translation\MessageCatalogueInterface;
@@ -31,6 +32,9 @@
 final class LokaliseProvider implements ProviderInterface
 {
     private const LOKALISE_GET_KEYS_LIMIT = 5000;
+    private const ERROR_STATUS = [
+        'PROJECT_TOO_BIG' => 413,
+    ];
 
     private HttpClientInterface $client;
     private LoaderInterface $loader;
@@ -96,7 +100,7 @@ public function write(TranslatorBagInterface $translatorBag): void
     public function read(array $domains, array $locales): TranslatorBag
     {
         $translatorBag = new TranslatorBag();
-        $translations = $this->exportFiles($locales, $domains);
+        $translations = $this->exportFilesAsync($locales, $domains);
 
         foreach ($translations as $locale => $files) {
             foreach ($files as $filename => $content) {
@@ -165,6 +169,9 @@ private function exportFiles(array $locales, array $domains): array
         }
 
         if (200 !== $response->getStatusCode()) {
+            if (($errorCode = $responseContent['error']['code'] ?? null) && self::ERROR_STATUS['PROJECT_TOO_BIG'] === $errorCode && \extension_loaded('zip')) {
+                return $this->exportFilesAsync($locales, $domains);
+            }
             throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
         }
 
@@ -176,6 +183,110 @@ private function exportFiles(array $locales, array $domains): array
         return array_combine($reformattedLanguages, $responseContent['files']);
     }
 
+    /**
+     * @see https://developers.lokalise.com/reference/download-files-async
+     */
+    private function exportFilesAsync(array $locales, array $domains): array
+    {
+        $response = $this->client->request('POST', 'files/async-download', [
+            'json' => [
+                'format' => 'symfony_xliff',
+                'original_filenames' => true,
+                'filter_langs' => array_values($locales),
+                'filter_filenames' => array_map($this->getLokaliseFilenameFromDomain(...), $domains),
+                'export_empty_as' => 'skip',
+                'replace_breaks' => false,
+            ],
+        ]);
+
+        if (200 !== $response->getStatusCode()) {
+            throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
+        }
+
+        $processId = $response->toArray()['process_id'];
+        $attempt = 0;
+        while (true) {
+            $response = $this->client->request('GET', \sprintf('processes/%s', $processId));
+            $process = $response->toArray()['process'];
+            if ('failed' === $process['status']) {
+                throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
+            }
+            if ('finished' === $process['status']) {
+                $downloadUrl = $process['details']['download_url'];
+                break;
+            }
+            ++$attempt;
+            usleep(500000 * $attempt);
+        }
+
+        try {
+            $newfile = \sprintf('%s%s%s.zip', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
+            $extractPath = \sprintf('%s%s%s', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
+
+            if (!copy($downloadUrl, $newfile)) {
+                throw new LogicException(\sprintf('failed to copy "%s".', $downloadUrl));
+            }
+
+            $zip = new \ZipArchive();
+            if (!$zip->open($newfile, \ZipArchive::CREATE)) {
+                throw new LogicException(\sprintf('failed to open zip file "%s".', $newfile));
+            }
+
+            try {
+                if (!$zip->extractTo($extractPath)) {
+                    throw new LogicException(\sprintf('failed to extract content from zip file "%s".', $newfile));
+                }
+            } finally {
+                $zip->close();
+            }
+
+            $fileContents = $this->getFileContents($extractPath);
+
+            // Lokalise returns languages with "-" separator, we need to reformat them to "_" separator.
+            /** @var array<array-key, array-key> $reformattedLanguages */
+            $reformattedLanguages = array_map(function ($language) {
+                return str_replace('-', '_', $language);
+            }, array_keys($fileContents));
+
+            return array_combine($reformattedLanguages, $fileContents);
+        } finally {
+            unlink($newfile);
+            $this->removeDir($extractPath);
+        }
+    }
+
+    private function getFileContents(string $dirName, string $baseDirName = ''): array
+    {
+        $fileContents = [];
+        foreach (scandir($dirName) as $filename) {
+            if (\in_array($filename, ['.', '..'])) {
+                continue;
+            }
+            $path = \sprintf('%s%s%s', $dirName, \DIRECTORY_SEPARATOR, $filename);
+            if (is_dir($path)) {
+                $fileContents = array_merge($fileContents, $this->getFileContents($path, $filename));
+                continue;
+            }
+            $fileContents[$baseDirName][$filename]['content'] = file_get_contents($path);
+        }
+
+        return $fileContents;
+    }
+
+    private function removeDir(string $dir): void
+    {
+        $it = new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS);
+        $files = new \RecursiveIteratorIterator($it, \RecursiveIteratorIterator::CHILD_FIRST);
+        foreach ($files as $file) {
+            if ($file->isDir()) {
+                rmdir($file->getPathname());
+            } else {
+                unlink($file->getPathname());
+            }
+        }
+        rmdir($dir);
+    }
+
     private function createKeys(array $keys, string $domain): array
     {
         $keysToCreate = [];
