Add chunk_callback to download_url()

ThomasFarstrike · ThomasFarstrike · commit c80fa05a771e · 2025-12-17T09:47:28.000+01:00
diff --git a/internal_filesystem/builtin/apps/com.micropythonos.appstore/assets/appstore.py b/internal_filesystem/builtin/apps/com.micropythonos.appstore/assets/appstore.py
@@ -177,62 +177,89 @@ def show_app_detail(self, app):
         intent.putExtra("appstore", self)
         self.startActivity(intent)
 
-    async def download_url(self, url, outfile=None, total_size=None, progress_callback=None):
+    '''
+    This async download function can be used in 3 ways:
+    - with just a url => returns the content
+    - with a url and an outfile => writes the content to the outfile
+    - with a url and a chunk_callback => calls the chunk_callback(chunk_data) for each chunk
+
+    Optionally:
+    - progress_callback is called with the % (0-100) progress
+    - if total_size is not provided, it will be taken from the response headers (if present) or default to 100KB
+
+    Can return either:
+    - the actual content
+    - None: if the content failed to download
+    - True: if the URL was successfully downloaded (and written to outfile, if provided)
+    - False: if the URL was not successfully download and written to outfile
+    '''
+    async def download_url(self, url, outfile=None, total_size=None, progress_callback=None, chunk_callback=None):
         print(f"Downloading {url}")
         #await TaskManager.sleep(4) # test slowness
         try:
             async with self.aiohttp_session.get(url) as response:
                 if response.status < 200 or response.status >= 400:
                     return False if outfile else None
 
-                # Always use chunked downloading
-                chunk_size = 1024
+                # Figure out total size
                 print("headers:") ; print(response.headers)
                 if total_size is None:
                     total_size = response.headers.get('Content-Length') # some servers don't send this in the headers
-                print(f"download_url {'writing to ' + outfile if outfile else 'reading'} {total_size} bytes in chunks of size {chunk_size}")
-
-                fd = open(outfile, 'wb') if outfile else None
-                chunks = [] if not outfile else None
+                    if total_size is None:
+                        print("WARNING: Unable to determine total_size from server's reply and function arguments, assuming 100KB")
+                        total_size = 100 * 1024
+
+                fd = None
+                if outfile:
+                    fd = open(outfile, 'wb')
+                    if not fd:
+                        print("WARNING: could not open {outfile} for writing!")
+                        return False
+                chunks = []
                 partial_size = 0
+                chunk_size = 1024
 
-                if fd:
-                    print("opened file...")
+                print(f"download_url {'writing to ' + outfile if outfile else 'downloading'} {total_size} bytes in chunks of size {chunk_size}")
 
                 while True:
                     tries_left = 3
-                    chunk = None
+                    chunk_data = None
                     while tries_left > 0:
                         try:
-                            chunk = await TaskManager.wait_for(response.content.read(chunk_size), 10)
+                            chunk_data = await TaskManager.wait_for(response.content.read(chunk_size), 10)
                             break
                         except Exception as e:
-                            print(f"Waiting for response.content.read of next chunk got error: {e}")
+                            print(f"Waiting for response.content.read of next chunk_data got error: {e}")
                             tries_left -= 1
 
                     if tries_left == 0:
-                        print("ERROR: failed to download chunk, even with retries!")
+                        print("ERROR: failed to download chunk_data, even with retries!")
                         if fd:
                             fd.close()
                         return False if outfile else None
 
-                    if chunk:
-                        partial_size += len(chunk)
+                    if chunk_data:
+                        # Output
+                        if fd:
+                            fd.write(chunk_data)
+                        elif chunk_callback:
+                            await chunk_callback(chunk_data)
+                        else:
+                            chunks.append(chunk_data)
+                        # Report progress
+                        partial_size += len(chunk_data)
                         progress_pct = round((partial_size * 100) / int(total_size))
                         print(f"progress: {partial_size} / {total_size} bytes = {progress_pct}%")
                         if progress_callback:
                             await progress_callback(progress_pct)
                             #await TaskManager.sleep(1) # test slowness
-                        if fd:
-                            fd.write(chunk)
-                        else:
-                            chunks.append(chunk)
                     else:
-                        print("chunk is None while there was no error so this was the last one")
-                        print(f"Done downloading {url}")
+                        print("chunk_data is None while there was no error so this was the last one.\n Finished downloading {url}")
                         if fd:
                             fd.close()
                             return True
+                        elif chunk_callback:
+                            return True
                         else:
                             return b''.join(chunks)
         except Exception as e:
