Merge pull request #4181 from orblivion/file-sizes-ansible

holta · web-flow · commit fd3a317b007d · 2025-12-18T18:53:51.000-05:00
Add file sizes and log reading instructions to map downloads
diff --git a/roles/maps/defaults/main.yml b/roles/maps/defaults/main.yml
@@ -68,7 +68,7 @@ maplibre_search_js_and_styles:
 
 maps_dot_black_naturalearth6_symlink_name: "naturalearth6-NE2_HR_SR_W_DR-WEBP.pmtiles"
 
-# Mostly colors, topography, etc. 58M
+# Mostly colors, topography, etc.
 maps_dot_black_naturalearth6_tiles: "naturalearth6-NE2_HR_SR_W_DR-WEBP.{{ maps_slow_data_date }}.full.pmtiles"
 
 # Two different symlinks because the front end could requset openstreetmap or naturalearth
@@ -80,37 +80,37 @@ maps_dot_black_ne_symlink_name: naturalearth-openmaptiles.pmtiles
 # option and just have low zoom osm. So then it will be `maps_vector_zoom` and
 # just have numbers, like for satellite.
 maps_dot_black_vector_tiles:
-  # "high res" full osm, including 3d buildings. 78GiB as of Oct 2025
+  # "high res" full osm, including 3d buildings.
   # (TODO does this include colors and topography? Or is it used along with naturalearth6 above in most styles?)
   # maps_vector_quality = "osm-full"
   osm-full: "openstreetmap-openmaptiles.{{ maps_data_date }}.full.pmtiles"
 
-  # "medium res" osm, up to zoom level 9 (original file has 14). 1.9GiB as of Oct 2025
+  # "medium res" osm, up to zoom level 9 (original file has 14).
   # (TODO does this include colors and topography? Or is it used along with naturalearth6 above in most styles?)
   # maps_vector_quality = "osm-z9"
   osm-z9: "openstreetmap-openmaptiles.{{ maps_data_date }}.zoom_0-09.pmtiles"
 
-  # "low res" - mostly borders, rivers, country names, large roads. 85MB
+  # "low res" - mostly borders, rivers, country names, large roads.
   # maps_vector_quality = "ne"
   # (ne = "Natural Earth")
   ne: "naturalearth-openmaptiles.{{ maps_slow_data_date }}.full.pmtiles"
 
 maps_dot_black_satellite_symlink_name: s2maps-sentinel2-2023.pmtiles
 
 maps_dot_black_satellite_tiles:
-  # Low quality satellite, up to zoom level 7 (original file has 13), 85MiB
+  # Low quality satellite, up to zoom level 7 (original file has 13)
   # maps_sat_zoom = 7
   7: "s2maps-sentinel2-2023.{{ maps_slow_data_date }}.zoom_0-07.pmtiles"
 
-  # Moderately high quality satellite, up to zoom level 9 (original file has 13), 1.2GiB
+  # Moderately high quality satellite, up to zoom level 9 (original file has 13)
   # maps_sat_zoom = 9
   9: "s2maps-sentinel2-2023.{{ maps_slow_data_date }}.zoom_0-09.pmtiles"
 
-  # Pretty high quality satellite, up to zoom level 11 (original file has 13), 21GiB
+  # Pretty high quality satellite, up to zoom level 11 (original file has 13)
   # maps_sat_zoom = 11
   11: "s2maps-sentinel2-2023.{{ maps_slow_data_date }}.zoom_0-11.pmtiles"
 
-  # Pretty high quality satellite, up to zoom level 12 (original file has 13), 80GiB
+  # Pretty high quality satellite, up to zoom level 12 (original file has 13)
   # maps_sat_zoom = 12
   12: "s2maps-sentinel2-2023.{{ maps_slow_data_date }}.zoom_0-12.pmtiles"
 
diff --git a/roles/maps/tasks/download_large_file.yml b/roles/maps/tasks/download_large_file.yml
@@ -35,33 +35,89 @@
 #
 # TODO - files should be owned by www-data
 
-- name: "Download {{ iiab_map_host_url }}/{{ item }} via .meta4 file"
-  # cd to the temp directory so that the aria2c file, data file, and (perhaps
-  # eventually) torrent file all end up there.
-  shell: |
-    set -euo pipefail
+- block:
+    - name: "Fetching file size for {{ iiab_map_host_url }}/{{ item }} via .meta4 file"
+      shell: |
+        import sys, xmltodict, requests, os
 
-    cd /library/downloads/maps/
+        if os.path.exists("{{ dest_path }}"):
+          sys.exit(0)
 
-    aria2c \
-        --connect-timeout="{{ download_timeout }}" \
-        --log-level=warn \
-        --console-log-level=warn \
-        --summary-interval=0 \
-        --show-console-readout=false \
-        --download-result=hide \
-        --follow-metalink=mem \
-        --max-connection-per-server=4 \
-        --file-allocation=falloc \
-        --enable-http-pipelining=true \
-        --seed-time=0 \
-        "{{ iiab_map_host_url }}/{{ item }}.meta4"
+        size = int(xmltodict.parse(
+          requests.get("{{ iiab_map_host_url }}/{{ item }}.meta4").content
+        )['metalink']['file']['size'])
+        KiB = 1024
+        MiB = KiB * 1024
+        GiB = MiB * 1024
 
-    chmod 644 "{{ item }}"
-    mv "{{ item }}" "{{ dest_path }}"
-  args:
-    executable: /bin/bash
-    creates: "{{ dest_path }}"
-  vars:
-    # Where the file eventually goes
-    dest_path: "{{ dest_base_path }}/{{ item }}"
+        if size < MiB:
+          size_disp = str(round(size / KiB, 2)) + ' KiB'
+        elif size < GiB:
+          size_disp = str(round(size / MiB, 2)) + ' MiB'
+        else:
+          size_disp = str(round(size / GiB, 2)) + ' GiB'
+        print(' (' + size_disp + ') To check progress, run: "tail {{ working_dir }}/{{ log_file }}"')
+      args:
+        executable: /usr/bin/python3
+      vars:
+        # Where the file eventually goes
+        dest_path: "{{ dest_base_path }}/{{ item }}"
+
+        log_file: "{{ item }}.log"
+        working_dir: "/library/downloads/maps"
+      register: download_file_size
+
+    - name: "Download {{ item }}{{ download_file_size.stdout_lines.0 | default(' (done)')}}"
+      # cd to the temp directory so that the aria2c file, data file, and (perhaps
+      # eventually) torrent file all end up there.
+      shell: |
+        set -euo pipefail
+
+        cd {{ working_dir }}
+
+        echo "Starting download..." > "{{ log_file }}"
+        aria2c \
+            --connect-timeout="{{ download_timeout }}" \
+            --log-level=warn \
+            --console-log-level=warn \
+            --summary-interval=60 \
+            --download-result=hide \
+            --follow-metalink=mem \
+            --max-connection-per-server=4 \
+            --file-allocation=falloc \
+            --show-console-readout=false \
+            --enable-http-pipelining=true \
+            --seed-time=0 \
+            "{{ iiab_map_host_url }}/{{ item }}.meta4" \
+            >> "{{ log_file }}"
+
+        chmod 644 "{{ item }}"
+        mv "{{ item }}" "{{ dest_path }}"
+        rm "{{ log_file }}"
+      args:
+        executable: /bin/bash
+        creates: "{{ dest_path }}"
+      vars:
+        # Where the file eventually goes
+        dest_path: "{{ dest_base_path }}/{{ item }}"
+
+        log_file: "{{ item }}.log"
+        working_dir: "/library/downloads/maps"
+  rescue:
+    # We output summaries to a log file for the user's benefit,
+    # but all of the errors necessarily show up there as well.
+    # Here, we parse out the error message, ignoring almost all of
+    # the summary lines, which could be a lot.
+    # Finally, we exit 1 so Ansible shows this error.
+    - name: "Error downloading {{ item }}"
+      shell: |
+        set -euo pipefail
+
+        cd {{ working_dir }}
+
+        echo "aria2c error:" 1>&2
+        tac "{{ log_file }}" | sed '/'"Download Progress Summary as of"'/q' | tac 1>&2
+        exit 1
+      vars:
+        log_file: "{{ item }}.log"
+        working_dir: "/library/downloads/maps"
diff --git a/roles/maps/tasks/install.yml b/roles/maps/tasks/install.yml
@@ -4,9 +4,11 @@
     state: directory
     # mode: 0755
 
-- name: Install aria2
+- name: Install downloading tools
   apt:
-    name: aria2
+    name:
+      - aria2
+      - python3-xmltodict
 
 - include_tasks: install_frontend.yml
 
