Add data set for accessing "daily" resolution

amotl · amotl · commit 39a8f1164932 · 2020-06-03T02:55:50.000+02:00
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -9,6 +9,7 @@ in progress
 - Naming things
 - Update knowledgebase, adjust some field names
 - Add some integration tests
+- Add data set for accessing "daily" resolution
 
 
 2020-05-30 0.12.1
diff --git a/README.rst b/README.rst
@@ -45,13 +45,17 @@ Synopsis
 Command line usage
 ==================
 
+Get all stations with ``daily`` resolution::
+
+    dwdweather stations --resolution=daily
+
 Get all stations with ``hourly`` resolution (default)::
 
-    dwdweather stations
+    dwdweather stations --resolution=hourly
 
 Get all stations with ``10_minutes`` resolution::
 
-    dwdweather stations --resolution 10_minutes
+    dwdweather stations --resolution=10_minutes
 
 Get closest station (first argument is longitude, second is latitude)::
 
@@ -77,6 +81,14 @@ Finally, to drop the cache database before performing any work, use the ``--rese
 
     dwdweather stations --reset-cache
 
+Choose dataset with ``daily`` resolution::
+
+    dwdweather weather 44 2020-06-01 --resolution=daily
+
+Choose dataset with ``hourly`` resolution::
+
+    dwdweather weather 44 2020-06-01T08 --resolution=hourly
+
 Choose dataset with ``10_minutes`` resolution::
 
     dwdweather weather 2667 2019-06-01T15:20 --resolution=10_minutes
@@ -139,28 +151,25 @@ Licenses
 
 Code license
 ============
-Licensed under the MIT license. See file ``LICENSE`` for details.
+Licensed under the MIT license. See `LICENSE <https://github.com/panodata/dwdweather2/blob/master/LICENSE>`__ for details.
 
 Data license
 ============
-The DWD has information about their re-use policy in
+The DWD has information about their terms of use policy in
 `German <https://www.dwd.de/DE/service/copyright/copyright_node.html>`__
 and
 `English <https://www.dwd.de/EN/service/copyright/copyright_node.html>`__.
 
 
-******
-Status
-******
-This piece of software is in a very early stage. No test cases yet. Only
-tested with Python 3.6. Use at your own risk.
+*******************
+Project information
+*******************
 
 Credits
 =======
-Thanks to `Marian Steinbach <https://github.com/marians>`__, `Philipp
-Klaus <https://github.com/pklaus>`__ and all people from
-`DWD <https://www.dwd.de/>`__.
+Thanks to `Marian Steinbach <https://github.com/marians>`__, all
+other contributors and the `DWD <https://www.dwd.de/>`__.
 
 Changelog
 =========
-See file ``CHANGES.rst``.
+See file `CHANGES.rst <https://github.com/panodata/dwdweather2/blob/master/CHANGES.rst>`__.
diff --git a/dwdweather/client.py b/dwdweather/client.py
@@ -7,6 +7,7 @@
 from urllib.parse import urlparse
 from zipfile import ZipFile
 
+from requests import HTTPError
 from requests_cache import CachedSession
 
 from dwdweather import __appname__ as APP_NAME
@@ -26,7 +27,7 @@ class DwdCdcClient:
 
     def __init__(self, resolution, cache_path):
 
-        # Data set selector by resolution (houry, 10_minutes).
+        # Data set selector by resolution (daily, hourly, 10_minutes).
         self.resolution = resolution
 
         # HTTP client.
@@ -67,7 +68,14 @@ def setup_cache(self):
 
     def get_resource_index(self, uri, extension):
         log.info(u'Requesting %s', uri)
-        resource_list = fetch_html_file_list(uri, extension)
+        try:
+            resource_list = fetch_html_file_list(uri, extension)
+        except HTTPError as ex:
+            if ex.response.status_code == 404:
+                #log.warning('Resource {} not found'.format(uri))
+                resource_list = []
+            else:
+                raise
         return resource_list
 
     def get_stations(self, categories):
@@ -77,11 +85,13 @@ def get_stations(self, categories):
         log.info("Loading station data from CDC")
         for category in categories:
             category_name = category["name"]
-            if category_name == "solar" and self.resolution == "hourly":
+            category_folder = category.get("folder", category_name)
+
+            if category_name == "solar" and self.resolution in ["daily", "hourly"]:
                 # workaround - solar has no subdirs
-                index_uri = u"%s/%s" % (self.uri, category_name)
+                index_uri = u"%s/%s" % (self.uri, category_folder)
             else:
-                index_uri = u"%s/%s/recent" % (self.uri, category_name)
+                index_uri = u"%s/%s/recent" % (self.uri, category_folder)
 
             try:
                 resource_list = self.get_resource_index(index_uri, "txt")
@@ -104,6 +114,7 @@ def get_stations(self, categories):
     def get_measurements(self, station_id, category, timeranges):
 
         category_name = category["name"]
+        category_folder = category.get("folder", category_name)
 
         def download_zip(uri):
             log.info("Fetching resource {}".format(uri))
@@ -144,7 +155,7 @@ def download_resource(uri):
                 for thing in download_zip(uri):
                     yield thing
 
-        if category_name == "solar" and self.resolution == "hourly":
+        if category_name == "solar" and self.resolution in ["daily", "hourly"]:
             index_uri = "%s/%s" % (self.uri, category_name)
             resource_uri_effective = find_resource_file(
                 index_uri, "_%05d_" % station_id
@@ -158,7 +169,7 @@ def download_resource(uri):
                 if timerange == "historical":
                     timerange_suffix = "hist"
 
-                index_uri = "%s/%s/%s" % (self.uri, category_name, timerange)
+                index_uri = "%s/%s/%s" % (self.uri, category_folder, timerange)
                 resource_uri_effective = find_resource_file(
                     index_uri, "_%05d_" % station_id
                 )
diff --git a/dwdweather/core.py b/dwdweather/core.py
@@ -51,7 +51,7 @@ def __init__(self, resolution="hourly", category_names=None, **kwargs):
         # Configure context
         # =================
 
-        # Data set selector by resolution (houry, 10_minutes).
+        # Data set selector by resolution (daily, hourly, 10_minutes).
         self.resolution = resolution
 
         # Categories of measurements.
diff --git a/dwdweather/knowledge.py b/dwdweather/knowledge.py
diff --git a/tests/test_dwdweather.py b/tests/test_dwdweather.py
