Update config docs and tests (#589)

ecmwf · Jan 21, 2025 · a932090 · a932090
1 parent c34bb5f
commit a932090
Show file tree

Hide file tree

Showing 13 changed files with 530 additions and 161 deletions.
diff --git a/docs/examples/config.ipynb b/docs/examples/config.ipynb
@@ -173,7 +173,7 @@
     {
      "data": {
       "text/plain": [
-       "5"
+       "30"
       ]
      },
      "execution_count": 4,
@@ -182,7 +182,7 @@
     }
    ],
    "source": [
-    "config.get(\"number-of-download-threads\")"
+    "config.get(\"url-download-timeout\")"
    ]
   },
   {
@@ -215,7 +215,7 @@
     {
      "data": {
       "text/plain": [
-       "6"
+       "5"
       ]
      },
      "execution_count": 5,
@@ -224,8 +224,8 @@
     }
    ],
    "source": [
-    "config.set(\"number-of-download-threads\", 6)\n",
-    "config.get(\"number-of-download-threads\")"
+    "config.set(\"url-download-timeout\", 5)\n",
+    "config.get(\"url-download-timeout\")"
    ]
   },
   {
@@ -246,15 +246,15 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "7\n",
-      "60\n"
+      "10\n",
+      "True\n"
      ]
     }
    ],
    "source": [
-    "config.set({\"number-of-download-threads\": 7, \"url-download-timeout\": \"1m\"})\n",
-    "print(config.get(\"number-of-download-threads\"))\n",
-    "print(config.get(\"url-download-timeout\"))"
+    "config.set({\"url-download-timeout\": 10, \"check-out-of-date-urls\": True})\n",
+    "print(config.get(\"url-download-timeout\"))\n",
+    "print(config.get(\"check-out-of-date-urls\"))"
    ]
   },
   {
@@ -281,15 +281,15 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "8\n",
-      "120\n"
+      "10\n",
+      "True\n"
      ]
     }
    ],
    "source": [
-    "config.set(number_of_download_threads=8, url_download_timeout=\"2m\")\n",
-    "print(config.get(\"number-of-download-threads\"))\n",
-    "print(config.get(\"url-download-timeout\"))"
+    "config.set(url_download_timeout=10, check_out_of_date_urls=True)\n",
+    "print(config.get(\"url-download-timeout\"))\n",
+    "print(config.get(\"check-out-of-date-urls\"))"
    ]
   },
   {
@@ -332,16 +332,16 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "8\n",
+      "10\n",
       "12\n"
      ]
     }
    ],
    "source": [
     "with config.temporary():\n",
-    "    print(config.get(\"number-of-download-threads\"))\n",
-    "    config.set(\"number-of-download-threads\", 12)\n",
-    "    print(config.get(\"number-of-download-threads\"))"
+    "    print(config.get(\"url-download-timeout\"))\n",
+    "    config.set(\"url-download-timeout\", 12)\n",
+    "    print(config.get(\"url-download-timeout\"))"
    ]
   },
   {
@@ -361,7 +361,7 @@
     {
      "data": {
       "text/plain": [
-       "8"
+       "10"
       ]
      },
      "execution_count": 9,
@@ -370,7 +370,7 @@
     }
    ],
    "source": [
-    "config.get(\"number-of-download-threads\")"
+    "config.get(\"url-download-timeout\")"
    ]
   },
   {
@@ -391,24 +391,24 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "11\n",
-      "8\n"
+      "12\n",
+      "10\n"
      ]
     }
    ],
    "source": [
-    "with config.temporary(\"number-of-download-threads\", 11):\n",
-    "    print(config.get(\"number-of-download-threads\"))\n",
+    "with config.temporary(\"url-download-timeout\", 12):\n",
+    "    print(config.get(\"url-download-timeout\"))\n",
     "\n",
-    "print(config.get(\"number-of-download-threads\"))"
+    "print(config.get(\"url-download-timeout\"))"
    ]
   },
   {
    "cell_type": "markdown",
    "id": "313fbf8f-a540-449e-b340-5c46014d931c",
    "metadata": {},
    "source": [
-    "#### Reset to defaults"
+    "#### Resetting to defaults"
    ]
   },
   {
@@ -444,17 +444,15 @@
      "output_type": "stream",
      "text": [
       "12\n",
-      "5\n"
+      "10\n"
      ]
     }
    ],
    "source": [
-    "with config.temporary():\n",
-    "    config.set(\"number-of-download-threads\", 12)\n",
-    "    print(config.get(\"number-of-download-threads\"))\n",
-    "    config.reset()\n",
-    "    print(config.get(\"number-of-download-threads\"))\n",
-    "    "
+    "with config.temporary(\"url-download-timeout\", 12):\n",
+    "    print(config.get(\"url-download-timeout\"))\n",
+    "\n",
+    "print(config.get(\"url-download-timeout\"))"
    ]
   },
   {

diff --git a/docs/examples/config_env_vars.ipynb b/docs/examples/config_env_vars.ipynb
@@ -105,7 +105,7 @@
     {
      "data": {
       "text/plain": [
-       "5"
+       "30"
       ]
      },
      "execution_count": 3,
@@ -114,7 +114,7 @@
     }
    ],
    "source": [
-    "config.get(\"number-of-download-threads\")"
+    "config.get(\"url-download-timeout\")"
    ]
   },
   {
@@ -148,12 +148,12 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "env: EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS=26\n"
+      "env: EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT=26\n"
      ]
     }
    ],
    "source": [
-    "%env EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS=26"
+    "%env EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT=26"
    ]
   },
   {
@@ -180,7 +180,7 @@
     }
    ],
    "source": [
-    "config.get(\"number-of-download-threads\")"
+    "config.get(\"url-download-timeout\")"
    ]
   },
   {
@@ -214,7 +214,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/cgr/git/earthkit-data/src/earthkit/data/core/config.py:406: UserWarning: Config option 'number-of-download-threads' is also set by environment variable 'EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS'.The environment variable takes precedence and its value is returned when calling get().\n",
+      "/Users/cgr/git/earthkit-data/src/earthkit/data/core/config.py:407: UserWarning: Config option 'url-download-timeout' is also set by environment variable 'EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT'.The environment variable takes precedence and its value is returned when calling get().\n",
       "  warnings.warn(msg)\n"
      ]
     },
@@ -230,8 +230,8 @@
     }
    ],
    "source": [
-    "config.set(\"number-of-download-threads\", 10)\n",
-    "config.get(\"number-of-download-threads\")"
+    "config.set(\"url-download-timeout\", 10)\n",
+    "config.get(\"url-download-timeout\")"
    ]
   },
   {
@@ -264,8 +264,7 @@
     {
      "data": {
       "text/plain": [
-       "{'number-of-download-threads': ('EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS',\n",
-       "  '26')}"
+       "{'url-download-timeout': ('EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT', '26')}"
       ]
      },
      "execution_count": 7,
@@ -315,7 +314,7 @@
        "    vertical-align: top;\n",
        "    text-align: left !important;\n",
        "}\n",
-       "</style><table class='ek'><tr><th>Name</th><th>Value</th><th>Default</th></tr><tr><td>cache-policy</td><td>'off'</td><td>'off'</td></tr><tr><td>check-out-of-date-urls</td><td>True</td><td>True</td></tr><tr><td>download-out-of-date-urls</td><td>False</td><td>False</td></tr><tr><td>grib-field-policy</td><td>'persistent'</td><td>'persistent'</td></tr><tr><td>grib-handle-cache-size</td><td>1</td><td>1</td></tr><tr><td>grib-handle-policy</td><td>'cache'</td><td>'cache'</td></tr><tr><td>maximum-cache-disk-usage</td><td>'95%'</td><td>'95%'</td></tr><tr><td>maximum-cache-size</td><td>None</td><td>None</td></tr><tr><td>number-of-download-threads</td><td>EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS='26'<br>(10)</td><td>5</td></tr><tr><td>reader-type-check-bytes</td><td>64</td><td>64</td></tr><tr><td>temporary-cache-directory-root</td><td>None</td><td>None</td></tr><tr><td>temporary-directory-root</td><td>None</td><td>None</td></tr><tr><td>url-download-timeout</td><td>'30s'</td><td>'30s'</td></tr><tr><td>use-grib-metadata-cache</td><td>True</td><td>True</td></tr><tr><td>use-message-position-index-cache</td><td>False</td><td>False</td></tr><tr><td>use-standalone-mars-client-when-available</td><td>True</td><td>True</td></tr><tr><td>user-cache-directory</td><td>'/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr'</td><td>'/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr'</td></tr><tr><td>version</td><td>'0.11.5.dev2+g384bbb0.d20241209'</td><td>''</td></tr></table>"
+       "</style><table class='ek'><tr><th>Name</th><th>Value</th><th>Default</th></tr><tr><td>cache-policy</td><td>'off'</td><td>'off'</td></tr><tr><td>check-out-of-date-urls</td><td>True</td><td>True</td></tr><tr><td>download-out-of-date-urls</td><td>False</td><td>False</td></tr><tr><td>grib-field-policy</td><td>'persistent'</td><td>'persistent'</td></tr><tr><td>grib-handle-cache-size</td><td>1</td><td>1</td></tr><tr><td>grib-handle-policy</td><td>'cache'</td><td>'cache'</td></tr><tr><td>maximum-cache-disk-usage</td><td>'95%'</td><td>'95%'</td></tr><tr><td>maximum-cache-size</td><td>None</td><td>None</td></tr><tr><td>number-of-download-threads</td><td>5</td><td>5</td></tr><tr><td>reader-type-check-bytes</td><td>64</td><td>64</td></tr><tr><td>temporary-cache-directory-root</td><td>None</td><td>None</td></tr><tr><td>temporary-directory-root</td><td>None</td><td>None</td></tr><tr><td>url-download-timeout</td><td>EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT='26'<br>(10)</td><td>'30s'</td></tr><tr><td>use-grib-metadata-cache</td><td>True</td><td>True</td></tr><tr><td>use-message-position-index-cache</td><td>False</td><td>False</td></tr><tr><td>use-standalone-mars-client-when-available</td><td>True</td><td>True</td></tr><tr><td>user-cache-directory</td><td>'/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr'</td><td>'/var/folders/93/w0p869rx17q98wxk83gn9ys40000gn/T/earthkit-data-cgr'</td></tr><tr><td>version</td><td>'0.11.5.dev2+g384bbb0.d20241209'</td><td>''</td></tr></table>"
       ],
       "text/plain": [
        "cache-policy: (off, off)\n",
@@ -326,11 +325,11 @@
        "grib-handle-policy: (cache, cache)\n",
        "maximum-cache-disk-usage: (95%, 95%)\n",
        "maximum-cache-size: (None, None)\n",
-       "number-of-download-threads: (EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS=26, 10, 5)\n",
+       "number-of-download-threads: (5, 5)\n",
        "reader-type-check-bytes: (64, 64)\n",
        "temporary-cache-directory-root: (None, None)\n",
        "temporary-directory-root: (None, None)\n",
-       "url-download-timeout: (30s, 30s)\n",
+       "url-download-timeout: (EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT=26, 10, 30s)\n",
        "use-grib-metadata-cache: (True, True)\n",
        "use-message-position-index-cache: (False, False)\n",
        "use-standalone-mars-client-when-available: (True, True)\n",

diff --git a/docs/guide/caching.rst b/docs/guide/caching.rst
@@ -15,7 +15,7 @@ Please note that the earthkit-data cache configuration is managed through the :d
 
 .. warning::
 
-  By default the caching is disabled, i.e. the :ref:`cache-policy <cache_policies>` is "off".
+  By default the caching is disabled, i.e. the :ref:`cache-policy <cache_policies>` is :ref:`off <off_cache_policy>`.
 
 .. warning::
 
@@ -206,7 +206,7 @@ Examples:
       >>> cache.policy.name
       'user'
       >>> cache.directory()
-      '/var/folders/ng/g0zkhc2s42xbslpsywwp_26m0000gn/T/earthkit-data-cgr'
+      '/var/folders/ng/g0zkhc2s42xbslpsywwp_26m0000gn/T/earthkit-data-myusername'
       >>> cache.size()
       846785699
       >>> cache.summary_dump_database()

diff --git a/docs/guide/config.rst b/docs/guide/config.rst
@@ -58,8 +58,8 @@ We can create a temporary configuration (as a context manager) as a copy of the
 
 Output::
 
-    8
-    12
+    30
+    5
     11
 
 .. warning::
@@ -93,47 +93,46 @@ Environment variables
 
 Each configuration parameter has a corresponding environment variable (see the full list :ref:`here <config_env_table>`). When an environment variable is set, it takes precedence over the config parameter as the following examples show.
 
-First, let us assume that the value of  ``number-of-download-threads`` is 5 in the config file and no environment variable is set.
+First, let us assume that the value of  ``url-download-timeout`` is 5 in the config file and no environment variable is set.
 
 .. code-block:: python
 
     >>> from earthkit.data import config
-    >>> config.get("number-of-download-threads")
-    5
+    >>> config.get("url-download-timeout")
+    30
 
-Then, set the environment variable ``EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS``.
+Then, set the environment variable ``EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT``.
 
 .. code-block:: bash
 
-    export EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS=26
-
+    export EARTHKIT_REGRID_URL_DOWNLOAD_TIMEOUT=5
 
 .. code-block:: python
 
     >>> from earthkit.data import config
-    >>> config.get("number-of-download-threads")
-    26
+    >>> config.get("url-download-timeout")
+    5
     >>> config.env()
-    {'number-of-download-threads': ('EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS', '26')}
-    >>> config.set("number-of-download-threads", 10)
-    UserWarning: Config option 'number-of-download-threads' is also set by environment variable
-    'EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS'.The environment variable takes precedence and
+    {'url-download-timeout': ('EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT', '5')}
+    >>> config.set("url-download-timeout", 10)
+    UserWarning: Config option 'url-download-timeout' is also set by environment variable
+    'EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT'.The environment variable takes precedence and
     its value is returned when calling get(). Still, the value set here will be
     saved to the config file.
-    >>> config.get("number-of-download-threads")
-    26
+    >>> config.get("url-download-timeout")
+    5
 
 Finally, unset the environment variable and check the config value again, which is now the value from the config file.
 
 .. code-block:: bash
 
-    unset EARTHKIT_DATA_NUMBER_OF_DOWNLOAD_THREADS
+    unset EARTHKIT_DATA_URL_DOWNLOAD_TIMEOUT
 
 
 .. code-block:: python
 
     >>> from earthkit.data import config
-    >>> config.get("number-of-download-threads")
+    >>> config.get("url-download-timeout")
     10
 
 

diff --git a/docs/guide/include/config-set.py b/docs/guide/include/config-set.py
@@ -3,14 +3,14 @@
 # Change the location of the user defined cache:
 earthkit.data.config.set("user-cache-directory", "/big-disk/earthkit-data-cache")
 
-# Change number of download threads
-earthkit.data.config.set("number-of-download-threads", 7)
+# Change download timeout
+earthkit.data.config.set("url-download-timeout", "1m")
 
 # Multiple values can be set together. The argument list
 # can be a dictionary:
-earthkit.data.config.set({"number-of-download-threads": 7, "url-download-timeout": "1m"})
+earthkit.data.config.set({"url-download-timeout": "1m", "check-out-of-date-urls": True})
 
 # Alternatively, we can use keyword arguments. However, because
 # the “-” character is not allowed in variable names in Python we have
 # to replace “-” with “_” in all the keyword arguments:
-earthkit.data.config.set(number_of_download_threads=8, url_download_timeout="2m")
+earthkit.data.config.set(url_download_timeout="1m", check_out_of_date_urls=True)
diff --git a/docs/guide/include/config-temporary.py b/docs/guide/include/config-temporary.py
@@ -1,11 +1,11 @@
 import earthkit.data
 
-print(earthkit.data.config.get("number-of-download-threads"))
+print(earthkit.data.config.get("url-download-timeout"))
 
 with earthkit.data.config.temporary():
-    earthkit.data.config.set("number-of-download-threads", 12)
-    print(earthkit.data.config.get("number-of-download-threads"))
+    earthkit.data.config.set("url-download-timeout", 5)
+    print(earthkit.data.config.get("url-download-timeout"))
 
 # Temporary config can also be created with arguments:
-with earthkit.data.config.temporary("number-of-download-threads", 11):
-    print(earthkit.data.config.get("number-of-download-threads"))
+with earthkit.data.config.temporary("url-download-timeout", 11):
+    print(earthkit.data.config.get("url-download-timeout"))