vimPlugins: backoff on timeout in update.py

Updating vim-plugins recently started timing out regularly for me. It
may have to do with an ISP switch on my side, but I don't think that
should cause timeouts. I guess I'm probably not the only one
experiencing this, so in this comment I introduce exponential backoff.
Every request will be retried up to 3 times (3 seconds, 6 seconds, 12
seconds delay).

1 files changed, 43 insertions, 3 deletions

diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py
index ddc19d7426e..0ef93ac569a 100755
--- a/pkgs/misc/vim-plugins/update.py
+++ b/pkgs/misc/vim-plugins/update.py
@@ -21,7 +21,7 @@ import xml.etree.ElementTree as ET
 from datetime import datetime
 from multiprocessing.dummy import Pool
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union, Any
+from typing import Dict, List, Optional, Tuple, Union, Any, Callable
 from urllib.parse import urljoin, urlparse
 from tempfile import NamedTemporaryFile
 
@@ -33,6 +33,42 @@ ROOT = Path(__file__).parent
 DEFAULT_IN = ROOT.joinpath("vim-plugin-names")
 DEFAULT_OUT = ROOT.joinpath("generated.nix")
 
+import time
+from functools import wraps
+
+
+def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
+    """Retry calling the decorated function using an exponential backoff.
+
+    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+    (BSD licensed)
+
+    :param ExceptionToCheck: the exception on which to retry
+    :param tries: number of times to try (not retry) before giving up
+    :param delay: initial delay between retries in seconds
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+        each retry
+    """
+
+    def deco_retry(f: Callable) -> Callable:
+        @wraps(f)
+        def f_retry(*args: Any, **kwargs: Any) -> Any:
+            mtries, mdelay = tries, delay
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except ExceptionToCheck as e:
+                    print(f"{str(e)}, Retrying in {mdelay} seconds...")
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+
+        return f_retry  # true decorator
+
+    return deco_retry
+
 
 class Repo:
     def __init__(self, owner: str, name: str) -> None:
@@ -45,9 +81,12 @@ class Repo:
     def __repr__(self) -> str:
         return f"Repo({self.owner}, {self.name})"
 
+    @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
     def has_submodules(self) -> bool:
         try:
-            urllib.request.urlopen(self.url("blob/master/.gitmodules")).close()
+            urllib.request.urlopen(
+                self.url("blob/master/.gitmodules"), timeout=10
+            ).close()
         except urllib.error.HTTPError as e:
             if e.code == 404:
                 return False
@@ -55,8 +94,9 @@ class Repo:
                 raise
         return True
 
+    @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
     def latest_commit(self) -> Tuple[str, datetime]:
-        with urllib.request.urlopen(self.url("commits/master.atom")) as req:
+        with urllib.request.urlopen(self.url("commits/master.atom"), timeout=10) as req:
             xml = req.read()
             root = ET.fromstring(xml)
             latest_entry = root.find(ATOM_ENTRY)