From cd01e78a9ca5c796755fc8770e2d981b6f75c719 Mon Sep 17 00:00:00 2001
From: Paul Marillonnet <pmarillonnet@entrouvert.com>
Date: Fri, 17 Nov 2017 16:29:59 +0100
Subject: [PATCH] WIP add federation_utils module (#19396)

---
 mellon/federation_utils.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++
 setup.py                   |  1 +
 2 files changed, 84 insertions(+)
 create mode 100644 mellon/federation_utils.py

diff --git a/mellon/federation_utils.py b/mellon/federation_utils.py
new file mode 100644
index 0000000..f6c95b2
--- /dev/null
+++ b/mellon/federation_utils.py
@@ -0,0 +1,83 @@
+import fcntl
+import logging
+import tempfile
+import threading
+from datetime import timedelta
+
+from django.utils.text import slugify
+from datetime import datetime
+
+import requests
+from pytz import utc
+from time import mktime, sleep
+import os
+import hashlib
+import os.path
+
+from django.core.files.storage import default_storage
+
+
+def truncate_unique(s, length):
+    if len(s) < length:
+        return s
+    md5 = hashlib.md5(s.encode('ascii')).hexdigest()
+    # we should be the first and last characters from the URL
+    l = (length - len(md5)) / 2 - 2  # four additional characters
+    assert l > 20
+    return s[:l] + '...' + s[-l:] + '_' + md5
+
+
+def dt_to_timestamp(input_dt):
+    return mktime(utc.localize(input_dt).utctimetuple())
+
+
+def load_federation_cache(url):
+    logger = logging.getLogger(__name__)
+    try:
+        filename = truncate_unique(slugify(url), 250)
+        path = os.path.join('metadata-cache', filename)
+
+        unix_path = default_storage.path(path)
+        if not os.path.exists('metadata-cache'):
+            os.makedirs('metadata-cache')
+        f = open(unix_path, 'w')
+        try:
+            fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except IOError:
+            return
+        else:
+            with tempfile.NamedTemporaryFile(dir=os.path.dirname(unix_path), delete=False) as temp:
+                try:
+                    # increase modified time by one hour to prevent too many updates
+                    st = os.stat(unix_path)
+                    os.utime(unix_path, (st.st_atime, st.st_mtime + 3600))
+                    response = requests.get(url)
+                    response.raise_for_status()
+                    temp.write(response.content)
+                    temp.flush()
+                    os.rename(temp.name, unix_path)
+                except:
+                    os.unlink(temp.name)
+                finally:
+                    fcntl.lockf(f, fcntl.LOCK_UN)
+        finally:
+            f.close()
+    except OSError:
+        logger.exception(u"could create the intermediary 'metadata-cache' "
+                         "folder")
+        return
+    except:
+        logger.exception(u'failed to load federation from %s', url)
+
+
+def get_federation_from_url(url, update_cache=False):
+    logger = logging.getLogger(__name__)
+    filename = truncate_unique(slugify(url), 250)
+    path = os.path.join('metadata-cache', filename)
+    if not default_storage.exists(path) or update_cache or \
+            default_storage.created_time(path) < datetime.now() - timedelta(days=1):
+        threading.Thread(target=load_federation_cache, args=(url,)).start()
+        sleep(2)
+    return default_storage.open(path).read()
+    logger.warning('federation %s has not been loaded', url)
+    return None
diff --git a/setup.py b/setup.py
index 27948bb..f865b72 100755
--- a/setup.py
+++ b/setup.py
@@ -94,6 +94,7 @@ setup(name="django-mellon",
           'django>=1.5',
           'requests',
           'isodate',
+          'pytz',
       ],
       setup_requires=[
           'django',
-- 
2.11.0