From fe872e7c3ca51910c5b13e42b4ada28b3687462a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Sat, 31 Aug 2019 14:24:07 +0200
Subject: [PATCH] cartads_cs: cache data (#35722)

---
 passerelle/apps/cartads_cs/models.py | 194 ++++++++++++++++++++++-----
 tests/test_cartads_cs.py             | 178 ++++++++++++------------
 2 files changed, 247 insertions(+), 125 deletions(-)

diff --git a/passerelle/apps/cartads_cs/models.py b/passerelle/apps/cartads_cs/models.py
index 02379d56..115285bc 100644
--- a/passerelle/apps/cartads_cs/models.py
+++ b/passerelle/apps/cartads_cs/models.py
@@ -25,6 +25,7 @@ import zipfile
 
 from Crypto.Cipher import AES
 
+from django.conf import settings
 from django.core.files.storage import default_storage
 from django.core.signing import Signer
 from django.core.urlresolvers import reverse
@@ -33,6 +34,8 @@ from django.http import HttpResponse
 from django.utils.translation import ugettext_lazy as _
 from django.utils.six.moves.urllib import parse as urlparse
 
+from jsonfield import JSONField
+
 from passerelle.base.models import BaseResource
 from passerelle.utils.api import endpoint
 
@@ -41,6 +44,13 @@ def cartads_file_location(instance, filename):
     return 'cartads_cs/%s/%s' % (instance.tracking_code, filename)
 
 
+class CartaDSDataCache(models.Model):
+    data_type = models.CharField(max_length=50)
+    data_parameters = JSONField(default={})
+    data_values = JSONField(default={})
+    last_update_datetime = models.DateTimeField(auto_now=True)
+
+
 class CartaDSFile(models.Model):
     tracking_code = models.CharField(max_length=20)
     id_piece = models.CharField(max_length=20)
@@ -137,28 +147,138 @@ class AbstractCartaDSCS(BaseResource):
         'description': _('Token for upload file'),
     }
 
-    @endpoint(description=_('Get list of collectivities'))
-    def communes(self, request):
+    def update_data_cache(self):
         client = self.soap_client()
+
+        # communes
         resp = client.service.GetCommunes(self.get_token())
-        return {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
+        communes_cache, created = CartaDSDataCache.objects.get_or_create(data_type='communes')
+        communes_cache.data_values = {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
+        communes_cache.save()
+
+        # types dossier
+        types_dossier_ids = {}
+        for commune in communes_cache.data_values['data']:
+            resp = client.service.GetTypesDossier(self.get_token(), int(commune['id']))
+            if resp is None:
+                continue
+            data_cache, created = CartaDSDataCache.objects.get_or_create(
+                    data_type='types_dossier',
+                    data_parameters={'commune_id': int(commune['id'])})
+            data_cache.data_values = {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
+            types_dossier_ids.update({x['id']: True for x in data_cache.data_values['data']})
+            data_cache.save()
+
+        # objets_demande
+        objets_demande_ids = {}
+        for type_dossier_id in types_dossier_ids.keys():
+            resp = client.service.GetObjetsDemande(self.get_token(), type_dossier_id)
+            if resp is None:
+                continue
+            data_cache, created = CartaDSDataCache.objects.get_or_create(
+                    data_type='objets_demande',
+                    data_parameters={'type_dossier_id': type_dossier_id})
+            data_cache.data_values = {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
+            objets_demande_ids.update({x['id']: True for x in data_cache.data_values['data']})
+            data_cache.save()
+
+        # liste_pdf
+        pdfs_path = default_storage.path('public/cartads_cs/%s/documents' % self.slug)
+        if not os.path.exists(pdfs_path):
+            os.makedirs(pdfs_path)
+        for type_compte in [1]:
+            for type_dossier_id in types_dossier_ids.keys():
+                resp = client.service.GetListePdf(self.get_token(), type_dossier_id,
+                        {'TypeCompteUtilisateur': type_compte})
+                if resp is None:
+                    continue
+
+                def format_cerfa_label(x):
+                    try:
+                        if x['Description']:
+                            return u'%(Nom)s: %(Description)s' % x
+                    except KeyError:
+                        pass
+                    return u'%(Nom)s' % x
+
+                data_cache, created = CartaDSDataCache.objects.get_or_create(
+                        data_type='liste_pdf',
+                        data_parameters={
+                            'type_dossier_id': type_dossier_id,
+                            'type_compte': type_compte,
+                            })
+                data_cache.data_values = {'data': [
+                    {'id': x['Identifiant'],
+                     'text': format_cerfa_label(x),
+                     'url': x['UrlTelechargement'],
+                    } for x in resp or []]}
+
+                for value in data_cache.data_values['data']:
+                    filepath = os.path.join(default_storage.path(self.pdf_path(value)))
+                    resp = self.requests.get(value['url'])
+                    if resp.ok:
+                        with open(filepath, 'wb') as fd:
+                            fd.write(resp.content)
+
+                data_cache.save()
+
+        # pieces
+        for type_dossier_id in types_dossier_ids.keys():
+            for objet_demande_id in objets_demande_ids.keys():
+                resp = client.service.GetPieces(self.get_token(), type_dossier_id, objet_demande_id)
+                if resp is None:
+                    continue
+                data_cache, created = CartaDSDataCache.objects.get_or_create(
+                        data_type='pieces',
+                        data_parameters={
+                            'type_dossier_id': type_dossier_id,
+                            'objet_demande_id': str(objet_demande_id),
+                            })
+                if resp is not None:
+                    data_cache.data_values = {'data': [
+                        {'id': str(x['IdPiece']),
+                            'text': x['Libelle'],
+                            'description': x['Descriptif'],
+                            'codePiece': x['CodePiece'],
+                            'reglementaire': x['Reglementaire'],
+                            'files': [],
+                            'max_files': 6,
+                            } for x in resp]}
+                data_cache.save()
+
+    def hourly(self):
+        super(AbstractCartaDSCS, self).hourly()
+        if CartaDSDataCache.objects.count() == 0:
+            # don't wait for daily job to get initial data
+            self.update_data_cache()
+
+    def daily(self):
+        super(AbstractCartaDSCS, self).daily()
+        self.update_data_cache()
 
-    @endpoint(description=_('Get lisf of file types'),
+    @endpoint(description=_('Get list of collectivities'))
+    def communes(self, request):
+        cache = CartaDSDataCache.objects.get(data_type='communes')
+        return cache.data_values
+
+    @endpoint(description=_('Get list of file types'),
               parameters={
                   'commune_id': COMMUNE_ID_PARAM
               })
     def types_dossier(self, request, commune_id):
-        client = self.soap_client()
-        resp = client.service.GetTypesDossier(self.get_token(), int(commune_id))
-        return {'data': [{'id': x['Key'], 'text': x['Value']} for x in resp]}
+        cache = CartaDSDataCache.objects.get(
+                    data_type='types_dossier',
+                    data_parameters={'commune_id': int(commune_id)})
+        return cache.data_values
 
     @endpoint(description=_('Get list of demand subjects'),
               parameters={'type_dossier_id': TYPE_DOSSIER_ID_PARAM},
               )
     def objets_demande(self, request, type_dossier_id):
-        client = self.soap_client()
-        resp = client.service.GetObjetsDemande(self.get_token(), type_dossier_id)
-        return {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp or []]}
+        cache = CartaDSDataCache.objects.get(
+                    data_type='objets_demande',
+                    data_parameters={'type_dossier_id': type_dossier_id})
+        return cache.data_values
 
     @endpoint(description=_('Get list of CERFA documents'),
               parameters={
@@ -166,21 +286,25 @@ class AbstractCartaDSCS(BaseResource):
                   'type_compte': {'description': _('Type of account')},
               })
     def liste_pdf(self, request, type_dossier_id, type_compte=1):
-        client = self.soap_client()
-        resp = client.service.GetListePdf(self.get_token(), type_dossier_id,
-                {'TypeCompteUtilisateur': type_compte})
-        def format_cerfa_label(x):
-            try:
-                if x['Description']:
-                    return u'%(Nom)s: %(Description)s' % x
-            except KeyError:
-                pass
-            return u'%(Nom)s' % x
-        return {'data': [
-            {'id': x['Identifiant'],
-             'text': format_cerfa_label(x),
-             'url': x['UrlTelechargement'],
-            } for x in resp or []]}
+        cache = CartaDSDataCache.objects.get(
+                    data_type='liste_pdf',
+                    data_parameters={
+                        'type_dossier_id': type_dossier_id,
+                        'type_compte': type_compte,
+                        })
+        if request:  # point to local documents cache
+            for pdf in cache.data_values['data']:
+                pdf['url'] = request.build_absolute_uri(
+                        os.path.join(settings.MEDIA_URL, self.pdf_path(pdf)))
+        return cache.data_values
+
+    def pdf_path(self, pdf):
+        if '*' in pdf['id']:  # cerfa
+            filename = 'cerfa_%s.pdf' % pdf['id'].replace('*', '-')
+        else:
+            filename = '%s.pdf' % pdf['id']
+        return os.path.join('public/cartads_cs', self.slug, 'documents', filename)
+
 
     @endpoint(perm='can_access',
               description=_('Get list of file items'),
@@ -190,9 +314,13 @@ class AbstractCartaDSCS(BaseResource):
                   'tracking_code': TRACKING_CODE_PARAM,
               })
     def pieces(self, request, type_dossier_id, objet_demande_id, tracking_code):
-        client = self.soap_client()
-        resp = client.service.GetPieces(self.get_token(), type_dossier_id,
-                objet_demande_id)
+        cache, created = CartaDSDataCache.objects.get_or_create(
+                        data_type='pieces',
+                        data_parameters={
+                            'type_dossier_id': type_dossier_id,
+                            'objet_demande_id': objet_demande_id,
+                            })
+
         signer = Signer(salt='cart@ds_cs')
         upload_token = signer.sign(tracking_code)
         cerfa_pieces = [
@@ -214,15 +342,7 @@ class AbstractCartaDSCS(BaseResource):
              'max_files': 6,
             }
         ]
-        pieces = [
-            {'id': str(x['IdPiece']),
-             'text': x['Libelle'],
-             'description': x['Descriptif'],
-             'codePiece': x['CodePiece'],
-             'reglementaire': x['Reglementaire'],
-             'files': [],
-             'max_files': 6,
-            } for x in resp]
+        pieces = cache.data_values['data']
         required_pieces = [x for x in pieces if x['reglementaire']]
         if required_pieces:
             required_pieces[0]['section_start'] = 'Pièces réglementaires'
diff --git a/tests/test_cartads_cs.py b/tests/test_cartads_cs.py
index ff566b63..4de60980 100644
--- a/tests/test_cartads_cs.py
+++ b/tests/test_cartads_cs.py
@@ -3,6 +3,7 @@
 import datetime
 
 import mock
+from httmock import HTTMock
 import pytest
 
 from django.core.files.storage import default_storage
@@ -76,94 +77,95 @@ class FakeService():
             }]
 
 
-def test_communes(connector, app):
-    with mock.patch('passerelle.apps.cartads_cs.models.CartaDSCS.soap_client') as client:
-        client.return_value = mock.Mock(service=FakeService())
-        resp = app.get('/cartads-cs/test/communes')
-        assert resp.json == {'data': [{'text': 'AIGREFEUILLE SUR MAINE', 'id': '2'}], 'err': 0}
+def pdf_mock(url, request):
+    return {'content': '%PDF...', 'status_code': 200}
 
-def test_types_dossier(connector, app):
-    with mock.patch('passerelle.apps.cartads_cs.models.CartaDSCS.soap_client') as client:
-        client.return_value = mock.Mock(service=FakeService())
-        resp = app.get('/cartads-cs/test/types_dossier', status=400)
-        resp = app.get('/cartads-cs/test/types_dossier?commune_id=1')
-        assert resp.json == {'data': [{'id': 'CU', 'text': "Certificat d'urbanisme"}], 'err': 0}
 
-def test_objets_demande(connector, app):
-    with mock.patch('passerelle.apps.cartads_cs.models.CartaDSCS.soap_client') as client:
-        client.return_value = mock.Mock(service=FakeService())
-        resp = app.get('/cartads-cs/test/objets_demande?type_dossier_id=CU')
-        assert resp.json == {'data': [{'id': '1', 'text': "CU d'information"}], 'err': 0}
-
-def test_liste_pdf(connector, app):
-    with mock.patch('passerelle.apps.cartads_cs.models.CartaDSCS.soap_client') as client:
-        client.return_value = mock.Mock(service=FakeService())
-        resp = app.get('/cartads-cs/test/liste_pdf?type_dossier_id=CU')
-        assert resp.json == {'data': [{'id': '13410*04',
-                                       'text': "Cerfa 13410-04: Demande de Certificat d'urbanisme",
-                                       'url': 'https://invalid/adscs/webservices/ServicePDF.ashx?pdf=13410*04'}],
-                             'err': 0}
-
-def test_pieces_management(connector, app):
+@pytest.fixture
+def cached_data(connector, app):
     with mock.patch('passerelle.apps.cartads_cs.models.CartaDSCS.soap_client') as client:
         client.return_value = mock.Mock(service=FakeService())
-        resp = app.get('/cartads-cs/test/pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
-        data = resp.json['data']
-        assert len(data) == 4
-        assert data[0]['text'] == 'Cerfa rempli'
-        assert data[0]['max_files'] == 1
-        assert data[1]['text'] == u'Cerfa demandeurs complémentaires'
-        assert data[1]['max_files'] == 6
-        assert data[2]['text'] == 'Plan de situation du terrain'
-        assert data[2]['max_files'] == 6
-        assert data[3]['text'] == 'DECLARATION PREALABLE INCOMPLETE'
-        assert data[3]['max_files'] == 6
-        for piece in data:
-            assert len(piece['files']) == 1
-            assert piece['files'][0].keys() == ['url']
-
-        resp = app.get('/cartads-cs/test/check_pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
-        assert resp.json == {'result': False, 'err': 0}
-
-        resp = app.post(data[0]['files'][0]['url'],
-                        upload_files=[('files[]', 'test.pdf', '%PDF...')])
-        cerfa_token = resp.json[0]['token']
-
-        resp = app.get('/cartads-cs/test/pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
-        data = resp.json['data']
-        assert data[0]['files'][0]['name']
-
-        resp = app.post(data[0]['files'][0]['url'] + '%s/delete/' % cerfa_token)
-        assert resp.json == {'err': 0}
-
-        resp = app.get('/cartads-cs/test/pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
-        data = resp.json['data']
-        assert 'name' not in data[0]['files'][0]
-
-        resp = app.post(data[0]['files'][0]['url'],
-                        upload_files=[('files[]', 'test.pdf', '%PDF...')])
-
-        resp = app.post(data[1]['files'][0]['url'],
-                        upload_files=[('files[]', 'test.pdf', '%PDF...')])
-        resp = app.post(data[1]['files'][0]['url'],
-                        upload_files=[('files[]', 'test.pdf', '%PDF...')])
-        resp = app.get('/cartads-cs/test/pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
-        data = resp.json['data']
-        assert len(data[1]['files']) == 3
-
-        resp = app.get('/cartads-cs/test/check_pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
-        assert resp.json == {'result': False, 'err': 0}
-
-        resp = app.post(data[2]['files'][0]['url'],
-                        upload_files=[('files[]', 'test.pdf', '%PDF...')])
-
-        resp = app.get('/cartads-cs/test/check_pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
-        assert resp.json == {'result': True, 'err': 0}
-
-def test_send(connector, app):
+        with HTTMock(pdf_mock):
+            connector.hourly()
+
+def test_communes(connector, app, cached_data):
+    resp = app.get('/cartads-cs/test/communes')
+    assert resp.json == {'data': [{'text': 'AIGREFEUILLE SUR MAINE', 'id': '2'}], 'err': 0}
+
+def test_types_dossier(connector, app, cached_data):
+    resp = app.get('/cartads-cs/test/types_dossier', status=400)
+    resp = app.get('/cartads-cs/test/types_dossier?commune_id=2')
+    assert resp.json == {'data': [{'id': 'CU', 'text': "Certificat d'urbanisme"}], 'err': 0}
+
+def test_objets_demande(connector, app, cached_data):
+    resp = app.get('/cartads-cs/test/objets_demande?type_dossier_id=CU')
+    assert resp.json == {'data': [{'id': '1', 'text': "CU d'information"}], 'err': 0}
+
+def test_liste_pdf(connector, app, cached_data):
+    resp = app.get('/cartads-cs/test/liste_pdf?type_dossier_id=CU')
+    assert resp.json == {'data': [{'id': '13410*04',
+                                   'text': "Cerfa 13410-04: Demande de Certificat d'urbanisme",
+                                   'url': 'http://testserver/media/public/cartads_cs/test/documents/cerfa_13410-04.pdf'}],
+                         'err': 0}
+
+def test_pieces_management(connector, app, cached_data):
+    resp = app.get('/cartads-cs/test/pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
+    data = resp.json['data']
+    assert len(data) == 4
+    assert data[0]['text'] == 'Cerfa rempli'
+    assert data[0]['max_files'] == 1
+    assert data[1]['text'] == u'Cerfa demandeurs complémentaires'
+    assert data[1]['max_files'] == 6
+    assert data[2]['text'] == 'Plan de situation du terrain'
+    assert data[2]['max_files'] == 6
+    assert data[3]['text'] == 'DECLARATION PREALABLE INCOMPLETE'
+    assert data[3]['max_files'] == 6
+    for piece in data:
+        assert len(piece['files']) == 1
+        assert piece['files'][0].keys() == ['url']
+
+    resp = app.get('/cartads-cs/test/check_pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
+    assert resp.json == {'result': False, 'err': 0}
+
+    resp = app.post(data[0]['files'][0]['url'],
+                    upload_files=[('files[]', 'test.pdf', '%PDF...')])
+    cerfa_token = resp.json[0]['token']
+
+    resp = app.get('/cartads-cs/test/pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
+    data = resp.json['data']
+    assert data[0]['files'][0]['name']
+
+    resp = app.post(data[0]['files'][0]['url'] + '%s/delete/' % cerfa_token)
+    assert resp.json == {'err': 0}
+
+    resp = app.get('/cartads-cs/test/pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
+    data = resp.json['data']
+    assert 'name' not in data[0]['files'][0]
+
+    resp = app.post(data[0]['files'][0]['url'],
+                    upload_files=[('files[]', 'test.pdf', '%PDF...')])
+
+    resp = app.post(data[1]['files'][0]['url'],
+                    upload_files=[('files[]', 'test.pdf', '%PDF...')])
+    resp = app.post(data[1]['files'][0]['url'],
+                    upload_files=[('files[]', 'test.pdf', '%PDF...')])
+    resp = app.get('/cartads-cs/test/pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
+    data = resp.json['data']
+    assert len(data[1]['files']) == 3
+
+    resp = app.get('/cartads-cs/test/check_pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
+    assert resp.json == {'result': False, 'err': 0}
+
+    resp = app.post(data[2]['files'][0]['url'],
+                    upload_files=[('files[]', 'test.pdf', '%PDF...')])
+
+    resp = app.get('/cartads-cs/test/check_pieces?type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB')
+    assert resp.json == {'result': True, 'err': 0}
+
+def test_send(connector, app, cached_data):
     CartaDSFile.objects.all().delete()
     Job.objects.all().delete()
-    test_pieces_management(connector, app)
+    test_pieces_management(connector, app, cached_data)
     resp = app.get('/cartads-cs/test/send?commune_id=1&type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB&email=test@invalid')
     assert CartaDSDossier.objects.all().count() == 1
     dossier = CartaDSDossier.objects.all().first()
@@ -199,10 +201,10 @@ def test_send(connector, app):
     assert dossier.cartads_id_dossier == '135792'
     assert dossier.cartads_numero_dossier == 'CU 044 043 19 A0006'
 
-def test_send_notification_error(connector, app):
+def test_send_notification_error(connector, app, cached_data):
     CartaDSFile.objects.all().delete()
     Job.objects.all().delete()
-    test_pieces_management(connector, app)
+    test_pieces_management(connector, app, cached_data)
     resp = app.get('/cartads-cs/test/send?commune_id=1&type_dossier_id=CU&objet_demande_id=1&tracking_code=BBBBBBBB&email=test@invalid')
     assert CartaDSDossier.objects.all().count() == 1
     dossier = CartaDSDossier.objects.all().first()
@@ -240,9 +242,9 @@ def test_send_notification_error(connector, app):
     assert dossier.notification_message is not None
 
 
-def test_status(connector, app):
+def test_status(connector, app, cached_data):
     CartaDSDossier.objects.all().delete()
-    test_send(connector, app)
+    test_send(connector, app, cached_data)
     dossier = CartaDSDossier.objects.all()[0]
 
     with mock.patch('passerelle.apps.cartads_cs.models.CartaDSCS.soap_client') as client:
@@ -251,9 +253,9 @@ def test_status(connector, app):
         assert resp.json['status_label'] == 'En cours de saisie'
 
 
-def test_status_error(connector, app):
+def test_status_error(connector, app, cached_data):
     CartaDSDossier.objects.all().delete()
-    test_send_notification_error(connector, app)
+    test_send_notification_error(connector, app, cached_data)
     dossier = CartaDSDossier.objects.all()[0]
 
     resp = app.get('/cartads-cs/test/status?dossier_id=%s' % dossier.id)
-- 
2.23.0

