0004-toulouse_smart-add-an-add-media-endpoint-57875.patch

Nicolas Roche, 02 novembre 2021 13:15

Voir les différences: en ligne côte à côte

Subject: [PATCH 4/5] toulouse_smart: add an add-media endpoint (#57875)

 .../migrations/0004_wcsrequestfile.py         |  36 ++++
 passerelle/contrib/toulouse_smart/models.py   |  91 ++++++++-
 passerelle/contrib/toulouse_smart/schemas.py  |  38 ++++
 passerelle/views.py                           |   2 -
 tests/data/toulouse_smart/201x201.jpg         | Bin 0 -> 795 bytes
 tests/test_toulouse_smart.py                  | 190 +++++++++++++++++-
 6 files changed, 346 insertions(+), 11 deletions(-)
 create mode 100644 passerelle/contrib/toulouse_smart/migrations/0004_wcsrequestfile.py
 create mode 100644 tests/data/toulouse_smart/201x201.jpg

     # Generated by Django 2.2.19 on 2021-10-29 15:45
     import django.db.models.deletion
     from django.db import migrations, models
     import passerelle.contrib.toulouse_smart.models
     class Migration(migrations.Migration):
         dependencies = [
             ('toulouse_smart', '0003_smartrequest'),
+        ]
         operations = [
             migrations.CreateModel(
                 name='WcsRequestFile',
                 fields=[
+                    (
                         'id',
                         models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
                     ),
                     ('filename', models.CharField(max_length=256)),
                     ('content_type', models.CharField(max_length=256)),
                     ('content', models.FileField(upload_to=passerelle.contrib.toulouse_smart.models.upload_to)),
+                    (
                         'resource',
                         models.ForeignKey(
                             on_delete=django.db.models.deletion.CASCADE,
                             related_name='files',
                             to='toulouse_smart.WcsRequest',
                         ),
                     ),
                 ],
             ),
+        ]

     # This program is distributed in the hope that it will be useful,
     # but WITHOUT ANY WARRANTY; without even the implied warranty of
     # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     # GNU Affero General Public License for more details.
+    #
     # You should have received a copy of the GNU Affero General Public License
     # along with this program.  If not, see <http://www.gnu.org/licenses/>.
     import base64
     import datetime
     import json
     from uuid import uuid4
     import lxml.etree as ET
     from django.conf import settings
     from django.contrib.postgres.fields import JSONField
     from django.core.files.base import ContentFile
     from django.db import models
     from django.db.transaction import atomic
     from django.urls import reverse
     from django.utils.six.moves.urllib import parse as urlparse
     from django.utils.text import slugify
     from django.utils.timezone import now
     from django.utils.translation import ugettext_lazy as _
     from requests import RequestException
-...
                 raise APIError(
                     "'%s' intervention already created" % post_data['external_number'],
                     http_status=400,
+                )
             wcs_request = self.wcs_requests.create(
                 wcs_form_api_url=post_data['form_api_url'],
                 wcs_form_number=post_data['external_number'],
+            )
             update_intervention_endpoint_url = request.build_absolute_uri(
                 reverse(
                     'generic-endpoint',
                     kwargs={'connector': 'toulouse-smart', 'endpoint': 'update-intervention', 'slug': self.slug},
             endpoint_url = {}
             for endpoint_name in 'update-intervention', 'add-media':
                 endpoint_url[endpoint_name] = request.build_absolute_uri(
                     reverse(
                         'generic-endpoint',
                         kwargs={'connector': 'toulouse-smart', 'endpoint': endpoint_name, 'slug': self.slug},
+                    )
+                )
+            )
             wcs_request.payload = {
                 'description': post_data['description'],
                 'cityId': post_data['cityId'],
                 'interventionCreated': post_data['interventionCreated'] + 'Z',
                 'interventionDesired': post_data['interventionDesired'] + 'Z',
                 'submitterFirstName': post_data['submitterFirstName'],
                 'submitterLastName': post_data['submitterLastName'],
                 'submitterMail': post_data['submitterMail'],
-...
                 'address': post_data['address'],
                 'interventionData': data,
                 'geom': {
                     'type': 'Point',
                     'coordinates': [post_data['lon'], post_data['lat']],
                     'crs': 'EPSG:4326',
                 },
                 'interventionTypeId': intervention_type['id'],
                 'notificationUrl': '%s?uuid=%s' % (update_intervention_endpoint_url, wcs_request.uuid),
                 'notificationUrl': '%s?uuid=%s' % (endpoint_url['update-intervention'], wcs_request.uuid),
                 'add_media_url': '%s?uuid=%s' % (endpoint_url['add-media'], wcs_request.uuid),
+            }
             for label in 'checkDuplicated', 'onPrivateLand', 'safeguardRequired':
                 if str(post_data.get(label)).lower() in ['true', 'oui', '1']:
                     wcs_request.payload[label] = 'true'
             wcs_request.save()
             if not wcs_request.push():
                 self.add_job(
                     'create_intervention_job',
-...
+            }
             smart_request = wcs_request.smart_requests.create(payload=payload)
             self.add_job(
                 'update_intervention_job',
                 id=smart_request.id,
                 natural_id='smart-request-%s' % smart_request.id,
+            )
         @endpoint(
             name='add-media',
             methods=['post'],
             description=_('Add a media'),
             parameters={
                 'uuid': {'description': _('Notification identifier')},
             },
             perm='can_access',
             post={'request_body': {'schema': {'application/json': schemas.MEDIA_SCHEMA}}},
+        )
         def add_media(self, request, uuid, post_data):
             try:
                 wcs_request = self.wcs_requests.get(uuid=uuid)
             except WcsRequest.DoesNotExist:
                 raise APIError("Cannot find intervention '%s'" % uuid, http_status=400)
             nb_registered = 0
             for media in post_data['files']:
                 if not media:
                     # silently ignore empty payload value
                     continue
                 wcs_request_file = wcs_request.files.create(
                     filename=media['filename'], content_type=media['content_type']
+                )
                 with ContentFile(base64.b64decode(media['content'])) as media_content:
                     wcs_request_file.content.save(media['filename'], media_content)
                 self.add_job(
                     'add_media_job',
                     id=wcs_request_file.id,
                     natural_id='wcs-request-file-%s' % wcs_request_file.id,
+                )
                 nb_registered += 1
             return {'data': {'uuid': wcs_request.uuid, 'nb_registered': nb_registered}}
         def add_media_job(self, *args, **kwargs):
             wcs_request_file = WcsRequestFile.objects.get(id=kwargs['id'])
             wcs_request = wcs_request_file.resource
             if not wcs_request.result or not wcs_request.result.get('id'):
                 raise SkipJob(datetime.timedelta(minutes=10))
             if not wcs_request_file.push():
                 raise SkipJob()
         @atomic
         @endpoint(
             name='update-intervention',
             methods=['post'],
             description=_('Update an intervention status'),
             parameters={
                 'uuid': {'description': _('Notification identifier')},
             },
-...
                 self.result = err_desc
                 self.save()
                 return False
             self.status = 'sent'
             self.save()
             return True
     def upload_to(wcs_request_file, filename):
         instance = wcs_request_file.resource.resource
         uuid = wcs_request_file.resource.uuid
         return '%s/%s/%s/%s' % (instance.get_connector_slug(), instance.slug, uuid, filename)
     class WcsRequestFile(models.Model):
         resource = models.ForeignKey(
             to=WcsRequest,
             on_delete=models.CASCADE,
             related_name='files',
+        )
         filename = models.CharField(max_length=256)
         content_type = models.CharField(max_length=256)
         content = models.FileField(upload_to=upload_to)
         def push(self):
             wcs_request = self.resource
             intervention_id = wcs_request.result.get('id')
             instance = wcs_request.resource
             url = '%sv1/intervention/%s/media' % (instance.webservice_base_url, intervention_id)
             files = {'media': (self.filename, self.content.open('rb'), self.content_type)}
             try:
                 instance.request(url, method='PUT', files=files)
             except APIError as e:
                 return False
             self.content.delete()
             return True
     class SmartRequest(models.Model):
         resource = models.ForeignKey(
             to=WcsRequest,
             on_delete=models.CASCADE,
             related_name='smart_requests',
+        )
         payload = JSONField()
         result = JSONField(null=True)

                         'type': 'string',
                     },
                 },
                 'required': ['status'],
             },
         },
         'required': ['data'],
+    }
     MEDIA_SCHEMA = {
         '$schema': 'http://json-schema.org/draft-04/schema#',
         'type': 'object',
         'properties': {
             'files': {
                 'type': 'array',
                 'items': {
                     'oneOf': [
+                        {
                             'type': 'null',
                         },
+                        {
                             'type': 'object',
                             'properties': {
                                 'filename': {
                                     'description': "Nom du ficher",
                                     'type': 'string',
                                 },
                                 'content_type': {
                                     'description': "Type MIME",
                                     'type': 'string',
                                 },
                                 'content': {
                                     'description': "Contenu",
                                     'type': 'string',
                                 },
                             },
                             'required': ['filename', 'content_type', 'content'],
                         },
                     ],
                 },
             },
         },
         'required': ['files'],
         'unflatten': True,
+    }

                     except ValueError:
                         raise InvalidParameterValue(parameter)
                 elif parameter_info.get('type') == 'float':
                     d[parameter] = d[parameter].replace(',', '.')
                     try:
                         d[parameter] = float(d[parameter])
                     except ValueError:
                         raise InvalidParameterValue(parameter)
             if request.method == 'POST' and self.endpoint.endpoint_info.post:
                 request_body = self.endpoint.endpoint_info.post.get('request_body', {})
                 if 'application/json' in request_body.get('schema', {}):
                     json_schema = request_body['schema']['application/json']
                     must_unflatten = hasattr(json_schema, 'items') and json_schema.get('unflatten', False)
                     merge_extra = hasattr(json_schema, 'items') and json_schema.get('merge_extra', False)
                     pre_process = hasattr(json_schema, 'items') and json_schema.get('pre_process')
                     try:
-...
                         data.update(data.pop('extra', {}))
                     if pre_process is not None:
                         pre_process(self.endpoint.__self__, data)
                     # disable validation on description and title in order to allow lazy translation strings
                     validator = validators.validator_for(json_schema)
                     validator.META_SCHEMA['properties'].pop('description', None)
                     validator.META_SCHEMA['properties'].pop('title', None)
                     try:
                         validate(data, json_schema)
                     except ValidationError as e:
                         error_msg = e.message
                         if e.path:
                             error_msg = '%s: %s' % ('/'.join(map(str, e.path)), error_msg)
                         raise APIError(error_msg, http_status=400)
                     d['post_data'] = data

     # This program is distributed in the hope that it will be useful,
     # but WITHOUT ANY WARRANTY; without even the implied warranty of
     # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     # GNU Affero General Public License for more details.
+    #
     # You should have received a copy of the GNU Affero General Public License
     # along with this program.  If not, see <http://www.gnu.org/licenses/>.
     import base64
     import cgi
     import functools
     import io
     import json
     import os
     import uuid
     import zipfile
     from copy import deepcopy
     import httmock
     import lxml.etree as ET
     import mock
     import pytest
     import utils
     from django.utils.encoding import force_text
     from requests.exceptions import ReadTimeout
     from test_manager import login
     from passerelle.base.models import Job
     from passerelle.contrib.toulouse_smart.models import SmartRequest, ToulouseSmartResource, WcsRequest
     from passerelle.contrib.toulouse_smart.models import (
         SmartRequest,
         ToulouseSmartResource,
         WcsRequest,
         WcsRequestFile,
+    )
     TEST_BASE_DIR = os.path.join(os.path.dirname(__file__), 'data', 'toulouse_smart')
     @pytest.fixture
     def smart(db):
         return utils.make_resource(
             ToulouseSmartResource,
-...
         def decorator(func):
             @httmock.urlmatch()
             def error(url, request):
                 assert False, 'request to %s' % url.geturl()
             def register(path, payload, content, status_code=200, exception=None):
                 @httmock.urlmatch(path=path)
                 def handler(url, request):
                     if payload and json.loads(request.body) != payload:
                         assert False, 'wrong payload sent to request to %s' % url.geturl()
                     if payload:
                         ctype, pdict = cgi.parse_header(request.headers["content-type"])
                         if ctype == 'multipart/form-data':
                             # here payload is an expected multipart contents list
                             pdict["boundary"] = bytes(pdict["boundary"], "utf-8")
                             pdict['CONTENT-LENGTH'] = request.headers['Content-Length']
                             postvars = cgi.parse_multipart(io.BytesIO(request.body), pdict)
                             for i, media_content in enumerate(postvars['media']):
                                 assert media_content == payload[i], (
                                     'wrong multipart content sent to %s' % url.geturl()
+                                )
                         else:
                             assert json.loads(request.body) == payload, (
                                 'wrong payload sent to request to %s' % url.geturl()
+                            )
                     if exception:
                         raise exception
                     return httmock.response(status_code, content)
                 return handler
             @functools.wraps(func)
             def wrapper(*args, **kwargs):
-...
         return decorator
     def get_json_file(filename):
         with open(os.path.join(TEST_BASE_DIR, "%s.json" % filename)) as desc:
             return desc.read()
     def get_media_file(filename):
         with open(os.path.join(TEST_BASE_DIR, "%s" % filename), 'rb') as desc:
             return desc.read()
     @mock_response(['/v1/type-intervention', None, b'<List></List>'])
     def test_empty_intervention_types(smart):
         assert smart.get_intervention_types() == []
     INTERVENTION_TYPES = '''<List>
        <item>
            <id>1234</id>
-...
     CREATE_INTERVENTION_PAYLOAD = {
         'fields': FIELDS_PAYLOAD,
         'extra': CREATE_INTERVENTION_PAYLOAD_EXTRA,
+    }
     UUID = uuid.UUID('12345678123456781234567812345678')
     CREATE_INTERVENTION_QUERY = {
         'add_media_url': 'http://testserver/toulouse-smart/test/add-media?uuid=%s' % str(UUID),
         'description': 'coin coin',
         'cityId': '12345',
         'interventionCreated': '2021-06-30T16:08:05Z',
         'interventionDesired': '2021-06-30T16:08:05Z',
         'submitterFirstName': 'John',
         'submitterLastName': 'Doe',
         'submitterMail': 'john.doe@example.com',
         'submitterPhone': '0123456789',
-...
         smart.jobs()
         job = Job.objects.get(method_name='update_intervention_job')
         assert job.status == 'registered'
         assert job.update_timestamp > job.creation_timestamp
         smart_request = smart.wcs_requests.get(uuid=UUID).smart_requests.get()
         assert smart_request.result == None
     ADD_MEDIA_PAYLOAD = {
         'files/0': {
             'filename': '201x201.jpg',
             'content_type': 'image/jpeg',
             'content': force_text(base64.b64encode(get_media_file('201x201.jpg'))),
         },
         'files/1': None,
+    }
     ADD_MEDIA_QUERY = [get_media_file('201x201.jpg')]
     @mock_response(
         ['/v1/type-intervention', None, INTERVENTION_TYPES],
         ['/v1/intervention/%s/media' % INTERVENTION_ID, ADD_MEDIA_QUERY, 200],
         ['/v1/intervention', CREATE_INTERVENTION_QUERY, get_json_file('create_intervention')],
+    )
     @mock.patch("django.db.models.fields.UUIDField.get_default", return_value=UUID)
     def test_add_media(mocked_uuid, app, smart):
         resp = app.post_json(URL + 'create-intervention/', params=CREATE_INTERVENTION_PAYLOAD)
         assert not resp.json['err']
         url = resp.json['data']['payload']['add_media_url']
         url = URL + 'add-media?uuid=%s' % str(UUID)
         resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD)
         assert not resp.json['err']
         assert resp.json['data']['uuid'] == str(UUID)
         assert resp.json['data']['nb_registered'] == 1
         assert Job.objects.count() == 1
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'registered'
         wcs_request = smart.wcs_requests.get(uuid=UUID)
         wcs_request_file = wcs_request.files.get(**job.parameters)
         path = wcs_request_file.content.path
         assert os.path.isfile(path)
         with wcs_request_file.content.open('rb') as desc:
             assert desc.read() == get_media_file('201x201.jpg')
         # smart not responding
         mocked_push = mock.patch(
             "passerelle.contrib.toulouse_smart.models.WcsRequestFile.push",
             return_value=False,
+        )
         mocked_push.start()
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'registered'
         # smart responding
         mocked_push.stop()
         smart.jobs()
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'completed'
         wcs_request_file = wcs_request.files.get(**job.parameters)
         with pytest.raises(ValueError, match='no file associated'):
             assert not wcs_request_file.content.path
         assert not os.path.isfile(path)
     def test_add_media_wrong_uuid(app, smart):
         with pytest.raises(WcsRequest.DoesNotExist):
             smart.wcs_requests.get(uuid=UUID)
         url = URL + 'add-media?uuid=%s' % str(UUID)
         resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD, status=400)
         assert resp.json['err']
         assert 'Cannot find intervention' in resp.json['err_desc']
         assert WcsRequestFile.objects.count() == 0
     @mock_response(
         ['/v1/type-intervention', None, INTERVENTION_TYPES],
         ['/v1/intervention/%s/media' % json.loads(get_json_file('create_intervention'))['id'], None, None, 500],
         ['/v1/intervention', CREATE_INTERVENTION_QUERY, get_json_file('create_intervention')],
+    )
     @mock.patch("django.db.models.fields.UUIDField.get_default", return_value=UUID)
     def test_add_media_error(mocked_uuid, app, freezer, smart):
         resp = app.post_json(URL + 'create-intervention/', params=CREATE_INTERVENTION_PAYLOAD)
         assert not resp.json['err']
         freezer.move_to('2021-10-30 00:00:00')
         url = resp.json['data']['payload']['add_media_url']
         resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD)
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'registered'
         freezer.move_to('2021-10-30 00:00:03')
         smart.jobs()
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'registered'
         assert job.update_timestamp > job.creation_timestamp
     @mock_response(
         ['/v1/type-intervention', None, INTERVENTION_TYPES],
         ['/v1/intervention/%s/media' % INTERVENTION_ID, None, None, None, ReadTimeout('timeout')],
         ['/v1/intervention', CREATE_INTERVENTION_QUERY, get_json_file('create_intervention')],
+    )
     @mock.patch("django.db.models.fields.UUIDField.get_default", return_value=UUID)
     def test_add_media_timeout_error(mocked_uuid, app, freezer, smart):
         resp = app.post_json(URL + 'create-intervention/', params=CREATE_INTERVENTION_PAYLOAD)
         assert not resp.json['err']
         freezer.move_to('2021-10-30 00:00:00')
         url = resp.json['data']['payload']['add_media_url']
         resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD)
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'registered'
         freezer.move_to('2021-10-30 00:00:03')
         smart.jobs()
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'registered'
         assert job.update_timestamp > job.creation_timestamp
     UPDATE_INTERVENTION_QUERY_ON_ASYNC_CREATION = {
         'creation_response': {
             'wcs_form_api_url': CREATE_INTERVENTION_PAYLOAD_EXTRA['form_api_url'],
             'wcs_form_number': CREATE_INTERVENTION_PAYLOAD_EXTRA['external_number'],
             'uuid': str(UUID),
             'payload': CREATE_INTERVENTION_QUERY,
             'result': json.loads(get_json_file('create_intervention')),
             'status': 'sent',
-...
         mocked_smart_request_push.stop()
         smart.jobs()
         job = Job.objects.get(method_name='update_intervention_job')
         assert job.status == 'completed'
         smart_request = wcs_request.smart_requests.get()
         assert smart_request.payload['creation_response']['uuid'] == str(UUID)
         assert smart_request.payload['creation_response']['result']['id'] == INTERVENTION_ID
     @mock_response(
         ['/v1/type-intervention', None, INTERVENTION_TYPES],
         ['/v1/intervention/%s/media' % INTERVENTION_ID, ADD_MEDIA_QUERY, 200],
         ['/v1/intervention', CREATE_INTERVENTION_QUERY, get_json_file('create_intervention')],
+    )
     @mock.patch("django.db.models.fields.UUIDField.get_default", return_value=UUID)
     def test_add_media_async(mocked_uuid4, app, smart, freezer):
         mocked_wcs_request_push = mock.patch(
             "passerelle.contrib.toulouse_smart.models.WcsRequest.push",
             return_value=False,
+        )
         # smart is down
         freezer.move_to('2021-10-30 00:00:00')
         mocked_wcs_request_push.start()
         resp = app.post_json(URL + 'create-intervention/', params=CREATE_INTERVENTION_PAYLOAD)
         assert not resp.json['err']
         url = resp.json['data']['payload']['add_media_url']
         resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD)
         smart.jobs()
         job = Job.objects.get(method_name='create_intervention_job')
         assert job.status == 'registered'
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'registered'
         assert str(job.after_timestamp) == '2021-10-30 00:10:00+00:00'
         # smart is up
         freezer.move_to('2021-10-30 00:00:03')
         mocked_wcs_request_push.stop()
         smart.jobs()
         job = Job.objects.get(method_name='create_intervention_job')
         assert job.status == 'completed'
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'registered'
         # 10 minutes later
         freezer.move_to('2021-10-30 00:10:03')
         smart.jobs()
         job = Job.objects.get(method_name='add_media_job')
         assert job.status == 'completed'
+    -

Projet

Général

Profil

Produits Entr'ouvert » Passerelle

0004-toulouse_smart-add-an-add-media-endpoint-57875.patch