Produits Entr'ouvert » Welco

    python-requests (>= 2.11),

        'python-dateutil',

# welco - multichannel request processing

# Copyright (C) 2015  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import pytest

import django_webtest

@pytest.fixture

def app(request):

    wtm = django_webtest.WebTestMixin()

    wtm._patch_settings()

    request.addfinalizer(wtm._unpatch_settings)

    return django_webtest.DjangoTestApp()

@pytest.fixture

def user(db):

    from django.contrib.auth.models import User

    user = User.objects.create(username='toto')

    user.set_password('toto')

    user.save()

    return user

@pytest.fixture

def mail_group(db, settings, user):

    from django.contrib.auth.models import Group

    # add mail group to default user

    group = Group.objects.create(name='mail')

    user.groups.add(group)

    # define authorization of mail group on mail channel

    channel_roles = getattr(settings, 'CHANNEL_ROLES', {})

    mail_roles = channel_roles.setdefault('mail', [])

    mail_roles.append('mail')

    return group

# welco - multichannel request processing

# Copyright (C) 2015  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json

import pytest

from httmock import urlmatch, HTTMock

class BaseMock(object):

    def __init__(self, netloc):

        self.netloc = netloc

        self.clear()

    def clear(self):

        self.requests = []

        self.responses = []

    def next_response(self):

        response, self.responses = self.responses[0], self.responses[1:]

        return response

    @property

    def ctx_manager(self):

        '''Create an HTTMock context manager for all endpoints of a mocked Maarch instance'''

        endpoints = []

        for attribute, value in self.__class__.__dict__.items():

            if hasattr(value, 'path'):

                value = getattr(self, attribute)

                match_decorator = urlmatch(netloc=self.netloc, path=value.path)

                print value, self.netloc, '^/rest' + value.path

                endpoints.append(match_decorator(value))

        return HTTMock(*endpoints)

class MaarchMock(BaseMock):

    def list_endpoint(self, url, request):

        self.requests.append(('list_endpoint', url, request, json.loads(request.body)))

        return {

            'content': json.dumps(self.next_response()),

            'headers': {

                'content-type': 'application/json',

            },

        }

    list_endpoint.path = '^/rest/res/list$'

    def update_external_infos(self, url, request):

        self.requests.append(('update_external_infos', url, request, json.loads(request.body)))

        return json.dumps({})

    update_external_infos.path = '^/rest/res/externalInfos$'

    def update_status(self, url, request):

        self.requests.append(('update_status', url, request, json.loads(request.body)))

        return {

            'content': json.dumps(self.next_response()),

            'headers': {

                'content-type': 'application/json',

            },

        }

    update_status.path = '^/rest/res/resource/status$'

    def post_courrier(self, url, request):

        self.requests.append(('post_courrier', url, request, json.loads(request.body)))

    post_courrier.path = '^/rest/res$'

@pytest.fixture

def maarch(settings, mail_group):

    # configure maarch server

    settings.MAARCH_FEED = {

        'ENABLE': True,

        'URL': 'http://maarch.example.net/',

        'USERNAME': 'admin',

        'PASSWORD': 'admin',

    }

    return MaarchMock('maarch.example.net')

class WcsMock(BaseMock):

    def api_formdefs(self, url, request):

        return json.dumps({

            'data': [{

                'slug': 'slug1',

                'title': 'title1',

            }]

        })

    api_formdefs.path = '^/api/formdefs/$'

    def json(self, url, request):

        return json.dumps({

            'data': [{

                'slug': 'slug1',

                'title': 'title1',

                'category': 'category1',

            }]

        })

    json.path = '^/json$'

    def api_formdefs_slug1_schema(self, url, request):

        return json.dumps({

        })

    api_formdefs_slug1_schema.path = '^/api/formdefs/slug-1/schema$'

    def api_formdefs_slug1_submit(self, url, request):

        return json.dumps({

            'err': 0,

            'data': {

                'id': 1,

                'backoffice_url': 'http://wcs.example.net/slug-1/1',

            },

        })

    api_formdefs_slug1_submit.path = '^/api/formdefs/slug-1/submit$'

@pytest.fixture

def wcs(settings):

    settings.KNOWN_SERVICES = {

        'wcs': {

            'demarches': {

                'url': 'http://wcs.example.net/',

            }

        }

    }

    return WcsMock('wcs.example.net')

def test_utils(maarch):

    from welco.sources.mail.utils import get_maarch

    welco_maarch_obj = get_maarch()

    assert welco_maarch_obj.url == 'http://maarch.example.net/'

    assert welco_maarch_obj.username == 'admin'

    assert welco_maarch_obj.password == 'admin'

    assert welco_maarch_obj.grc_status == 'GRC'

    assert welco_maarch_obj.grc_received_status == 'GRC_TRT'

    assert welco_maarch_obj.grc_send_status == 'GRCSENT'

    assert welco_maarch_obj.grc_refused_status == 'GRCREFUSED'

PDF_MOCK = '%PDF-1.4 ...'

def test_feed(app, maarch, wcs, user):

    import base64

    from django.core.management import call_command

    from django.contrib.contenttypes.models import ContentType

    from welco.sources.mail.models import Mail

    app.set_user(user.username)

    response = app.get('/').follow()

    # no mail

    assert len(response.pyquery('li[data-registered-mail-number]')) == 0

    # feed mails from maarch

    with maarch.ctx_manager:

        # list request

        maarch.responses.append({

            'resources': [

                {

                    'res_id': 1,

                    'fileBase64Content': base64.b64encode(PDF_MOCK),

                }

            ],

        })

        # update status request

        maarch.responses.append({})

        # last list request

        maarch.responses.append({'resources': []})

        call_command('feed_mail_maarch')

        assert len(maarch.requests) == 3

        assert maarch.requests[0][3] == {

            'select': '*',

            'clause': "status='GRC'",

            'withFile': True,

            'orderBy': ['res_id'],

            'limit': 10,

        }

        assert maarch.requests[1][3] == {

            'resId': [1],

            'status': 'GRC_TRT',

        }

        assert maarch.requests[2][3] == {

            'select': '*',

            'clause': "status='GRC'",

            'withFile': True,

            'orderBy': ['res_id'],

            'limit': 10,

        }

    response = app.get('/').follow()

    # new mail is visible

    assert len(response.pyquery('li[data-registered-mail-number]')) == 1

    assert len(response.pyquery('li[data-registered-mail-number=maarch-1]')) == 1

    # start qualification

    maarch.clear()

    pk = Mail.objects.get().pk

    with wcs.ctx_manager, maarch.ctx_manager:

        source_type = str(ContentType.objects.get_for_model(Mail).pk),

        source_pk = str(pk)

        response = app.get('/ajax/qualification', params={

            'source_type': source_type,

            'source_pk': source_pk,

        })

        assert len(response.pyquery('a[data-association-pk]')) == 0

        response = app.post('/ajax/qualification', params={

            'source_type': source_type,

            'source_pk': str(pk),

            'formdef_reference': 'demarches:slug-1',

        })

        # verify qualification was done

        assert len(response.pyquery('a[data-association-pk]')) == 1

        association_pk = response.pyquery('a[data-association-pk]')[0].attrib['data-association-pk']

        response = app.post('/ajax/create-formdata/%s' % association_pk)

    assert len(maarch.requests) == 1

    assert maarch.requests[0][3] == {

        'status': 'GRCSENT',

        'externalInfos': [

            {

                'external_id': 1,

                'external_link': 'http://wcs.example.net/slug-1/1',

                'res_id': 1,

            }

        ]

    }

    def ready(self):

        from welco.qualif.models import Association

        from django.db.models import signals

        signals.post_save.connect(self.association_post_save,

            sender=Association)

    def association_post_save(self, sender, instance, **kwargs):

        from .utils import get_maarch

        if not instance.formdata_id:

            return

        source = instance.source

        if not hasattr(source, 'registered_mail_number'):

            return

        registered_mail_number = source.registered_mail_number

        if not registered_mail_number.startswith('maarch-'):

            return

        maarch_pk = int(registered_mail_number.split('-', 1)[-1])

        maarch = get_maarch()

        maarch.set_grc_sent_status(

            mail_pk=maarch_pk,

            formdata_id=instance.formdata_id,

            formdata_url_backoffice=instance.formdata_url_backoffice)

# welco - multichannel request processing

# Copyright (C) 2015  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import urlparse

import base64

from dateutil.parser import parse as parse_datetime

import requests

from requests.adapters import HTTPAdapter

from requests.packages.urllib3.util.retry import Retry

class MaarchError(Exception):

    pass

class MaarchCourrier(object):

    url = None

    username = None

    password = None

    default_limit = 100

    max_retries = 3

    def __init__(self, url, username, password):

        self.url = url

        self.username = username

        self.password = password

    def __repr__(self):

        return '<MaarchCourrier url:%s>' % self.url

    class Courrier(object):

        content = None

        format = None

        status = None

        def __init__(self, maarch_courrier, **kwargs):

            self.maarch_courrier = maarch_courrier

            self.pk = kwargs.pop('res_id', None)

            # decode file content

            if 'fileBase64Content' in kwargs:

                kwargs['content'] = base64.b64decode(kwargs.pop('fileBase64Content'))

            # decode date fields

            for key in kwargs:

                if key.endswith('_date') and kwargs[key]:

                    kwargs[key] = parse_datetime(kwargs[key])

            self.__dict__.update(kwargs)

        def __repr__(self):

            descriptions = []

            for key in ['pk', 'status']:

                if getattr(self, key, None):

                    descriptions.append('%s:%s' % (key, getattr(self, key)))

            return '<Courrier %s>' % ' '.join(descriptions)

        @classmethod

        def new_with_file(cls, maarch_courrier, content, format, status, **kwargs):

            if hasattr(content, 'read'):

                content = content.read()

            else:

                content = content

            return cls(maarch_courrier, content=content, format=format, status=status, **kwargs)

        def post_serialize(self):

            payload = {}

            assert self.content

            assert self.status

            payload['encodedFile'] = base64.b64encode(self.content)

            payload['collId'] = 'letterbox_coll'

            payload['table'] = 'res_letterbox'

            payload['fileFormat'] = self.fileFormat

            payload['data'] = d = []

            excluded_keys = ['content', 'format', 'status', 'maarch_courrier', 'pk']

            data = {key: self.__dict__[key] for key in self.__dict__ if key not in excluded_keys}

            if data:

                for key, value in data.iteritems():

                    if isinstance(value, basestring):

                        d.append({'column': key, 'value': value, 'type': 'string'})

                    elif isinstance(value, int):

                        d.append({'column': key, 'value': str(value), 'type': 'int'})

                    else:

                        raise NotImplementedError

            payload['status'] = self.status

            return payload

        def get_serialize(self):

            d = {'res_id': self.pk}

            for key in self.__dict__:

                if key in ['pk', 'maarch_courrier']:

                    continue

                value = getattr(self, key)

                if key == 'content':

                    value = base64.b64encode(value)

                    key = 'fileBase64Content'

                if key.endswith('_date'):

                    value = value.isoformat()

                d[key] = value

            return d

    def new_courrier_with_file(self, content, format, status, **kwargs):

        return self.Courrier.new_with_file(self, content, format, status, **kwargs)

    @property

    def session(self):

        s = requests.Session()

        if self.username and self.password:

            s.auth = (self.username, self.password)

        retry = Retry(

            total=self.max_retries,

            read=self.max_retries,

            connect=self.max_retries,

            backoff_factor=0.5,

            status_forcelist=(500, 502, 504)

        )

        adapter = HTTPAdapter(max_retries=retry)

        s.mount('http://', adapter)

        s.mount('https://', adapter)

        return s

    def post_json(self, url, payload, verb='post'):

        try:

            method = getattr(self.session, verb)

            response = method(url, json=payload)

        except requests.RequestException as e:

            raise MaarchError('HTTP request to maarch failed', e, payload)

        try:

            response.raise_for_status()

        except requests.RequestException as e:

            raise MaarchError('HTTP request to maarch failed', e, payload, repr(response.content[:1000]))

        try:

            response_payload = response.json()

        except ValueError:

            raise MaarchError('maarch returned non-JSON data', repr(response.content[:1000]), payload)

        return response_payload

    def put_json(self, url, payload):

        return self.post_json(url, payload, verb='put')

    @property

    def list_url(self):

        return urlparse.urljoin(self.url, 'rest/res/list')

    @property

    def update_external_infos_url(self):

        return urlparse.urljoin(self.url, 'rest/res/externalInfos')

    @property

    def update_status_url(self):

        return urlparse.urljoin(self.url, 'rest/res/resource/status')

    @property

    def post_courrier_url(self):

        return urlparse.urljoin(self.url, 'rest/res')

    def get_courriers(self, clause, fields=None, limit=None, include_file=False, order_by=None):

        if fields:

            # res_id is mandatory

            fields = set(fields)

            fields.add('res_id')

        fields = ','.join(fields) if fields else '*'

        limit = limit or self.default_limit

        order_by = order_by or []

        response = self.post_json(self.list_url, {

            'select': fields,

            'clause': clause,

            'limit': limit,

            'withFile': include_file,

            'orderBy': order_by,

        })

        if not hasattr(response.get('resources'), 'append'):

            raise MaarchError('missing resources field or bad type', response)

        return [self.Courrier(self, **resource) for resource in response['resources']]

    def update_external_infos(self, courriers, status):

        if not courriers:

            return

        external_infos = []

        payload = {

            'externalInfos': external_infos,

            'status': status,

        }

        for courrier in courriers:

            assert courrier.pk, 'courrier must already exist in Maarch and have a pk'

            external_info = {'res_id': courrier.pk}

            if getattr(courrier, 'external_id', None):

                external_info['external_id'] = courrier.external_id

            if getattr(courrier, 'external_link', None):

                external_info['external_link'] = courrier.external_link

            external_infos.append(external_info)

        response = self.put_json(self.update_external_infos_url, payload)

        if 'errors' in response:

            raise MaarchError('update_external_infos failed with errors', response['errors'], response)

    def update_status(self, courriers, status, history_message=None):

        if not courriers:

            return

        res_ids = []

        for courrier in courriers:

            assert courrier.pk

            res_ids.append(courrier.pk)

        payload = {

            'status': status,

            'resId': res_ids,

        }

        if history_message:

            payload['historyMessage'] = history_message

        response = self.put_json(self.update_status_url, payload)

        if 'errors' in response:

            raise MaarchError('update_status failed with errors', response['errors'], response)

    def post_courrier(self, courrier):

        response = self.post_json(self.post_courrier_url, courrier.post_serialize())

        if 'errors' in response:

            raise MaarchError('update_external_infos failed with errors', response['errors'], response)

        if 'resId' not in response:

            raise MaarchError('update_external_infos failed with errors, missing resId', response)

        courrier.pk = response['resId']

        return courrier

# welco - multichannel request processing

# Copyright (C) 2015  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from optparse import make_option

import os

from django.core.files.base import ContentFile

from django.core.management.base import BaseCommand, CommandError

from django.conf import settings

from django.db import transaction

from ...models import Mail

from ...utils import get_maarch

class Command(BaseCommand):

    """Inject mail coming from Maarch into welco.

       Only mail with a status "GRC" are injected,

       After injection, their status is immediately changed to "GRC_TRT".

       After injection in w.c.s., their status is changed to "GRCSENT" and an

       id and an URL of the request in w.c.s. is attached to the mail in

       Maarch.

    """

    def handle(self, *args, **kwargs):

        verbosity = kwargs['verbosity']

        maarch = get_maarch()

        maarch_mails = maarch.get_mails()

        count = 0

        while maarch_mails:

            with transaction.atomic():

                for maarch_mail in maarch_mails:

                    Mail.objects.create(

                        content=ContentFile(maarch_mail.content, name='maarch-%s' % maarch_mail.pk),

                        registered_mail_number='maarch-%s' % str(maarch_mail.pk),  # res_id

                    )

                # update maarch inside transaction, if it fails all imports will be

                # rollbacked

                maarch.set_grc_received_status(maarch_mails)

                count += len(maarch_mails)

            maarch_mails = maarch.get_mails()

        if verbosity > 1:

            self.stdout.write('Injected %d mails from %s.' % (count, maarch.url))

# welco - multichannel request processing

# Copyright (C) 2015  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.conf import settings

from .maarch import MaarchCourrier

class WelcoMaarchCourrier(MaarchCourrier):

    def __init__(self, url, username, password, grc_status,

                 grc_received_status, grc_send_status, grc_refused_status,

                 batch_size=10):

        super(WelcoMaarchCourrier, self).__init__(url, username, password)

        self.grc_status = grc_status

        self.grc_received_status = grc_received_status

        self.grc_send_status = grc_send_status

        self.grc_refused_status = grc_refused_status

        self.batch_size = batch_size

    def get_mails(self):

        return self.get_courriers(

            clause="status='%s'" % self.grc_status,

            include_file=True,

            order_by=['res_id'],

            limit=self.batch_size)

    def get_mail(self, mail_id):

        return self.get_courriers(clause="res_id=%s" % mail_id)[0]

    def set_grc_received_status(self, mails):

        self.update_status(mails, self.grc_received_status)

    def set_grc_sent_status(self, mail_pk, formdata_id, formdata_url_backoffice):

        mail = self.Courrier(self, pk=mail_pk)

        mail.external_id = formdata_id

        mail.external_link = formdata_url_backoffice

        self.update_external_infos([mail], self.grc_send_status)

    def set_grc_refused_status(self, mail_pk):

        mail = self.Courrier(self, pk=mail_pk)

        self.update_status([mail], self.grc_refused_status)

def get_maarch():

    config = getattr(settings, 'MAARCH_FEED', {})

    if not config.get('ENABLE'):

        return

    url = config['URL']

    username = config['USERNAME']

    password = config['PASSWORD']

    return WelcoMaarchCourrier(

        url=url,

        username=username,

        password=password,

        grc_status=config.get('STATUS_GRC', 'GRC'),

        grc_received_status=config.get('STATUS_RECEIVED', 'GRC_TRT'),

        grc_send_status=config.get('STATUS_SEND', 'GRCSENT'),

        grc_refused_status=config.get('STATUS_REFUSED', 'GRCREFUSED'))

import logging

from django.db.transaction import atomic

from .utils import get_maarch

logger = logging.getLogger(__name__)

    maarch = get_maarch()

    mail = Mail.objects.filter(id=request.POST['source_pk']).first()

    if mail:

        try:

            with atomic():

                if maarch and mail.registered_mail_number and mail.registered_mail_number.startswith('maarch-'):

                    mail_pk = mail.registered_mail_number.split('-', 1)[1]

                    maarch.set_grc_refused_status(mail_pk)

                mail.delete()

        except Exception:

            logger.exception('rejection request to maarch failed')

            messages.error(request, _('Rejection request to Maarch failed'))