From bf3f20b73eb436f770f64ff82c3fd473f9899e02 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Wed, 19 Jan 2022 16:52:48 +0100 Subject: [PATCH 01/11] add a generic soap connector (#60836) --- passerelle/apps/soap/__init__.py | 0 .../apps/soap/migrations/0001_initial.py | 83 +++++ passerelle/apps/soap/migrations/__init__.py | 0 passerelle/apps/soap/models.py | 182 ++++++++++ .../templates/soap/soapconnector_detail.html | 2 + passerelle/settings.py | 1 + tests/test_soap.py | 313 +++++++++++++----- tests/test_utils_soap.py | 109 ++++++ 8 files changed, 613 insertions(+), 77 deletions(-) create mode 100644 passerelle/apps/soap/__init__.py create mode 100644 passerelle/apps/soap/migrations/0001_initial.py create mode 100644 passerelle/apps/soap/migrations/__init__.py create mode 100644 passerelle/apps/soap/models.py create mode 100644 passerelle/apps/soap/templates/soap/soapconnector_detail.html create mode 100644 tests/test_utils_soap.py diff --git a/passerelle/apps/soap/__init__.py b/passerelle/apps/soap/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/apps/soap/migrations/0001_initial.py b/passerelle/apps/soap/migrations/0001_initial.py new file mode 100644 index 00000000..74938ed4 --- /dev/null +++ b/passerelle/apps/soap/migrations/0001_initial.py @@ -0,0 +1,83 @@ +# Generated by Django 2.2.24 on 2022-01-19 16:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('base', '0029_auto_20210202_1627'), + ] + + operations = [ + migrations.CreateModel( + name='SOAPConnector', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('title', models.CharField(max_length=50, verbose_name='Title')), + ('slug', models.SlugField(unique=True, verbose_name='Identifier')), + ('description', models.TextField(verbose_name='Description')), + ( + 'basic_auth_username', + models.CharField( + blank=True, max_length=128, verbose_name='Basic authentication username' + ), + ), + ( + 'basic_auth_password', + models.CharField( + blank=True, max_length=128, verbose_name='Basic authentication password' + ), + ), + ( + 'client_certificate', + models.FileField( + blank=True, null=True, upload_to='', verbose_name='TLS client certificate' + ), + ), + ( + 'trusted_certificate_authorities', + models.FileField(blank=True, null=True, upload_to='', verbose_name='TLS trusted CAs'), + ), + ( + 'verify_cert', + models.BooleanField(blank=True, default=True, verbose_name='TLS verify certificates'), + ), + ( + 'http_proxy', + models.CharField(blank=True, max_length=128, verbose_name='HTTP and HTTPS proxy'), + ), + ( + 'wsdl_url', + models.URLField( + help_text='URL of the WSDL file', max_length=400, verbose_name='WSDL URL' + ), + ), + ( + 'zeep_strict', + models.BooleanField(default=True, verbose_name='Be strict with returned XML'), + ), + ( + 'zeep_xsd_ignore_sequence_order', + models.BooleanField(default=False, verbose_name='Ignore sequence order'), + ), + ( + 'users', + models.ManyToManyField( + blank=True, + related_name='_soapconnector_users_+', + related_query_name='+', + to='base.ApiUser', + ), + ), + ], + options={ + 'verbose_name': 'SOAP connector', + }, + ), + ] diff --git a/passerelle/apps/soap/migrations/__init__.py b/passerelle/apps/soap/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/apps/soap/models.py b/passerelle/apps/soap/models.py new file mode 100644 index 00000000..0965fe5c --- /dev/null +++ b/passerelle/apps/soap/models.py @@ -0,0 +1,182 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import collections + +import zeep +import zeep.exceptions +import zeep.helpers +import zeep.xsd +from django.db import models +from django.utils.functional import cached_property +from django.utils.translation import ugettext_lazy as _ + +from passerelle.base.models import BaseResource, HTTPResource +from passerelle.utils.api import endpoint +from passerelle.utils.json import unflatten + + +class SOAPConnector(BaseResource, HTTPResource): + wsdl_url = models.URLField( + max_length=400, verbose_name=_('WSDL URL'), help_text=_('URL of the WSDL file') + ) + zeep_strict = models.BooleanField(default=False, verbose_name=_('Be strict with returned XML')) + zeep_xsd_ignore_sequence_order = models.BooleanField( + default=True, verbose_name=_('Ignore sequence order') + ) + category = _('Business Process Connectors') + + class Meta: + verbose_name = _('SOAP connector') + + @classmethod + def get_manager_form_class(cls, **kwargs): + form_class = super().get_manager_form_class(**kwargs) + fields = list(form_class.base_fields.items()) + form_class.base_fields = collections.OrderedDict(fields[:2] + fields[-3:] + fields[2:-3]) + return form_class + + @cached_property + def client(self): + return self.soap_client( + wsdl_url=self.wsdl_url, + settings=zeep.Settings( + strict=self.zeep_strict, xsd_ignore_sequence_order=self.zeep_xsd_ignore_sequence_order + ), + ) + + @endpoint( + methods=['post'], + perm='can_access', + name='method', + pattern=r'^(?P\w+)/$', + example_pattern='method_name/', + description_get=_('Call a SOAP method'), + description_post=_('Call a SOAP method'), + post_json_schema={'type': 'object'}, + ) + def method(self, request, method_name, post_data=None, **kwargs): + def jsonify(data): + if isinstance(data, (dict, collections.OrderedDict)): + # ignore _raw_elements, zeep put there nodes not maching the + # XSD when strict parsing is disabled. + return { + jsonify(k): jsonify(v) + for k, v in data.items() + if (self.zeep_strict or k != '_raw_elements') + } + elif isinstance(data, (list, tuple, collections.deque)): + return [jsonify(item) for item in data] + else: + return data + + payload = {} + for k in request.GET: + if k == 'raise': + continue + value = request.GET.getlist(k) + if len(value) > 1: + payload[k] = value + else: + payload[k] = value[0] + payload.update(post_data or {}) + payload = unflatten(payload) + try: + soap_response = getattr(self.client.service, method_name)(**payload) + except zeep.exceptions.ValidationError as e: + e.status_code = 400 + raise e + serialized = zeep.helpers.serialize_object(soap_response) + json_response = jsonify(serialized) + return {'err': 0, 'data': json_response} + + method.endpoint_info.methods.append('get') + + def get_endpoints_infos(self): + endpoints = super().get_endpoints_infos() + for name, input_schema, output_schema in self.operations_and_schemas: + kwargs = dict( + name='method', + pattern=f'{name}/', + example_pattern=f'{name}/', + description=f'Method {name}', + json_schema_response={ + 'type': 'object', + 'properties': collections.OrderedDict( + [ + ('err', {'type': 'integer'}), + ('data', output_schema), + ] + ), + }, + ) + if input_schema: + kwargs['post_json_schema'] = input_schema + endpoints.append(endpoint(**kwargs)) + endpoints[-1].object = self + endpoints[-1].func = lambda request: None + if input_schema: + endpoints[-1].http_method = 'post' + else: + endpoints[-1].http_method = 'get' + return endpoints + + @property + def operations_and_schemas(self): + operations = self.client.service._binding._operations + for name in operations: + operation = operations[name] + input_type = operation.input.body.type + output_type = operation.output.body.type + input_schema = self.type2schema(input_type, keep_root=True) + output_schema = self.type2schema(output_type, compress=True) + yield name, input_schema, output_schema + + def type2schema(self, xsd_type, keep_root=False, compress=False): + # simplify schema: when a type contains a unique element, it will try + # to match any dict or list with it on input and will flatten the + # schema on output. + if ( + isinstance(xsd_type, zeep.xsd.ComplexType) + and len(xsd_type.elements) == 1 + and not keep_root + and compress + ): + if xsd_type.elements[0][1].max_occurs != 1: + return { + 'type': 'array', + 'items': self.type2schema(xsd_type.elements[0][1].type, compress=compress), + } + return self.type2schema(xsd_type.elements[0][1].type, compress=compress) + if isinstance(xsd_type, zeep.xsd.ComplexType): + properties = collections.OrderedDict() + schema = { + 'type': 'object', + 'properties': properties, + } + for key, element in xsd_type.elements: + if element.min_occurs > 0: + schema.setdefault('required', []).append(key) + element_schema = self.type2schema(element.type, compress=compress) + if element.max_occurs == 'unbounded' or element.max_occurs > 1: + element_schema = {'type': 'array', 'items': element_schema} + properties[key] = element_schema + if not properties: + return None + return schema + if isinstance(xsd_type, zeep.xsd.BuiltinType): + return {'type': 'string'} + return f'!!! UNKNOWN TYPE {xsd_type} !!!' diff --git a/passerelle/apps/soap/templates/soap/soapconnector_detail.html b/passerelle/apps/soap/templates/soap/soapconnector_detail.html new file mode 100644 index 00000000..67a705ec --- /dev/null +++ b/passerelle/apps/soap/templates/soap/soapconnector_detail.html @@ -0,0 +1,2 @@ +{% extends "passerelle/manage/service_view.html" %} +{% load i18n passerelle %} diff --git a/passerelle/settings.py b/passerelle/settings.py index 0152f331..b51442e5 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -162,6 +162,7 @@ INSTALLED_APPS = ( 'passerelle.apps.plone_restapi', 'passerelle.apps.sector', 'passerelle.apps.sfr_dmc', + 'passerelle.apps.soap', 'passerelle.apps.solis', 'passerelle.apps.sp_fr', 'passerelle.apps.twilio', diff --git a/tests/test_soap.py b/tests/test_soap.py index 5e81c628..6fb906cf 100644 --- a/tests/test_soap.py +++ b/tests/test_soap.py @@ -13,97 +13,256 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import mock +import urllib.parse + import pytest -import requests -from django.utils.encoding import force_bytes -from zeep import Settings -from zeep.exceptions import TransportError, XMLParseError -from zeep.plugins import Plugin +import utils -from passerelle.utils.soap import SOAPClient +from passerelle.apps.soap.models import SOAPConnector -WSDL = 'tests/data/soap.wsdl' +class SOAP11: + VERSION = '1.1' + ENDPOINT_URL = 'https://www.examples.com/SayHello/' + WSDL_CONTENT = '''\ + -class FooPlugin(Plugin): - pass + + + + + + + + + + + + + + + -class BarPlugin(Plugin): - pass + + + + + + + + + -class SOAPResource(object): - def __init__(self): - self.requests = requests.Session() - self.wsdl_url = WSDL + + + + + + + + + + + + -def test_soap_client(): - soap_resource = SOAPResource() - plugins = [FooPlugin, BarPlugin] - client = SOAPClient(soap_resource, plugins=plugins) - assert client.wsdl.location.endswith(WSDL) - assert client.transport.session == soap_resource.requests - assert client.transport.cache - assert client.plugins == plugins + + WSDL File for HelloService + + + + +''' + WSDL_URL = 'https://example.com/service.wsdl' + SOAP_RESPONSE = '''\ + + + + + Hello John! + + +''' + INPUT_SCHEMA = { + 'properties': { + 'firstName': { + 'properties': { + 'string': {'items': {'type': 'string'}, 'type': 'array'}, + }, + 'required': ['string'], + 'type': 'object', + }, + 'lastName': {'type': 'string'}, + }, + 'required': ['firstName', 'lastName'], + 'type': 'object', + } + OUTPUT_SCHEMA = {'type': 'string'} + INPUT_DATA = { + 'firstName/string/0': 'John', + 'firstName/string/1': 'Bill', + 'lastName': 'Doe', + } -@mock.patch('requests.sessions.Session.post') -def test_disable_strict_mode(mocked_post): - response = requests.Response() - response.status_code = 200 - response._content = force_bytes( - ''' - - - - 4.20 - - -''' - ) - mocked_post.return_value = response - - soap_resource = SOAPResource() - client = SOAPClient(soap_resource) - match = "Unexpected element %s, expected %s" % (repr(u'price'), repr(u'skipMe')) - with pytest.raises(XMLParseError, match=match): - client.service.GetLastTradePrice(tickerSymbol='banana') - - client = SOAPClient(soap_resource, settings=Settings(strict=False)) - result = client.service.GetLastTradePrice(tickerSymbol='banana') - assert len(result) == 2 - assert result['skipMe'] is None - assert result['price'] == 4.2 - - -@mock.patch('requests.sessions.Session.post') -def test_remove_first_bytes_for_xml(mocked_post): - response = requests.Response() - response.status_code = 200 - response._content = force_bytes( - '''blabla \n - - - - 1.2 - 4.20 - - -\n bloublou''' +class SOAP12(SOAP11): + VERSION = '1.2' + ENDPOINT_URL = 'https://www.examples.com/SayHello/' + WSDL_CONTENT = f'''\ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +''' + SOAP_RESPONSE = '''\ + + + + + + Hello John! + + +''' + INPUT_SCHEMA = { + 'type': 'object', + 'properties': { + 'firstName': {'type': 'array', 'items': {'type': 'string'}}, + 'lastName': {'type': 'string'}, + }, + 'required': ['firstName', 'lastName'], + } + INPUT_DATA = { + 'firstName/0': 'John', + 'firstName/1': 'Bill', + 'lastName': 'Doe', + } + + +@pytest.fixture(params=[SOAP11, SOAP12]) +def soap(request): + p = request.param() + with utils.mock_url(p.WSDL_URL, response=p.WSDL_CONTENT): + with utils.mock_url(p.ENDPOINT_URL, response=p.SOAP_RESPONSE) as mock: + p.endpoint_mock = mock + yield p + + +@pytest.fixture +def connector(db, soap): + return utils.setup_access_rights( + SOAPConnector.objects.create( + slug='test', wsdl_url=soap.WSDL_URL, zeep_strict=True, zeep_xsd_ignore_sequence_order=False + ) ) - mocked_post.return_value = response - soap_resource = SOAPResource() - client = SOAPClient(soap_resource) - with pytest.raises(TransportError): - client.service.GetLastTradePrice(tickerSymbol='banana') +def test_schemas(connector, soap): + assert list(connector.operations_and_schemas) == [('sayHello', soap.INPUT_SCHEMA, soap.OUTPUT_SCHEMA)] + + +def test_say_hello_method_validation_error(connector, app): + resp = app.get('/soap/test/method/sayHello/', status=500) + assert dict(resp.json, err_desc=None) == { + 'err': 1, + 'err_class': 'zeep.exceptions.ValidationError', + 'err_desc': None, + 'data': None, + } + + +def test_say_hello_method_ok_get(connector, app, caplog, soap): + resp = app.get('/soap/test/method/sayHello/?' + urllib.parse.urlencode(soap.INPUT_DATA)) + assert '>John<' in soap.endpoint_mock.handlers[0].call['requests'][-1].body.decode() + assert '>Bill<' in soap.endpoint_mock.handlers[0].call['requests'][-1].body.decode() + + assert '>Doe<' in soap.endpoint_mock.handlers[0].call['requests'][-1].body.decode() + assert resp.json == {'data': 'Hello John!', 'err': 0} + - client = SOAPClient(soap_resource, transport_kwargs={'remove_first_bytes_for_xml': True}) - result = client.service.GetLastTradePrice(tickerSymbol='banana') - assert len(result) == 2 - assert result['skipMe'] == 1.2 - assert result['price'] == 4.2 +def test_say_hello_method_ok_post_json(connector, app, caplog, soap): + resp = app.post_json('/soap/test/method/sayHello/', params=soap.INPUT_DATA) + assert '>John<' in soap.endpoint_mock.handlers[0].call['requests'][-1].body.decode() + assert '>Bill<' in soap.endpoint_mock.handlers[0].call['requests'][-1].body.decode() + assert '>Doe<' in soap.endpoint_mock.handlers[0].call['requests'][-1].body.decode() + assert resp.json == {'data': 'Hello John!', 'err': 0} diff --git a/tests/test_utils_soap.py b/tests/test_utils_soap.py new file mode 100644 index 00000000..5e81c628 --- /dev/null +++ b/tests/test_utils_soap.py @@ -0,0 +1,109 @@ +# Copyright (C) 2021 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import mock +import pytest +import requests +from django.utils.encoding import force_bytes +from zeep import Settings +from zeep.exceptions import TransportError, XMLParseError +from zeep.plugins import Plugin + +from passerelle.utils.soap import SOAPClient + +WSDL = 'tests/data/soap.wsdl' + + +class FooPlugin(Plugin): + pass + + +class BarPlugin(Plugin): + pass + + +class SOAPResource(object): + def __init__(self): + self.requests = requests.Session() + self.wsdl_url = WSDL + + +def test_soap_client(): + soap_resource = SOAPResource() + plugins = [FooPlugin, BarPlugin] + client = SOAPClient(soap_resource, plugins=plugins) + assert client.wsdl.location.endswith(WSDL) + assert client.transport.session == soap_resource.requests + assert client.transport.cache + assert client.plugins == plugins + + +@mock.patch('requests.sessions.Session.post') +def test_disable_strict_mode(mocked_post): + response = requests.Response() + response.status_code = 200 + response._content = force_bytes( + ''' + + + + 4.20 + + +''' + ) + mocked_post.return_value = response + + soap_resource = SOAPResource() + client = SOAPClient(soap_resource) + match = "Unexpected element %s, expected %s" % (repr(u'price'), repr(u'skipMe')) + with pytest.raises(XMLParseError, match=match): + client.service.GetLastTradePrice(tickerSymbol='banana') + + client = SOAPClient(soap_resource, settings=Settings(strict=False)) + result = client.service.GetLastTradePrice(tickerSymbol='banana') + assert len(result) == 2 + assert result['skipMe'] is None + assert result['price'] == 4.2 + + +@mock.patch('requests.sessions.Session.post') +def test_remove_first_bytes_for_xml(mocked_post): + response = requests.Response() + response.status_code = 200 + response._content = force_bytes( + '''blabla \n + + + + 1.2 + 4.20 + + +\n bloublou''' + ) + mocked_post.return_value = response + + soap_resource = SOAPResource() + + client = SOAPClient(soap_resource) + with pytest.raises(TransportError): + client.service.GetLastTradePrice(tickerSymbol='banana') + + client = SOAPClient(soap_resource, transport_kwargs={'remove_first_bytes_for_xml': True}) + result = client.service.GetLastTradePrice(tickerSymbol='banana') + assert len(result) == 2 + assert result['skipMe'] == 1.2 + assert result['price'] == 4.2 -- 2.35.1