From 760ccb41fd16f3907ae0ff921ab6b8d50dd0f5da Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Sat, 5 Dec 2020 09:24:33 +0100 Subject: [PATCH 3/3] lingo: add poll_backend method to PaymentBackend and Transaction (#49149) Some payment backends in eopayment (like PayFiP) allow polling the status of currently running transaction, and can signal if a running transaction has expired. The new can_poll_backend() and poll_backend() method on Transaction implement this conditional behaviour in lingo. --- combo/apps/lingo/forms.py | 8 +- .../management/commands/lingo-poll-backend.py | 96 ++++++ combo/apps/lingo/models.py | 174 +++++++++-- .../lingo/templates/lingo/combo/item.html | 5 +- .../lingo/templates/lingo/combo/items.html | 2 +- combo/apps/lingo/views.py | 64 ++-- debian/combo.cron.d | 1 + tests/test_lingo_manager.py | 6 +- tests/test_lingo_payment.py | 279 ++++++++++++++---- tests/test_lingo_remote_regie.py | 98 ++++++ 10 files changed, 619 insertions(+), 114 deletions(-) create mode 100644 combo/apps/lingo/management/commands/lingo-poll-backend.py diff --git a/combo/apps/lingo/forms.py b/combo/apps/lingo/forms.py index 078e93bd..dc279de0 100644 --- a/combo/apps/lingo/forms.py +++ b/combo/apps/lingo/forms.py @@ -100,7 +100,7 @@ class RegieForm(forms.ModelForm): def __init__(self, *args, **kwargs): super(RegieForm, self).__init__(*args, **kwargs) fields, initial = create_form_fields( - self.instance.payment_backend.get_payment().get_parameters(scope='transaction'), + self.instance.eopayment.get_parameters(scope='transaction'), self.instance.transaction_options, ) self.fields.update(fields) @@ -109,7 +109,7 @@ class RegieForm(forms.ModelForm): def save(self): instance = super(RegieForm, self).save() instance.transaction_options = compute_json_field( - self.instance.payment_backend.get_payment().get_parameters(scope='transaction'), self.cleaned_data + self.instance.eopayment.get_parameters(scope='transaction'), self.cleaned_data ) instance.save() return instance @@ -123,7 +123,7 @@ class PaymentBackendForm(forms.ModelForm): def __init__(self, *args, **kwargs): super(PaymentBackendForm, self).__init__(*args, **kwargs) fields, initial = create_form_fields( - self.instance.get_payment().get_parameters(scope='global'), self.instance.service_options + self.instance.eopayment.get_parameters(scope='global'), self.instance.service_options ) self.fields.update(fields) self.initial.update(initial) @@ -133,7 +133,7 @@ class PaymentBackendForm(forms.ModelForm): def save(self): instance = super(PaymentBackendForm, self).save() instance.service_options = compute_json_field( - self.instance.get_payment().get_parameters(scope='global'), self.cleaned_data + self.instance.eopayment.get_parameters(scope='global'), self.cleaned_data ) instance.save() return instance diff --git a/combo/apps/lingo/management/commands/lingo-poll-backend.py b/combo/apps/lingo/management/commands/lingo-poll-backend.py new file mode 100644 index 00000000..498d815f --- /dev/null +++ b/combo/apps/lingo/management/commands/lingo-poll-backend.py @@ -0,0 +1,96 @@ +# lingo - basket and payment system +# Copyright (C) 2021 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import datetime +import logging + +from django.core.management.base import BaseCommand, CommandError + +from combo.apps.lingo.models import PaymentBackend, PaymentException + +logger = logging.getLogger('combo.apps.lingo') + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--backend', default=None, help='slug of the backend') + + parser.add_argument('--all-backends', default=False, action='store_true', help='target all backends') + + parser.add_argument( + '--noinput', + '--no-input', + action='store_false', + dest='interactive', + help='Tells Django to NOT prompt the user for input of any kind.', + ) + + parser.add_argument( + '--max-age-in-days', default=3, type=int, help='max age of the transaction in days' + ) + + def handle( + self, + *args, + backend=None, + all_backends=False, + max_age_in_days=None, + interactive=True, + verbosity=1, + **options, + ): + qs = PaymentBackend.objects.all() + if backend and all_backends: + raise CommandError('--backend and --all-baskends cannot be used together') + elif backend: + try: + backend = qs.get(slug=backend) + except PaymentBackend.DoesNotExist: + raise CommandError('no backend with slug "%s".' % backend) + else: + if not backend.can_poll_backend(): + raise CommandError('backend "%s" cannot be polled.' % backend) + backends = [backend] + else: + backends = [backend for backend in qs if backend.can_poll_backend()] + if not backends: + raise CommandError('no backend found.') + if not all_backends and interactive: + print('Choose backend by slug:') + while True: + for backend in backends: + print(' - %s: %s' % (backend.slug, backend)) + print('> ', end=' ') + slug = input().strip() + if not slug: + continue + filtered_backends = qs.filter(slug__icontains=slug) + if filtered_backends: + backends = filtered_backends + break + + for backend in backends: + if verbosity >= 1: + print('Polling backend', backend, '... ', end='') + try: + backend.poll_backend(max_age=max_age_in_days and datetime.timedelta(days=max_age_in_days)) + except PaymentException: + logger.exception('polling failed') + if interactive: + # show error + raise + if verbosity >= 1: + print('DONE') diff --git a/combo/apps/lingo/models.py b/combo/apps/lingo/models.py index 93f20eb8..f703ada8 100644 --- a/combo/apps/lingo/models.py +++ b/combo/apps/lingo/models.py @@ -27,11 +27,11 @@ import eopayment from dateutil import parser from django import template from django.conf import settings -from django.contrib.auth.models import User from django.core import serializers -from django.core.exceptions import ObjectDoesNotExist, PermissionDenied, ValidationError +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.core.mail import EmailMultiAlternatives from django.db import models +from django.db.transaction import atomic from django.forms import Select from django.forms import models as model_forms from django.template.loader import render_to_string @@ -39,9 +39,8 @@ from django.urls import reverse from django.utils import dateparse, six, timezone from django.utils.encoding import force_bytes, python_2_unicode_compatible from django.utils.formats import localize -from django.utils.http import urlencode from django.utils.six.moves.urllib import parse as urlparse -from django.utils.timezone import make_aware, utc +from django.utils.timezone import make_aware, now, utc from django.utils.translation import ugettext_lazy as _ from jsonfield import JSONField from requests import RequestException @@ -156,7 +155,11 @@ class PaymentBackend(models.Model): def __str__(self): return self.label - def get_payment(self): + @property + def eopayment(self): + return self.make_eopayment() + + def make_eopayment(self, *, request=None, automatic_return_url=None, normal_return_url=None, **kwargs): options = self.service_options or {} if isinstance(options, six.string_types): # backward compatibility when used againt postgresql < 9.4 and @@ -167,6 +170,23 @@ class PaymentBackend(models.Model): pass if not isinstance(options, dict): options = {} + if request: + if not automatic_return_url: + automatic_return_url = reverse( + 'lingo-callback-payment-backend', kwargs={'payment_backend_pk': self.id} + ) + + if automatic_return_url: + automatic_return_url = request.build_absolute_uri(automatic_return_url) + if normal_return_url: + normal_return_url = request.build_absolute_uri(normal_return_url) + options['automatic_return_url'] = automatic_return_url + options['normal_return_url'] = normal_return_url + else: + assert ( + not automatic_return_url and not normal_return_url + ), 'make_eopayment must be used with a request to set automatic_return_url or normal_return_url' + options.update(**kwargs) return eopayment.Payment(self.service, options) def natural_key(self): @@ -231,6 +251,35 @@ class PaymentBackend(models.Model): transaction.handle_backend_response(response, callback=callback) return transaction + def can_poll_backend(self): + return self.eopayment.has_payment_status + + def poll_backend(self, min_age=None, max_age=None): + if not self.can_poll_backend(): + return + current_time = now() + # poll transactions linked to the current backend + # aged between 5 minutes and 3 hours, max_age can be overriden + min_age = min_age or datetime.timedelta(minutes=5) + not_after = current_time - min_age + max_age = max_age or datetime.timedelta(hours=3) + not_before = current_time - max_age + transactions = Transaction.objects.filter( + regie__payment_backend=self, + start_date__lt=not_after, + start_date__gt=not_before, + status__in=Transaction.RUNNING_STATUSES, + ).order_by('pk') + last_pk = -1 + while True: + # lock each transaction before trying to poll it + with atomic(): + transaction = transactions.filter(pk__gt=last_pk).select_for_update(skip_locked=True).first() + if not transaction: + break + last_pk = transaction.pk + transaction.poll_backend(ignore_errors=False) + @python_2_unicode_compatible class Regie(models.Model): @@ -547,6 +596,16 @@ class Regie(models.Model): regie = next(serializers.deserialize('json', json.dumps([json_regie]), ignorenonexistent=True)) regie.save() + def can_poll_backend(self): + return self.payment_backend.can_poll_backend() + + @property + def eopayment(self): + return self.make_eopayment() + + def make_eopayment(self, **kwargs): + return self.payment_backend.make_eopayment(**kwargs) + class BasketItem(models.Model): user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True) @@ -572,9 +631,13 @@ class BasketItem(models.Model): @classmethod def get_items_to_be_paid(cls, user): - return cls.objects.filter( + qs = cls.objects.filter( user=user, payment_date__isnull=True, waiting_date__isnull=True, cancellation_date__isnull=True ) + for transaction in Transaction.objects.filter(items__in=qs): + if transaction.can_poll_backend(): + transaction.poll_backend() + return qs def notify(self, status): if not self.source_url: @@ -656,6 +719,7 @@ class RemoteItem(object): self.reference_id = reference_id if payment_date: self.payment_date = parser.parse(payment_date) + self.waiting_date = None @property def no_online_payment_reason_details(self): @@ -673,28 +737,44 @@ class RemoteItem(object): return aes_hex_encrypt(settings.SECRET_KEY, force_bytes(str(self.id))) @classmethod - def update_paid(cls, regie, remote_items): - remote_item_ids = [remote_item.id for remote_item in remote_items if not remote_item.paid] + def transactions_for_remote_items(cls, queryset, remote_items): + remote_item_ids = set(remote_item.id for remote_item in remote_items if not remote_item.paid) if not remote_item_ids: - return + return Transaction.objects.none() - paid_items = {} # filter transactions by regie, status and contained remote_item id - transaction_qs = Transaction.objects.filter( - regie=regie, status__in=[eopayment.PAID, eopayment.ACCEPTED] - ) query = reduce( models.Q.__or__, (models.Q(remote_items__contains=remote_item_id) for remote_item_id in remote_item_ids), ) # accumulate in paid_items each remote_item earliest payment_date - for transaction in transaction_qs.filter(query): + for transaction in queryset.filter(query): + for remote_item_id in transaction.remote_items.split(','): + if remote_item_id in remote_item_ids: + yield transaction + break + + @classmethod + def update_paid(cls, regie, remote_items): + paid_items = {} + waiting_items = {} + transaction_qs = Transaction.objects.filter(regie=regie) + + can_poll_backend = regie.can_poll_backend() + + # accumulate in paid_items each remote_item earliest payment_date + for transaction in cls.transactions_for_remote_items(transaction_qs, remote_items): + if transaction.is_running() and can_poll_backend: + transaction.poll_backend() for remote_item in transaction.remote_items.split(','): - if remote_item not in paid_items: - paid_items[remote_item] = transaction.end_date - else: - paid_items[remote_item] = min(transaction.end_date, paid_items[remote_item]) + if transaction.end_date and transaction.is_paid(): + if remote_item not in paid_items: + paid_items[remote_item] = transaction.end_date + else: + paid_items[remote_item] = min(transaction.end_date, paid_items[remote_item]) + elif transaction.status == eopayment.WAITING and can_poll_backend: + waiting_items[remote_item] = transaction.start_date # update remote_item.paid using paid_items for remote_item in remote_items: @@ -703,6 +783,8 @@ class RemoteItem(object): if remote_item.id in paid_items: remote_item.paid = True remote_item.payment_date = paid_items[remote_item.id] + elif remote_item.id in waiting_items: + remote_item.waiting_date = waiting_items[remote_item.id] def status_label(status): @@ -731,6 +813,9 @@ class Transaction(models.Model): status = models.IntegerField(null=True) amount = models.DecimalField(default=0, max_digits=7, decimal_places=2) + RUNNING_STATUSES = [0, eopayment.WAITING, eopayment.RECEIVED] + PAID_STATUSES = [eopayment.PAID, eopayment.ACCEPTED] + def is_remote(self): return self.remote_items != '' @@ -740,10 +825,10 @@ class Transaction(models.Model): return _('Anonymous User') def is_paid(self): - return self.status in (eopayment.PAID, eopayment.ACCEPTED) + return self.status in self.PAID_STATUSES def is_running(self): - return self.status in [0, eopayment.WAITING, eopayment.RECEIVED] + return self.status in self.RUNNING_STATUSES def get_status_label(self): return status_label(self.status) @@ -890,6 +975,49 @@ class Transaction(models.Model): if self.remote_items: self.first_notify_remote_items_of_payments() + @property + def eopayment(self): + return self.regie.eopayment + + def make_eopayment(self, **kwargs): + normal_return_url = reverse( + 'lingo-return-payment-backend', + kwargs={ + 'payment_backend_pk': self.regie.payment_backend.id, + 'transaction_signature': signing_dumps(self.pk), + }, + ) + return self.regie.make_eopayment(normal_return_url=normal_return_url, **kwargs) + + def can_poll_backend(self): + return self.regie and self.regie.can_poll_backend() + + def poll_backend(self, ignore_errors=True): + with atomic(): + # lock the transaction + Transaction.objects.filter(pk=self.pk).select_for_update().first() + try: + response = self.eopayment.payment_status(self.order_id, transaction_date=self.start_date) + except eopayment.PaymentException: + if ignore_errors: + logger.warning( + 'lingo: regie "%s" polling backend for transaction "%%s(%%s)" failed' % self.regie, + self.order_id, + self.id, + exc_info=True, + ) + return + raise PaymentException('polling failed') + + logger.debug( + 'lingo: regie "%s" polling backend for transaction "%%s(%%s)"' % self.regie, + self.order_id, + self.id, + ) + + if self.status != response.result: + self.handle_backend_response(response) + class TransactionOperation(models.Model): OPERATIONS = [ @@ -968,9 +1096,13 @@ class LingoRecentTransactionsCell(CellBase): # list transactions : # * paid by the user # * or linked to a BasketItem of the user - return Transaction.objects.filter(models.Q(user=user) | models.Q(items__user=user)).filter( + qs = Transaction.objects.filter(models.Q(user=user) | models.Q(items__user=user)).filter( start_date__gte=timezone.now() - datetime.timedelta(days=7) ) + for transaction in qs: + if transaction.can_poll_backend() and transaction.is_running(): + transaction.poll_backend() + return qs def is_relevant(self, context): if not (getattr(context['request'], 'user', None) and context['request'].user.is_authenticated): diff --git a/combo/apps/lingo/templates/lingo/combo/item.html b/combo/apps/lingo/templates/lingo/combo/item.html index 6ff7d877..1c43988f 100644 --- a/combo/apps/lingo/templates/lingo/combo/item.html +++ b/combo/apps/lingo/templates/lingo/combo/item.html @@ -48,7 +48,10 @@ {% if item.no_online_payment_reason_details %}
{{ item.no_online_payment_reason_details }}
{% endif %} - {% if not item.paid and item.online_payment and item.amount >= regie.payment_min_amount %} + {% if item.waiting_date and not item.paid %} + + {% endif %} + {% if not item.paid and item.online_payment and item.amount >= regie.payment_min_amount and not item.waiting_date %} {% csrf_token %} {% if not user.is_authenticated %}