From e33855cbad8dee60766c963299eb530ddbf08cc6 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 16 Nov 2018 14:46:52 +0100 Subject: [PATCH] =?UTF-8?q?nanterre:=20d=C3=A9tecte=20et=20supprime=20les?= =?UTF-8?q?=20fiches=20inactives=20(fixes=20#28080)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debian/control | 1 + setup.py | 1 + zoo/zoo_nanterre/apps.py | 7 + zoo/zoo_nanterre/inactivity.py | 123 ++++++++++++++++++ zoo/zoo_nanterre/models.py | 16 +++ zoo/zoo_nanterre/templates/admin/index.html | 11 ++ .../admin/zoo_data/entity/inactive_index.html | 72 ++++++++++ zoo/zoo_nanterre/utils.py | 35 ++++- zoo/zoo_nanterre/views.py | 89 ++++++++++++- 9 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 zoo/zoo_nanterre/inactivity.py create mode 100644 zoo/zoo_nanterre/templates/admin/zoo_data/entity/inactive_index.html diff --git a/debian/control b/debian/control index 893c52a..99dc561 100644 --- a/debian/control +++ b/debian/control @@ -17,6 +17,7 @@ Depends: ${misc:Depends}, adduser, python-psycopg2, python-memcache, python-django-mellon, + python-cached-property, uwsgi Recommends: nginx, postgresql, memcached Description: Maintain a graph of objects diff --git a/setup.py b/setup.py index 63ae207..d9be446 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,7 @@ setup( 'python-dateutil', 'django-admin-rangefilter', 'requests', + 'cached-property', ], zip_safe=False, cmdclass={ diff --git a/zoo/zoo_nanterre/apps.py b/zoo/zoo_nanterre/apps.py index ce65f60..06c098c 100644 --- a/zoo/zoo_nanterre/apps.py +++ b/zoo/zoo_nanterre/apps.py @@ -95,6 +95,13 @@ class ZooNanterreConfig(AppConfig): kwargs={'model_admin': model_admin}, name='synchronize-federations' + desc['name'], )) + urls.append(url( + r'^inactive/', + model_admin.admin_site.admin_view( + getattr(views, 'inactive_index')), + kwargs={'model_admin': model_admin}, + name='inactive-index', + )) return urls def post_migrate(self, *args, **kwargs): diff --git a/zoo/zoo_nanterre/inactivity.py b/zoo/zoo_nanterre/inactivity.py new file mode 100644 index 0000000..4f468c9 --- /dev/null +++ b/zoo/zoo_nanterre/inactivity.py @@ -0,0 +1,123 @@ +# zoo - versatile objects management +# Copyright (C) 2016 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import datetime +import operator +try: + from functools import reduce +except ImportError: + pass + +from django.db.models.query import Q +from django.utils.timezone import now + +from cached_property import cached_property + +from zoo.zoo_data.models import Entity + +from . import utils + + +class Inactivity(object): + def __init__(self, child_delay=365, adult_delay=365): + self.child_delay = child_delay + self.adult_delay = adult_delay + + @property + def child_threshold(self): + return now() - datetime.timedelta(days=self.child_delay) + + @property + def adult_threshold(self): + return now() - datetime.timedelta(days=self.adult_delay) + + def exclude_newer_than_threshold(self, qs, threshold): + return qs.exclude( + created__created__gte=threshold, + modified__created__gte=threshold, + log__timestamp__gte=threshold) + + @property + def query_no_federation(self): + queries = [] + + for app_id in utils.get_applications(rsu_ws_url=True): + query = Q(**{'content__cles_de_federation__%s__isnull' % app_id: True}) + query |= Q(**{'content__cles_de_federation__%s__exact' % app_id: ''}) + queries.append(query) + return reduce(operator.__and__, queries) + + def filter_no_federation(self, qs): + return qs.filter(self.query_no_federation) + + @property + def entities(self): + qs = Entity.objects.all() + # prefetch to optimize accesors to spouses and siblings + qs = qs.prefetch_related( + 'left_relations__schema', 'left_relations__right', + 'right_relations__schema', 'right_relations__left', + 'right_relations__left__left_relations__left', + 'right_relations__left__left_relations__schema', + ) + return qs + + @property + def children(self): + return self.entities.filter(content__statut_legal='mineur') + + @property + def adults(self): + return self.entities.filter(content__statut_legal='majeur') + + @cached_property + def deletable_children(self): + potent = self.exclude_newer_than_threshold(self.filter_no_federation(self.children), self.child_threshold) + potent_ids = potent.values_list('id', flat=True) + + def filter_siblings_are_potent(): + for child in potent: + for sibling in utils.fratrie(child): + if sibling.id not in potent_ids: + break + else: + yield child.id + potent2_ids = list(filter_siblings_are_potent()) + + return potent.filter(id__in=potent2_ids) + + @cached_property + def deletable_adults(self): + deletable_children_ids = self.deletable_children.values_list('id', flat=True) + potent = self.exclude_newer_than_threshold(self.filter_no_federation(self.adults), self.adult_threshold) + + def filter_children_are_deletable(): + for adult in potent: + for enfant, rele in utils.enfants(adult): + if enfant.id not in deletable_children_ids: + break + else: + yield adult + potent2 = list(filter_children_are_deletable()) + potent2_ids = [adult.id for adult in potent2] + + def filter_spouse_is_deletable(): + for adult in potent2: + conjoint = utils.conjoint(adult)[0] + if conjoint and conjoint.id not in potent2_ids: + continue + yield adult.id + return potent.filter(id__in=filter_spouse_is_deletable()) diff --git a/zoo/zoo_nanterre/models.py b/zoo/zoo_nanterre/models.py index 8886d76..6ba8264 100644 --- a/zoo/zoo_nanterre/models.py +++ b/zoo/zoo_nanterre/models.py @@ -1,3 +1,19 @@ +# zoo - versatile objects management +# Copyright (C) 2016 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from django.utils.translation import ugettext_lazy as _ from django.contrib.postgres.fields import JSONField from django.db import models diff --git a/zoo/zoo_nanterre/templates/admin/index.html b/zoo/zoo_nanterre/templates/admin/index.html index 879d5fc..2c9f62a 100644 --- a/zoo/zoo_nanterre/templates/admin/index.html +++ b/zoo/zoo_nanterre/templates/admin/index.html @@ -61,6 +61,17 @@   {% endif %} + {% if perms.zoo_data.action2_entity %} + + + + Fiches inactives + + +   +   + + {% endif %} diff --git a/zoo/zoo_nanterre/templates/admin/zoo_data/entity/inactive_index.html b/zoo/zoo_nanterre/templates/admin/zoo_data/entity/inactive_index.html new file mode 100644 index 0000000..3c5f5d4 --- /dev/null +++ b/zoo/zoo_nanterre/templates/admin/zoo_data/entity/inactive_index.html @@ -0,0 +1,72 @@ +{% extends "admin/base_site.html" %} +{% load i18n admin_urls static %} + +{% block extrastyle %} + {{ block.super }} + +{% endblock %} + +{% block coltype %}flex{% endblock %} + + +{% block breadcrumbs %} +
+ {% trans 'Home' %} + › Fiches inactives +
+{% endblock %} + +{% block content %} +
+ +
+

{{ fiches|length }} fiches. Dernier calcul le {{ timestamp }}. {% if duration %} Durée {{ duration }} secondes.{% endif %}{% if queries %}{{ queries }} requêtes.{% endif %}

+

Explication du calcul

+
    +
  • Une fiche sans activité est une fiche dont la dernière date de modification et les dernières lignes du journal n'ont pas bougé depuis un certain nombre de jours
    • +
  • Les fiches enfants sans activité depuis {{ child_delay }} jours sont considérées inactives.
    • +
  • Les fiches adultes sans activité depuis {{ adult_delay }} jours sont considérées inactives.
    • +
  • Les fratries (tous les enfants partageant au moins un parent commun) inactives et sans fédérations sont à supprimer.
    • +
  • Les adultes sans liens matrimoniaux, inactifs, sans fédérations et dont tous les enfants sont à supprimer sont à supprimer.
    • +
  • Les couples inactifs, sans fédérations et dont tous les enfants sont à supprimer sont à supprimer.
    • +
+

Configuration

+
    +
  • Le délai d'inactivité en jours pour les fiches enfants est configuré par le setting ZOO_NANTERRE_INACTIVITY_CHILD_DELAY
    • +
  • Le délai d'inactivité en jours pour les fiches adultes est configuré par le setting ZOO_NANTERRE_INACTIVITY_ADULT_DELAY
    • +
+
+ + {% comment %}Fiches inactives{% endcomment %} + + + + + + + + + + + + + {% for fiche in fiches %} + + + + + + + + + + {% endfor %} + +
Identifiant RSUPrénomNom d'usageNom de naissanceDate de naissanceStatut légalÂge
{{ fiche.id }}{{ fiche.prenoms }}{{ fiche.nom_d_usage }}{{ fiche.nom_de_naissance }}{{ fiche.date_de_naissance }}{{ fiche.statut_legal }}{{ fiche.age }}
+
+{% endblock %} diff --git a/zoo/zoo_nanterre/utils.py b/zoo/zoo_nanterre/utils.py index b101760..c0cd4c9 100644 --- a/zoo/zoo_nanterre/utils.py +++ b/zoo/zoo_nanterre/utils.py @@ -20,6 +20,8 @@ from __future__ import print_function import six import functools import uuid +import io +import csv import sys import re @@ -39,8 +41,11 @@ from django.db import connection from django.db.models import Q, F, Value, ExpressionWrapper, CharField, When, Case from django.db.models.functions import Least, Greatest, Coalesce, Concat from django.db import transaction -from django.utils.timezone import now, make_aware from django.contrib.auth.hashers import make_password +from django.http import HttpResponse + +from django.utils.timezone import now, make_aware +from django.utils.encoding import force_bytes from zoo.zoo_meta.models import EntitySchema, RelationSchema from zoo.zoo_data.models import Entity, Relation, Transaction, Log @@ -141,6 +146,16 @@ def adresses(individu): return adresses +def fratrie(individu): + assert individu.content['statut_legal'] == 'mineur' + + def helper_fratrie(): + for parent, relp in parents(individu): + for enfant, rele in enfants(parent): + yield enfant + return set(helper_fratrie()) + + def adresses_norel(individu): return [adresse for adresse, rel in adresses(individu)] @@ -1249,3 +1264,21 @@ def individu_caption(individu): if c['date_de_naissance']: s += u' - ' + c['date_de_naissance'] return s + + +def csv_export_response(rows, filename): + if sys.version >= (3,): + with io.StringIO(newline='') as f: + writer = csv.writer(f) + for row in rows: + writer.writerow(map(str, row)) + r = HttpResponse(f.getvalue(), content_type='text/csv') + else: + with io.BytesIO() as f: + writer = csv.writer(f) + for row in rows: + writer.writerow(map(force_bytes, row)) + r = HttpResponse(f.getvalue(), content_type='text/csv') + r['Content-Disposition'] = 'attachment; filename="%s"' % filename + return r + diff --git a/zoo/zoo_nanterre/views.py b/zoo/zoo_nanterre/views.py index 126a7e8..57160d1 100644 --- a/zoo/zoo_nanterre/views.py +++ b/zoo/zoo_nanterre/views.py @@ -17,19 +17,28 @@ # along with this program. If not, see . import csv +import itertools +import time from django.template.response import TemplateResponse from django.views.generic import TemplateView from django.shortcuts import redirect, get_object_or_404 -from django.http import Http404, FileResponse -from django.db.transaction import non_atomic_requests +from django.http import Http404, FileResponse, HttpResponseRedirect +from django.db.transaction import non_atomic_requests, atomic from django.db import connection +from django.conf import settings +from django.core.cache import cache +from django.utils.timezone import now + from django.contrib.auth.decorators import permission_required +from django.contrib import messages -from zoo.zoo_data.models import Job +from zoo.zoo_data.models import Entity from . import forms from .synchronize_federations import SynchronizeFederationsAction +from .inactivity import Inactivity +from . import utils class Demo(TemplateView): @@ -186,3 +195,77 @@ def synchronize_federations_apply(request, job_id, model_admin, **kwargs): raise Http404 job.action.set_apply(job) return redirect('admin:synchronize-federations') + + +def fiches_inactives(): + inactivity = Inactivity( + child_delay=getattr(settings, 'ZOO_NANTERRE_INACTIVITY_CHILD_DELAY', 365), + adult_delay=getattr(settings, 'ZOO_NANTERRE_INACTIVITY_ADULT_DELAY', 365), + ) + fiches = [] + for child in itertools.chain(inactivity.deletable_children, inactivity.deletable_adults): + utils.PersonSearch.add_age(child) + fiches.append({ + 'id': child.id, + 'prenoms': child.content['prenoms'], + 'nom_de_naissance': child.content['nom_de_naissance'], + 'nom_d_usage': child.content['nom_d_usage'], + 'date_de_naissance': child.content['date_de_naissance'], + 'statut_legal': child.content['statut_legal'], + 'age': child.age_label, + }) + fiches.sort(key=lambda f: f['id']) + return fiches + + +@permission_required('zoo_data.action1_entity') +@atomic +def inactive_index(request, model_admin, *args, **kwargs): + try: + timestamp, fiches = cache.get('fiches-inactives') + if 'recompute' in request.GET: + cache.delete('fiches-inactives') + return HttpResponseRedirect(request.path) + duration = None + queries = None + except (TypeError, ValueError): + try: + connection.force_debug_cursor = True + start = time.time() + fiches = fiches_inactives() + queries = len(connection.queries_log) + duration = time.time() - start + timestamp = now() + cache.set('fiches-inactives', (timestamp, fiches), 365 * 24 * 3600) + finally: + connection.force_debug_cursor = False + + # delete operation + if 'delete' in request.GET and request.method == 'POST': + Entity.objects.filter(id__in=[fiche['id'] for fiche in fiches]).delete() + messages.info(request, '%d fiches ont été supprimées.' % len(fiches)) + cache.delete('fiches-inactives') + return HttpResponseRedirect(request.path) + + # download csv export + if 'csv' in request.GET: + header = ['id', 'prenoms', 'nom_d_usage', 'nom_de_naissance', + 'date_de_naissance', 'statut_legal', 'age'] + + def rows(): + yield header + for fiche in fiches: + yield [fiche[key] for key in header] + return utils.csv_export_response(rows(), 'fiches-inactives-%s.csv' % timestamp) + + context = dict( + model_admin.admin_site.each_context(request), + title='Fiches inactives à supprimer', + fiches=fiches, + timestamp=timestamp, + duration=duration, + queries=queries, + child_delay=getattr(settings, 'ZOO_NANTERRE_INACTIVITY_CHILD_DELAY', 365), + adult_delay=getattr(settings, 'ZOO_NANTERRE_INACTIVITY_ADULT_DELAY', 365), + ) + return TemplateResponse(request, "admin/zoo_data/entity/inactive_index.html", context) -- 2.18.0