From 0df14616eaf55ca15c63e33e24aee089197a1e64 Mon Sep 17 00:00:00 2001
From: Paul Marillonnet <pmarillonnet@entrouvert.com>
Date: Mon, 6 Feb 2023 11:51:00 +0100
Subject: [PATCH] commands: get phone attributes usage count & misc statistics
 (#73677)

---
 .../get-phone-attributes-usage-count.py       | 145 ++++++++++++++++++
 tests/test_commands.py                        |  45 +++++-
 2 files changed, 189 insertions(+), 1 deletion(-)
 create mode 100644 src/authentic2/management/commands/get-phone-attributes-usage-count.py

diff --git a/src/authentic2/management/commands/get-phone-attributes-usage-count.py b/src/authentic2/management/commands/get-phone-attributes-usage-count.py
new file mode 100644
index 00000000..54a2e8ac
--- /dev/null
+++ b/src/authentic2/management/commands/get-phone-attributes-usage-count.py
@@ -0,0 +1,145 @@
+# authentic2 - versatile identity manager
+# Copyright (C) 2010-2023 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+from datetime import timedelta
+
+import phonenumbers
+from django.contrib.auth import get_user_model
+from django.contrib.contenttypes.models import ContentType
+from django.core.management.base import BaseCommand
+from django.utils.timezone import now
+from phonenumbers.phonenumberutil import PhoneNumberType, number_type
+
+from authentic2.models import Attribute, AttributeValue
+
+logger = logging.getLogger(__name__)
+
+User = get_user_model()
+
+
+class Command(BaseCommand):
+    help = 'Gather phone attributes usage statistics'
+
+    verbosity_to_log_level = {
+        0: logging.CRITICAL,
+        1: logging.WARNING,
+        2: logging.INFO,
+        3: logging.DEBUG,
+    }
+
+    def handle(self, **options):
+        # add StreamHandler for console output
+        handler = logging.StreamHandler()
+        handler.setLevel(level=self.verbosity_to_log_level[options['verbosity']])
+        logger.addHandler(handler)
+
+        phone = None
+        phone_count = 0
+
+        mobile = None
+        mobile_count = 0
+
+        try:
+            phone = Attribute.objects.get(name='phone', disabled=False)
+        except Attribute.DoesNotExist:
+            pass
+        try:
+            mobile = Attribute.objects.get(name='mobile', disabled=False)
+        except Attribute.DoesNotExist:
+            pass
+        user_ct = ContentType.objects.get_for_model(User)
+
+        login_threshold = now() - timedelta(days=365.25)
+        users = User.objects.filter(
+            is_active=True,
+            last_login__gte=login_threshold,
+            userexternalid__isnull=True,
+            oidc_account__isnull=True,
+        )
+        user_ids = users.values_list('id', flat=True)
+        user_count = users.count()
+        if not user_count:
+            return
+
+        if phone:
+            phone_count = users.filter(phone__isnull=False).count()
+        if mobile:
+            mobile_count = AttributeValue.objects.filter(
+                content_type=user_ct,
+                object_id__in=user_ids,
+                attribute=mobile,
+                content__isnull=False,
+            ).count()
+
+        phone_ratio = round(phone_count / user_count, 3)
+        mobile_ratio = round(mobile_count / user_count, 3)
+
+        logger.info('global count: %s', user_count)
+        logger.info('phone: %s (%s)', phone_count, phone_ratio)
+        logger.info('mobile: %s (%s)', mobile_count, mobile_ratio)
+
+        with_phone_ids = users.filter(phone__isnull=False).values_list('id', flat=True)
+        with_mobile_ids = AttributeValue.objects.filter(
+            content_type=user_ct,
+            object_id__in=user_ids,
+            content__isnull=False,
+            attribute=mobile,
+        ).values_list('object_id', flat=True)
+
+        both_fields_ids = set(with_phone_ids).intersection(set(with_mobile_ids))
+        both_fields_count = len(both_fields_ids)
+        both_fields_ratio = round(both_fields_count / user_count, 3)
+
+        logger.info('both fields: %s (%s)', both_fields_count, both_fields_ratio)
+
+        same_field_ids = []
+        for identifier in both_fields_ids:
+            try:
+                user = users.get(id=identifier)
+            except User.DoesNotExist:
+                continue
+            try:
+                atv = AttributeValue.objects.get(content_type=user_ct, object_id=identifier, attribute=mobile)
+            except AttributeValue.DoesNotExist:
+                continue
+            if user.phone and user.phone == atv.to_python():
+                same_field_ids.append(identifier)
+        same_field_count = len(same_field_ids)
+        same_field_ratio = round(same_field_count / user_count, 3)
+
+        logger.info('same value for both fields: %s (%s)', same_field_count, same_field_ratio)
+
+        if not with_phone_ids:
+            return
+
+        mobile_as_phone_ids = []
+        for user in users.filter(phone__isnull=False):
+            try:
+                pn = phonenumbers.parse(user.phone, 'FR')
+            except phonenumbers.NumberParseException:
+                continue
+            if number_type(pn) == PhoneNumberType.MOBILE:
+                mobile_as_phone_ids.append(user.id)
+
+        mobile_as_phone_count = len(mobile_as_phone_ids)
+        mobile_as_phone_ratio = round(mobile_as_phone_count / len(with_phone_ids), 3)
+        logger.info(
+            'phone field used for mobile: %s of %s (%s)',
+            mobile_as_phone_count,
+            len(with_phone_ids),
+            mobile_as_phone_ratio,
+        )
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 72821af9..0622fb99 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -42,7 +42,7 @@ from authentic2.a2_rbac.models import (
 from authentic2.a2_rbac.utils import get_default_ou, get_operation
 from authentic2.apps.journal.models import Event
 from authentic2.custom_user.models import DeletedUser
-from authentic2.models import UserExternalId
+from authentic2.models import Attribute, AttributeValue, UserExternalId
 from authentic2.utils import crypto
 from authentic2_auth_oidc.models import OIDCAccount, OIDCClaimMapping, OIDCProvider
 
@@ -655,3 +655,46 @@ def test_oidc_sync_provider(db, app, admin, settings, caplog, deletion_number, d
             assert User.objects.filter(first_name='Mod', last_name='Ified').count() in range(
                 20 - deletion_number, 21
             )
+
+
+def test_get_phone_attributes_usage_count(db, app, admin, settings, caplog, capsys):
+    # recreated (soon-to-be-deprecated) separate phone/mobile number attributes
+    admin.last_login = now()
+    admin.save()
+    mobile_attr = Attribute.objects.create(
+        label='Mobile',
+        name='mobile',
+        kind='phone_number',
+    )
+    Attribute.objects.create(
+        label='Phone',
+        name='phone',
+        kind='phone_number',
+    )
+
+    user_ct = ContentType.objects.get_for_model(User)
+    for i in range(900):
+        user = User.objects.create(
+            username=f'user_{i}',
+            first_name='User',
+            last_name=f'{i}',
+            email=f'user_{i}@example.com',
+            phone=f'+33612345{i:03d}' if i % 2 else f'+33112345{i:03d}',
+            last_login=now() if i % 9 in range(8) else now() - datetime.timedelta(days=700),
+        )
+
+        AttributeValue.objects.create(
+            content_type=user_ct,
+            object_id=user.id,
+            attribute=mobile_attr,
+            content=f'+33612345{i:03d}',
+        )
+
+    call_command('get-phone-attributes-usage-count', '-v1')
+    assert caplog.records
+    assert 'global count: 801' == caplog.records[0].message
+    assert 'phone: 800 (0.999)' == caplog.records[1].message
+    assert 'mobile: 800 (0.999)' == caplog.records[2].message
+    assert 'both fields: 800 (0.999)' == caplog.records[3].message
+    assert 'same value for both fields: 400 (0.499)' == caplog.records[4].message
+    assert 'phone field used for mobile: 400 of 800 (0.5)' == caplog.records[5].message
-- 
2.39.2

