From 7808f51a92b4cf2f57bbdf75738e433f6cd7c110 Mon Sep 17 00:00:00 2001
From: Paul Marillonnet <pmarillonnet@entrouvert.com>
Date: Mon, 6 Feb 2023 11:51:00 +0100
Subject: [PATCH] commands: get phone attributes usage count & misc statistics
 (#73677)

---
 .../get-phone-attributes-usage-count.py       | 148 ++++++++++++++++++
 tests/test_commands.py                        |  41 ++++-
 2 files changed, 188 insertions(+), 1 deletion(-)
 create mode 100644 src/authentic2/management/commands/get-phone-attributes-usage-count.py

diff --git a/src/authentic2/management/commands/get-phone-attributes-usage-count.py b/src/authentic2/management/commands/get-phone-attributes-usage-count.py
new file mode 100644
index 00000000..df4ef7aa
--- /dev/null
+++ b/src/authentic2/management/commands/get-phone-attributes-usage-count.py
@@ -0,0 +1,148 @@
+# authentic2 - versatile identity manager
+# Copyright (C) 2010-2023 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+
+import phonenumbers
+from django.contrib.auth import get_user_model
+from django.contrib.contenttypes.models import ContentType
+from django.core.management.base import BaseCommand
+from phonenumbers.phonenumberutil import PhoneNumberType, number_type
+
+from authentic2.models import Attribute, AttributeValue
+
+logger = logging.getLogger(__name__)
+
+User = get_user_model()
+
+
+class Command(BaseCommand):
+    help = 'Gather phone attributes usage statistics'
+
+    verbosity_to_log_level = {
+        0: logging.CRITICAL,
+        1: logging.WARNING,
+        2: logging.INFO,
+        3: logging.DEBUG,
+    }
+
+    def handle(self, **options):
+        # add StreamHandler for console output
+        handler = logging.StreamHandler()
+        handler.setLevel(level=self.verbosity_to_log_level[options['verbosity']])
+        logger.addHandler(handler)
+
+        phone = None
+        phone_count = 0
+
+        mobile = None
+        mobile_count = 0
+
+        try:
+            phone = Attribute.objects.get(name='phone', disabled=False)
+        except Attribute.DoesNotExist:
+            pass
+        try:
+            mobile = Attribute.objects.get(name='mobile', disabled=False)
+        except Attribute.DoesNotExist:
+            pass
+        user_ct = ContentType.objects.get_for_model(User)
+        user_count = User.objects.filter(is_active=True).count()
+
+        if user_count:
+            if phone:
+                phone_count = AttributeValue.objects.filter(
+                    content_type=user_ct,
+                    object_id__isnull=False,
+                    attribute=phone,
+                    content__isnull=False,
+                ).count()
+            if mobile:
+                mobile_count = AttributeValue.objects.filter(
+                    content_type=user_ct,
+                    object_id__isnull=False,
+                    attribute=mobile,
+                    content__isnull=False,
+                ).count()
+
+            phone_ratio = round(phone_count / user_count, 3)
+            mobile_ratio = round(mobile_count / user_count, 3)
+
+            logger.info('global count: %s', user_count)
+            logger.info('phone: %s (%s)', phone_count, phone_ratio)
+            logger.info('mobile: %s (%s)', mobile_count, mobile_ratio)
+
+        # take a random sample of 500 Users
+        sample = User.objects.order_by('?')[:500]
+        sample_ids = sample.values_list('id', flat=True)
+        sample_count = sample.count()
+        if sample_count:
+            with_phone_ids = User.objects.filter(id__in=sample_ids, phone__isnull=False).values_list(
+                'id', flat=True
+            )
+            with_mobile_ids = AttributeValue.objects.filter(
+                content_type=user_ct,
+                object_id__in=[u.id for u in sample],
+                content__isnull=False,
+                attribute=mobile,
+            ).values_list('object_id', flat=True)
+
+            both_fields_ids = set(with_phone_ids).intersection(set(with_mobile_ids))
+            both_fields_count = len(both_fields_ids)
+            both_fields_ratio = round(both_fields_count / sample_count, 3)
+
+            logger.info('both fields: %s of %s (%s)', both_fields_count, sample_count, both_fields_ratio)
+
+            same_field_ids = []
+            for identifier in both_fields_ids:
+                try:
+                    user = User.objects.get(id=identifier)
+                except User.DoesNotExist:
+                    continue
+                try:
+                    atv = AttributeValue.objects.get(
+                        content_type=user_ct, object_id=identifier, attribute=mobile
+                    )
+                except AttributeValue.DoesNotExist:
+                    continue
+                if user.phone and user.phone == atv.to_python():
+                    same_field_ids.append(identifier)
+            same_field_count = len(same_field_ids)
+            same_field_ratio = round(same_field_count / sample_count, 3)
+
+            logger.info(
+                'same value for both fields: %s of %s (%s)', same_field_count, sample_count, same_field_ratio
+            )
+
+            mobile_as_phone_ids = []
+            for user in sample:
+                if not user.phone:
+                    continue
+                try:
+                    pn = phonenumbers.parse(user.phone, 'FR')
+                except phonenumbers.NumberParseException:
+                    continue
+                if number_type(pn) == PhoneNumberType.MOBILE:
+                    mobile_as_phone_ids.append(user.id)
+
+            mobile_as_phone_count = len(mobile_as_phone_ids)
+            mobile_as_phone_ratio = round(mobile_as_phone_count / sample_count, 3)
+            logger.info(
+                'phone field used for mobile: %s of %s (%s)',
+                mobile_as_phone_count,
+                sample_count,
+                mobile_as_phone_ratio,
+            )
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 72821af9..4c9f0f7c 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -42,7 +42,7 @@ from authentic2.a2_rbac.models import (
 from authentic2.a2_rbac.utils import get_default_ou, get_operation
 from authentic2.apps.journal.models import Event
 from authentic2.custom_user.models import DeletedUser
-from authentic2.models import UserExternalId
+from authentic2.models import Attribute, AttributeValue, UserExternalId
 from authentic2.utils import crypto
 from authentic2_auth_oidc.models import OIDCAccount, OIDCClaimMapping, OIDCProvider
 
@@ -655,3 +655,42 @@ def test_oidc_sync_provider(db, app, admin, settings, caplog, deletion_number, d
             assert User.objects.filter(first_name='Mod', last_name='Ified').count() in range(
                 20 - deletion_number, 21
             )
+
+
+def test_get_phone_attributes_usage_count(db, app, admin, settings, caplog, capsys):
+    # recreated (soon-to-be-deprecated) separate phone/mobile number attributes
+    mobile_attr = Attribute.objects.create(
+        label='Mobile',
+        name='mobile',
+        kind='phone_number',
+    )
+    Attribute.objects.create(
+        label='Phone',
+        name='phone',
+        kind='phone_number',
+    )
+
+    user_ct = ContentType.objects.get_for_model(User)
+    for i in range(300):
+        user = User.objects.create(
+            username=f'user_{i}',
+            first_name='User',
+            last_name=f'{i}',
+            email=f'user_{i}@example.com',
+            phone=f'+33612345{i:03d}' if i % 2 else f'+33112345{i:03d}',
+        )
+        AttributeValue.objects.create(
+            content_type=user_ct,
+            object_id=user.id,
+            attribute=mobile_attr,
+            content=f'+33612345{i:03d}',
+        )
+
+    call_command('get-phone-attributes-usage-count', '-v1')
+    assert caplog.records
+    assert 'global count: 301' == caplog.records[0].message
+    assert 'phone: 300 (0.997)' == caplog.records[1].message
+    assert 'mobile: 300 (0.997)' == caplog.records[2].message
+    assert 'both fields: 300 of 301 (0.997)' == caplog.records[3].message
+    assert 'same value for both fields: 150 of 301 (0.498)' == caplog.records[4].message
+    assert 'phone field used for mobile: 150 of 301 (0.498)' == caplog.records[5].message
-- 
2.39.2

