From 0e806a13e24cc40a792c07eea60b91bac9a99b6a Mon Sep 17 00:00:00 2001
From: Paul Marillonnet <pmarillonnet@entrouvert.com>
Date: Wed, 21 Sep 2022 15:09:39 +0200
Subject: [PATCH 2/3] csv_import adapt user csv logic to new phone_number kind
 (#69365)

---
 src/authentic2/csv_import.py | 34 ++++++++++++++++++++++++++++++++--
 tests/test_csv_import.py     | 30 ++++++++++++++++--------------
 tests/test_user_manager.py   | 13 ++++++-------
 3 files changed, 54 insertions(+), 23 deletions(-)

diff --git a/src/authentic2/csv_import.py b/src/authentic2/csv_import.py
index 8e20342e0..d8ec7adbf 100644
--- a/src/authentic2/csv_import.py
+++ b/src/authentic2/csv_import.py
@@ -19,14 +19,17 @@ import csv
 import io
 
 import attr
+import phonenumbers
 from chardet.universaldetector import UniversalDetector
 from django import forms
+from django.conf import settings
 from django.contrib.auth.hashers import identify_hasher
 from django.core.exceptions import FieldDoesNotExist, ValidationError
 from django.core.validators import RegexValidator
 from django.db import IntegrityError, models
 from django.db.transaction import atomic
 from django.utils.encoding import force_bytes, force_text
+from django.utils.timezone import now
 from django.utils.translation import ugettext as _
 
 from authentic2 import app_settings
@@ -467,7 +470,7 @@ class UserCsvImporter:
                     header.key = True
             elif header.name not in SPECIAL_COLUMNS:
                 try:
-                    if header.name in ['email', 'first_name', 'last_name', 'username']:
+                    if header.name in ['email', 'phone', 'first_name', 'last_name', 'username']:
                         User._meta.get_field(header.name)
                         header.field = True
                         if header.name == 'email':
@@ -540,14 +543,39 @@ class UserCsvImporter:
 
     def parse_row(self, form_class, row, line):
         data = {}
-
+        errors = {}
         for header in self.headers:
             try:
                 data[header.name] = row[header.column - 1]
             except IndexError:
                 pass
 
+        if 'phone' in data:
+            pn = None
+            try:
+                # ensure E.164 international representation of phone numbers
+                pn = phonenumbers.parse(data['phone'])
+            except phonenumbers.NumberParseException:
+                try:
+                    # fallback on default local dial
+                    pn = phonenumbers.parse(
+                        data['phone'],
+                        settings.PHONE_COUNTRY_CODES[settings.DEFAULT_COUNTRY_CODE],
+                    )
+                except phonenumbers.NumberParseException:
+                    pass
+
+            if pn:
+                # fill multi value field
+                data['phone_0'] = str(pn.country_code)
+                data['phone_1'] = '0' * (pn.number_of_leading_zeros or 0) + str(pn.national_number)
+                data.pop('phone')
+            else:
+                # E.164 compliant parsing failed, leave the number untouched, add an error
+                errors.update({'phone': [_('Enter a valid phone number.')]})
+
         form = form_class(data=data)
+        form.errors.update(errors)
         form.is_valid()
 
         def get_form_errors(form, name):
@@ -713,6 +741,8 @@ class UserCsvImporter:
                     setattr(user, cell.header.name, cell.value)
                     if cell.header.name == 'email' and cell.header.verified:
                         user.set_email_verified(True)
+                    if cell.header.name == 'phone' and cell.header.verified:
+                        user.phone_verified_on = now()
                     cell.action = 'updated'
                     continue
             cell.action = 'nothing'
diff --git a/tests/test_csv_import.py b/tests/test_csv_import.py
index 91c761968..035b4b609 100644
--- a/tests/test_csv_import.py
+++ b/tests/test_csv_import.py
@@ -201,7 +201,7 @@ x,x,x,x'''
         CsvHeader(1, 'email', field=True, key=True, verified=True),
         CsvHeader(2, 'first_name', field=True),
         CsvHeader(3, 'last_name', field=True),
-        CsvHeader(4, 'phone', attribute=True),
+        CsvHeader(4, 'phone', field=True),
     ]
     assert importer.has_errors
     assert len(importer.rows) == 3
@@ -211,7 +211,7 @@ x,x,x,x'''
     assert all(error == Error('data-error') for error in importer.rows[2].cells[0].errors)
     assert not importer.rows[2].cells[1].errors
     assert not importer.rows[2].cells[2].errors
-    assert not importer.rows[2].cells[3].errors
+    assert importer.rows[2].cells[3].errors
     assert all(error == Error('data-error') for error in importer.rows[2].cells[3].errors)
 
     assert importer.updated == 0
@@ -225,7 +225,8 @@ x,x,x,x'''
     assert thomas.attributes.first_name == 'Thomas'
     assert thomas.last_name == 'Noël'
     assert thomas.attributes.last_name == 'Noël'
-    assert thomas.attributes.phone == '1234'
+    # phonenumbers' e.164 representation from a settings.DEFAULT_COUNTRY_CODE dial:
+    assert thomas.attributes.phone == '+331234'
     assert thomas.password
 
     fpeters = User.objects.get(email='fpeters@entrouvert.com')
@@ -235,7 +236,8 @@ x,x,x,x'''
     assert fpeters.attributes.first_name == 'Frédéric'
     assert fpeters.last_name == 'Péters'
     assert fpeters.attributes.last_name == 'Péters'
-    assert fpeters.attributes.phone == '5678'
+    # phonenumbers' e.164 representation from a settings.DEFAULT_COUNTRY_CODE dial:
+    assert fpeters.attributes.phone == '+335678'
 
 
 def test_simulate(profile, user_csv_importer_factory):
@@ -251,7 +253,7 @@ x,x,x,x'''
         CsvHeader(1, 'email', field=True, key=True, verified=True),
         CsvHeader(2, 'first_name', field=True),
         CsvHeader(3, 'last_name', field=True),
-        CsvHeader(4, 'phone', attribute=True),
+        CsvHeader(4, 'phone', field=True),
     ]
     assert importer.has_errors
     assert len(importer.rows) == 3
@@ -277,7 +279,7 @@ tnoel@entrouvert.com,Thomas,Noël,1234'''
     importer = user_csv_importer_factory(content)
 
     user = User.objects.create(ou=get_default_ou())
-    user.attributes.phone = '1234'
+    user.attributes.phone = '+331234'
 
     assert importer.run()
 
@@ -298,7 +300,7 @@ tnoel@entrouvert.com,Thomas,Noël,1234'''
     importer = user_csv_importer_factory(content)
 
     user = User.objects.create()
-    user.attributes.phone = '1234'
+    user.attributes.phone = '+331234'
 
     assert importer.run()
 
@@ -318,7 +320,7 @@ tnoel@entrouvert.com,Thomas,Noël,1234'''
     importer = user_csv_importer_factory(content)
 
     user = User.objects.create()
-    user.attributes.phone = '1234'
+    user.attributes.phone = '+331234'
 
     assert importer.run()
 
@@ -356,7 +358,7 @@ tnoel@entrouvert.com,Thomas,Noël,1234'''
     importer = user_csv_importer_factory(content)
 
     user = User.objects.create(ou=get_default_ou())
-    user.attributes.phone = '1234'
+    user.attributes.phone = '+331234'
 
     user = User.objects.create(email='tnoel@entrouvert.com', ou=get_default_ou())
 
@@ -378,7 +380,7 @@ tnoel@entrouvert.com,Thomas,Noël,1234'''
     importer = user_csv_importer_factory(content)
 
     user = User.objects.create()
-    user.attributes.phone = '1234'
+    user.attributes.phone = '+331234'
 
     User.objects.create(email='tnoel@entrouvert.com', ou=get_default_ou())
 
@@ -400,7 +402,7 @@ tnoel@entrouvert.com,Thomas,Noël,1234'''
     importer = user_csv_importer_factory(content)
 
     user = User.objects.create()
-    user.attributes.phone = '1234'
+    user.attributes.phone = '+331234'
 
     thomas = User.objects.create(email='tnoel@entrouvert.com', ou=get_default_ou())
 
@@ -418,7 +420,7 @@ tnoel@entrouvert.com,Thomas,Noël,1234'''
     thomas.refresh_from_db()
     assert not thomas.first_name
     assert not thomas.last_name
-    assert thomas.attributes.phone == '1234'
+    assert thomas.attributes.phone == '+331234'
 
 
 def test_external_id(profile, user_csv_importer_factory):
@@ -436,7 +438,7 @@ app1,2,tnoel@entrouvert.com,Thomas,Noël,1234
         CsvHeader(3, 'email', field=True, verified=True),
         CsvHeader(4, 'first_name', field=True),
         CsvHeader(5, 'last_name', field=True),
-        CsvHeader(6, 'phone', attribute=True),
+        CsvHeader(6, 'phone', field=True),
     ]
     assert not importer.has_errors
     assert len(importer.rows) == 2
@@ -449,7 +451,7 @@ app1,2,tnoel@entrouvert.com,Thomas,Noël,1234
         assert thomas.attributes.first_name == 'Thomas'
         assert thomas.last_name == 'Noël'
         assert thomas.attributes.last_name == 'Noël'
-        assert thomas.attributes.phone == '1234'
+        assert thomas.attributes.phone == '+331234'
 
     importer = user_csv_importer_factory(content)
     assert importer.run(), importer.errors
diff --git a/tests/test_user_manager.py b/tests/test_user_manager.py
index e113972f2..b18c68127 100644
--- a/tests/test_user_manager.py
+++ b/tests/test_user_manager.py
@@ -468,7 +468,7 @@ def test_user_import(encoding, transactional_db, app, admin, ou1, admin_ou1):
             'users.csv',
             '''email key verified,first_name,last_name,phone
 tnoel@entrouvert.com,Thomas,Noël,1234
-fpeters@entrouvert.com,Frédéric,Péters,5678
+fpeters@entrouvert.com,Frédéric,Péters,+325678
 john.doe@entrouvert.com,John,Doe,9101112
 x,x,x,x'''.encode(
                 encoding
@@ -530,7 +530,7 @@ x,x,x,x'''.encode(
     assert len(response.pyquery('tr.row-cells-errors')) == 1
     assert sum(bool(response.pyquery(td).text()) for td in response.pyquery('tr.row-cells-errors td li')) == 2
     assert 'Enter a valid email address' in response.pyquery('tr.row-cells-errors td.cell-email li').text()
-    assert 'Phone number can start with' in response.pyquery('tr.row-cells-errors td.cell-phone li').text()
+    assert 'Enter a valid phone number' in response.pyquery('tr.row-cells-errors td.cell-phone li').text()
 
     assert User.objects.count() == user_count
 
@@ -549,7 +549,7 @@ x,x,x,x'''.encode(
             email='tnoel@entrouvert.com',
             first_name='Thomas',
             last_name='Noël',
-            attribute_values__content='1234',
+            attribute_values__content='+331234',
         ).count()
         == 1
     )
@@ -558,7 +558,7 @@ x,x,x,x'''.encode(
             email='fpeters@entrouvert.com',
             first_name='Frédéric',
             last_name='Péters',
-            attribute_values__content='5678',
+            attribute_values__content='+325678',
         ).count()
         == 1
     )
@@ -567,7 +567,7 @@ x,x,x,x'''.encode(
             email='john.doe@entrouvert.com',
             first_name='John',
             last_name='Doe',
-            attribute_values__content='9101112',
+            attribute_values__content='+339101112',
         ).count()
         == 1
     )
@@ -697,7 +697,6 @@ def test_user_import_attributes(transactional_db, app, admin):
     assert 'Enter a valid date.' in response.text
     assert 'birthdate must be in the past and greater or equal than 1900-01-01.' in response.text
     assert 'The value must be a valid french postcode' in response.text
-    assert 'Phone number can start with a + and must contain only digits' in response.text
 
     assert User.objects.count() == user_count + 1
     elliot = User.objects.filter(email='elliot@universalpictures.com')[0]
@@ -707,7 +706,7 @@ def test_user_import_attributes(transactional_db, app, admin):
     assert elliot.attributes.values['saintsday'].content == '2019-07-20'
     assert elliot.attributes.values['birthdate'].content == '1972-05-26'
     assert elliot.attributes.values['zip'].content == '75014'
-    assert elliot.attributes.values['phone'].content == '1234'
+    assert elliot.attributes.values['phone'].content == '+331234'
 
     csv_lines[2] = "et@universalpictures.com,ET,the Extra-Terrestrial,,,,,,42000,+888 5678"
     response = import_csv('\n'.join(csv_lines), app)
-- 
2.37.2

