From d879e4132a1466fa7c3a3d7db95d8fefbaf213f7 Mon Sep 17 00:00:00 2001
From: Paul Marillonnet <pmarillonnet@entrouvert.com>
Date: Thu, 10 Jan 2019 18:33:25 +0100
Subject: [PATCH] add python3 compatibility (#29418)

---
 setup.py                      |  6 +++---
 src/ldaptools/ldap_source.py  |  2 +-
 src/ldaptools/ldapsync/cmd.py | 28 ++++++++++++++------------
 src/ldaptools/ldif_utils.py   |  9 ++++-----
 src/ldaptools/slapd.py        | 15 +++++++++-----
 src/ldaptools/synchronize.py  | 38 ++++++++++++++++++++++-------------
 src/ldaptools/utils.py        | 33 ++++++++++++++++++++++++------
 tests/conftest.py             |  4 +++-
 tests/test_ldif_utils.py      |  7 +++++--
 tests/test_synchronize.py     |  9 ++++++---
 tox.ini                       |  5 +++--
 11 files changed, 101 insertions(+), 55 deletions(-)

diff --git a/setup.py b/setup.py
index 92704a7..317cd22 100755
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@ from setuptools.command.sdist import sdist
 
 class eo_sdist(sdist):
     def run(self):
-        print "creating VERSION file"
+        print("creating VERSION file")
         if os.path.exists('VERSION'):
             os.remove('VERSION')
         version = get_version()
@@ -17,7 +17,7 @@ class eo_sdist(sdist):
         version_file.write(version)
         version_file.close()
         sdist.run(self)
-        print "removing VERSION file"
+        print("removing VERSION file")
         if os.path.exists('VERSION'):
             os.remove('VERSION')
 
@@ -38,7 +38,7 @@ def get_version():
         else:
             result = '0.0.0-%s' % len(subprocess.check_output(
                 ['git', 'rev-list', 'HEAD']).splitlines())
-        return result.replace('-', '.').replace('.g', '+g')
+        return result.decode('utf-8').replace('-', '.').replace('.g', '+g')
     return '0.0.0'
 
 
diff --git a/src/ldaptools/ldap_source.py b/src/ldaptools/ldap_source.py
index 9fd85eb..43f5ca3 100644
--- a/src/ldaptools/ldap_source.py
+++ b/src/ldaptools/ldap_source.py
@@ -25,7 +25,7 @@ class LDAPSource(object):
                 continue
             entry = idict(entry)
             if 'objectclass' in entry:
-                entry['objectclass'] = [istr(v) for v in entry['objectclass']]
+                entry['objectclass'] = [istr(v.decode('utf-8')) for v in entry['objectclass']]
             yield dn, entry
 
     def __iter__(self):
diff --git a/src/ldaptools/ldapsync/cmd.py b/src/ldaptools/ldapsync/cmd.py
index e1e62d0..c26b834 100644
--- a/src/ldaptools/ldapsync/cmd.py
+++ b/src/ldaptools/ldapsync/cmd.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import argparse
 import sys
 
@@ -18,16 +20,16 @@ def or_type(f1, f2):
     def f(value):
         try:
             return f1(value)
-        except argparse.ArgumentTypeError, e1:
+        except argparse.ArgumentTypeError as e1:
             try:
                 return f2(value)
-            except argparse.ArgumentTypeError, e2:
+            except argparse.ArgumentTypeError as e2:
                 raise argparse.ArgumentTypeError('%s and %s' % (e1.args[0], e2.args[0]))
     return f
 
 
 def object_class_pivot(value):
-    value = filter(None, map(str.strip, map(str.lower, value.split())))
+    value = list(filter(None, map(str.strip, map(str.lower, value.split()))))
     if len(value) != 2:
         raise argparse.ArgumentTypeError('%r is not a pair of an objectClass and an attribute name')
     return value
@@ -100,18 +102,18 @@ Base DN of the source is remapped to another DN in the target directory''')
     attributes = list(attributes)
     if not attributes:
         parser.print_help()
-        print 'Yout must give at least one attribute to synchronize'
+        print('You must give at least one attribute to synchronize')
 
     if options.verbose:
-        print 'Synchronizing ',
+        print('Synchronizing', end=' ')
     if hasattr(options.source_uri, 'read'):
         if options.verbose:
-            print options.source_uri.name,
+            print(options.source_uri.name, end=' ')
         source = ldif_utils.ListLDIFParser(options.source_uri)
         source.parse()
     else:
         if options.verbose:
-            print options.source_uri,
+            print(options.source_uri, end=' ')
         conn = paged.PagedLDAPObject(options.source_uri)
         if options.source_uri.startswith('ldapi://'):
             conn.sasl_interactive_bind_s("", ldap.sasl.external())
@@ -122,7 +124,7 @@ Base DN of the source is remapped to another DN in the target directory''')
                                        filterstr=options.source_filter)
 
     if options.verbose:
-        print 'to', options.target_uri
+        print('to', options.target_uri, end=' ')
     target_conn = paged.PagedLDAPObject(options.target_uri)
     if options.target_uri.startswith('ldapi://'):
         target_conn.sasl_interactive_bind_s("", ldap.sasl.external())
@@ -142,16 +144,16 @@ Base DN of the source is remapped to another DN in the target directory''')
     synchronize.build_actions()
     if options.verbose:
         for action in synchronize.actions:
-            print ' -', action
+            print(' -', action)
         if not synchronize.actions:
-            print 'Nothing to do.'
+            print('Nothing to do.')
     if not options.fake:
         synchronize.apply_actions()
     failed_actions = [action for action in synchronize.actions if action.errors]
     if failed_actions:
-        print >>sys.stderr, 'Some actions failed:'
+        print('Some actions failed:', file=sys.stderr)
         for action in failed_actions:
-            print ' -', action
+            print(' -', action)
             for error in action.errors:
-                print '  *', error
+                print('  *', error)
         raise SystemExit(1)
diff --git a/src/ldaptools/ldif_utils.py b/src/ldaptools/ldif_utils.py
index 37fbe8b..714fa08 100644
--- a/src/ldaptools/ldif_utils.py
+++ b/src/ldaptools/ldif_utils.py
@@ -2,8 +2,7 @@ import ldap
 import ldif
 from ldap.dn import dn2str
 
-from ldaptools.utils import idict, str2dn
-
+from ldaptools.utils import idict, str2dn, bytes2str_entry, str2bytes_entry
 
 class AddError(Exception):
     pass
@@ -18,13 +17,13 @@ class ListLDIFParser(ldif.LDIFParser):
         dn = str2dn(dn)
         dn = [[(part[0].lower(),) + part[1:] for part in rdn] for rdn in dn]
         dn = dn2str(dn)
-        self.entries.append((dn, entry))
+        self.entries.append((dn, bytes2str_entry(entry)))
 
     def add(self, conn):
         for dn, entry in self.entries:
             try:
-                conn.add_s(dn, ldap.modlist.addModlist(entry))
-            except Exception, e:
+                conn.add_s(dn, ldap.modlist.addModlist(str2bytes_entry(entry)))
+            except Exception as e:
                 raise AddError('error when adding %s' % dn, e)
 
     def __iter__(self):
diff --git a/src/ldaptools/slapd.py b/src/ldaptools/slapd.py
index a18f54c..53247f8 100644
--- a/src/ldaptools/slapd.py
+++ b/src/ldaptools/slapd.py
@@ -1,3 +1,4 @@
+import codecs
 import time
 import tempfile
 import shutil
@@ -6,7 +7,10 @@ import os
 import ldap
 import ldap.modlist
 import ldap.sasl
-import StringIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 import atexit
 
 from ldaptools.ldif_utils import ListLDIFParser
@@ -87,8 +91,9 @@ olcAccess: {{0}}to *
     process = None
     schemas = ['core', 'cosine', 'inetorgperson', 'nis', 'eduorg-200210-openldap', 'eduperson',
                'supann-2009']
-    schemas_ldif = [open(os.path.join(os.path.dirname(__file__),
-                                      'schemas', '%s.ldif' % schema)).read() for schema in schemas]
+    schemas_ldif = [codecs.open(os.path.join(os.path.dirname(__file__),
+                                      'schemas', '%s.ldif' % schema),
+                                      encoding='utf-8').read() for schema in schemas]
     checkpoints = None
     data_dirs = None
     db_index = 1
@@ -172,7 +177,7 @@ olcAccess: {{0}}to * by * manage
         if context:
             ldif = ldif.format(**context)
         slapadd = self.create_process([SLAPADD_PATH, '-v', '-n%d' % db, '-F', self.config_dir])
-        stdout, stderr = slapadd.communicate(input=ldif)
+        stdout, stderr = slapadd.communicate(input=bytearray(ldif, 'utf-8'))
         assert slapadd.returncode == 0, 'slapadd failed: %s' % stderr
 
     def start(self):
@@ -266,7 +271,7 @@ olcAccess: {{0}}to * by * manage
 
         if context:
             ldif = ldif.format(**context)
-        parser = ListLDIFParser(StringIO.StringIO(ldif))
+        parser = ListLDIFParser(StringIO(ldif))
         parser.parse()
         conn = self.get_connection_admin()
         parser.add(conn)
diff --git a/src/ldaptools/synchronize.py b/src/ldaptools/synchronize.py
index ed9f601..2d573b7 100644
--- a/src/ldaptools/synchronize.py
+++ b/src/ldaptools/synchronize.py
@@ -8,7 +8,8 @@ import ldap.modlist
 import ldap.dn
 
 
-from .utils import batch_generator, to_dict_of_set, idict, str2dn, istr
+from .utils import batch_generator, to_dict_of_set, idict, str2dn, istr, \
+        bytes2str_entry, str2bytes_entry
 
 
 @functools.total_ordering
@@ -48,7 +49,7 @@ class Action(object):
         for msgid in self.msgids:
             try:
                 self.results.append(conn.result2(msgid))
-            except ldap.LDAPError, e:
+            except ldap.LDAPError as e:
                 self.errors.append(e)
 
     def __str__(self):
@@ -62,7 +63,7 @@ class Create(Action):
     order = 3
 
     def do(self, conn):
-        self.msgids.append(conn.add(self.dn, ldap.modlist.addModlist(self.entry)))
+        self.msgids.append(conn.add(self.dn, ldap.modlist.addModlist(str2bytes_entry(self.entry))))
 
 
 class Rename(Action):
@@ -83,7 +84,7 @@ class Update(Action):
 
     def do(self, conn):
         modlist = []
-        for key, values in self.entry.iteritems():
+        for key, values in str2bytes_entry(self.entry).items():
             modlist.append((ldap.MOD_REPLACE, key, values))
         self.msgids.append(conn.modify(self.dn, modlist))
 
@@ -144,30 +145,39 @@ class Synchronize(object):
     def get_pivot_attribute(self, dn, entry):
         '''Find a pivot attribute value for an LDAP entry'''
         for objc, attr in self.pivot_attributes:
-            entry['objectclass'] = map(istr, entry['objectclass'])
-            if objc in entry['objectclass']:
+            if istr(objc) in [istr(oc.decode('utf-8'))
+                    if isinstance(oc, bytes) else oc
+                    for oc in entry['objectclass']]:
                 try:
                     value = entry[attr]
                 except KeyError:
                     raise Exception('entry %s missing pivot attribute %s: %s' % (dn, attr, entry))
                 break
         else:
-            raise Exception('entry %s has unknown objectclasses %s' % (dn, entry['objectclass']))
+            raise Exception('entry %s has unknown objectclasses %s' % (dn,
+                    [objclass for objclass in entry['objectclass']]))
         if len(value) != 1:
             raise Exception('entry %s pivot attribute %s must have only one value' % (dn, attr))
+        value = value[0]
+        """
+        may be used for input entries or output entries.
+        decoding may be required
+        """
+        if isinstance(value, bytes):
+            value = value.decode('utf-8')
         if attr in self.case_insensitive_attribute:
-            value = map(istr, value)
-        return objc, attr, value[0]
+            value = istr(value)
+        return objc, attr, value
 
     def get_target_entries(self, filterstr=None, attributes=[]):
         '''Return all target entries'''
         try:
             # Check base DN exist
             self.target_conn.search_s(self.target_dn, ldap.SCOPE_BASE)
-            l = self.target_conn.paged_search_ext_s(self.target_dn, ldap.SCOPE_SUBTREE,
+            res = self.target_conn.paged_search_ext_s(self.target_dn, ldap.SCOPE_SUBTREE,
                                                     filterstr=filterstr or self.all_filter,
                                                     attrlist=attributes)
-            return ((dn, idict(entry)) for dn, entry in l if dn)
+            return ((dn, idict(bytes2str_entry(entry))) for dn, entry in res if dn)
         except ldap.NO_SUCH_OBJECT:
             return []
 
@@ -215,7 +225,7 @@ class Synchronize(object):
                     seen_dn.add(out_dn)
                     self.rename(new_out_dn, target_dn)
                     renamed_dn[str2dn(new_out_dn)] = str2dn(target_dn)
-                if to_dict_of_set(out_entry) != to_dict_of_set(entry):
+                if to_dict_of_set(out_entry) != to_dict_of_set(bytes2str_entry(entry)):
                     new_entry = {}
                     for attribute in self.attributes:
                         if attribute in to_dict_of_set(entry):
@@ -233,10 +243,10 @@ class Synchronize(object):
         self.actions = []
         # Order source entries by DN depth
         entries = list(self.source)
-        entries.sort(key=lambda (dn, entry): len(str2dn(dn)))
+        entries.sort(key=lambda dn_entry: len(str2dn(dn_entry[0])))
         for dn, entry in entries:
             for key in entry.keys():
-                if not key in self.attributes:
+                if not str(key.lower()) in self.attributes:
                     del entry[key]
         # First create, rename and update
         for batch in batch_generator(entries, self.BATCH_SIZE):
diff --git a/src/ldaptools/utils.py b/src/ldaptools/utils.py
index cf4a2e1..2881927 100644
--- a/src/ldaptools/utils.py
+++ b/src/ldaptools/utils.py
@@ -1,4 +1,5 @@
 import ldap.dn
+from six import string_types
 
 
 # Copied from http://code.activestate.com/recipes/194371-case-insensitive-strings/
@@ -7,7 +8,7 @@ class istr(str):
     Performs like str except comparisons are case insensitive."""
 
     def __init__(self, strMe):
-        str.__init__(self, strMe)
+        super(str, self).__init__()
         self.__lowerCaseMe = strMe.lower()
 
     def __repr__(self):
@@ -80,7 +81,7 @@ class idict(dict):
     def findkey(self, item):
         """A caseless way of checking if a key exists or not.
         It returns None or the correct key."""
-        if not isinstance(item, str):
+        if not isinstance(item, string_types):
             raise TypeError('Keywords for this object must be strings. You supplied %s' % type(item))
         key = item.lower()
         try:
@@ -147,13 +148,13 @@ class idict(dict):
 
     def has_key(self, item):
         """A case insensitive test for keys."""
-        if not isinstance(item, str):
+        if not isinstance(item, string_types):
             return False               # should never have a non-string key
         return item.lower() in self._keydict           # does the key exist
 
     def __contains__(self, item):
         """A case insensitive __contains__."""
-        if not isinstance(item, str):
+        if not isinstance(item, string_types):
             return False               # should never have a non-string key
         return item.lower() in self._keydict           # does the key exist
 
@@ -212,7 +213,7 @@ class idict(dict):
         return True
 
     def __ne__(self, other):
-	return not (self == other)
+        return not (self == other)
 
 
 def batch_generator(gen, *batch_size):
@@ -229,7 +230,7 @@ def batch_generator(gen, *batch_size):
 
 
 def to_dict_of_set(d):
-    r = idict({k: set(v) for k, v in d.iteritems()})
+    r = idict({k: set(v) for k, v in d.items()})
     if 'objectclass' in r:
         r['objectclass'] = set(istr(v) for v in r['objectclass'])
     return r
@@ -237,3 +238,23 @@ def to_dict_of_set(d):
 
 def str2dn(s):
     return tuple(map(tuple, ldap.dn.str2dn(s)))
+
+
+def bytes2str_entry(entry):
+    str_entry = {}
+    for key, values in entry.items():
+        str_entry[key] = [v.decode('utf-8') if isinstance(v, bytes) else v for v in values]
+    return str_entry
+
+def str2bytes_entry(entry):
+    bytes_entry = {}
+
+    # python2 backward compat
+    try:
+        tested_types = basestring
+    except NameError:
+        tested_types = str
+
+    for key, values in entry.items():
+        bytes_entry[key] = [v.encode('utf-8') if isinstance(v, tested_types) else v for v in values]
+    return bytes_entry
diff --git a/tests/conftest.py b/tests/conftest.py
index 021bf0d..e350908 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import pytest
 import tempfile
 import os
@@ -98,7 +100,7 @@ def attributes_path(request, attributes):
     handle, path = tempfile.mkstemp()
     with open(path, 'w') as f:
         for attribute in attributes:
-            print >>f, ' %s ' % attribute
+            print(' %s ' % attribute, file=f)
         f.flush()
     def finalize():
         os.unlink(path)
diff --git a/tests/test_ldif_utils.py b/tests/test_ldif_utils.py
index ad1b961..0ed157c 100644
--- a/tests/test_ldif_utils.py
+++ b/tests/test_ldif_utils.py
@@ -1,10 +1,13 @@
-import StringIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 
 from ldaptools.ldif_utils import ListLDIFParser
 
 
 def test_ldifparser():
-    parser = ListLDIFParser(StringIO.StringIO('''dn: o=orga
+    parser = ListLDIFParser(StringIO('''dn: o=orga
 objectClass: organization
 
 '''))
diff --git a/tests/test_synchronize.py b/tests/test_synchronize.py
index a1d1f69..292b56a 100644
--- a/tests/test_synchronize.py
+++ b/tests/test_synchronize.py
@@ -1,4 +1,7 @@
-import StringIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 
 import ldap
 
@@ -17,7 +20,7 @@ def test_synchronize_ldif(slapd):
     conn = slapd.get_connection_admin()
 
     def syn_ldif(ldif):
-        parser = ListLDIFParser(StringIO.StringIO(ldif))
+        parser = ListLDIFParser(StringIO(ldif))
         parser.parse()
         synchronize = Synchronize(parser, 'o=orga', conn, 'o=orga',
                                   pivot_attributes=pivot_attributes,
@@ -163,7 +166,7 @@ def test_synchronize_deep_rename(slapd):
     conn = slapd.get_connection_admin()
 
     def syn_ldif(ldif):
-        parser = ListLDIFParser(StringIO.StringIO(ldif))
+        parser = ListLDIFParser(StringIO(ldif))
         parser.parse()
         synchronize = Synchronize(parser, 'o=orga', conn, 'o=orga',
                                   pivot_attributes=pivot_attributes,
diff --git a/tox.ini b/tox.ini
index 063b256..fec2f88 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,7 @@
 
 [tox]
 toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/ldaptools/
-envlist = coverage,package
+envlist = py2-coverage-ldap2,py{2,3}-coverage-ldap3,package
 
 [testenv]
 usedevelop = true
@@ -16,7 +16,8 @@ deps =
 	pytest
 	pytest-cov
 	pytest-random
-	python-ldap<3
+	ldap3: python-ldap>2
+	ldap2: python-ldap<3
 commands =
 	py.test {env:COVERAGE:} {posargs:--random tests}
 
-- 
2.20.1

