0001-feeder-preseve-categories-and-form-names-order-36930.patch

Serghei Mihai (congés, retour 15/05), 02 mars 2020 17:04

Voir les différences: en ligne côte à côte

Subject: [PATCH] feeder: preseve categories and form names order (#36930)

 tests/olap.model   |  4 +-
 tests/test_wcs.py  | 24 +++++++++++-
 wcs_olap/feeder.py | 92 ++++++++++++++++++++++++++++++++++++----------
 3 files changed, 98 insertions(+), 22 deletions(-)

                    "name" : "category",
                    "order_by" : "category.label",
                    "type" : "integer",
                    "value" : "category.id",
                    "value" : "category.ref",
                    "value_label" : "category.label"
                 },
+                {
-...
                    "name": "formdef",
                    "order_by": "formdef.label",
                    "type": "integer",
                    "value": "formdef.id",
                    "value": "formdef.ref",
                    "value_label": "formdef.label"
                 },
+                {

     import utils
     from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
     def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog):
         # create temp schema remnant to see if it's cleaned
-...
             ('agent', 'id'),
             ('agent', 'label'),
             ('category', 'id'),
             ('category', 'ref'),
             ('category', 'label'),
             ('channel', 'id'),
             ('channel', 'label'),
-...
             ('formdata_demande_field_itemOpen', 'id'),
             ('formdata_demande_field_itemOpen', 'label'),
             ('formdef', 'id'),
             ('formdef', 'ref'),
             ('formdef', 'category_id'),
             ('formdef', 'label'),
             ('hour', 'id'),
-...
                 olap_cmd(no_log_errors=False)
         assert 'invalid signature' in caplog.text
     def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog):
         @httmock.urlmatch()
         def return_invalid_json(url, request):
-...
                 formdata = c.fetchone()
                 assert formdata[0] == bazouka_id
                 assert formdata[1] == open_new_id
     def test_create_reference_column(postgres_db, olap_cmd):
         olap_cmd()
         conn = postgres_db.conn()
         with postgres_db.conn() as conn:
             with conn.cursor() as c:
                 conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
                 c.execute('ALTER TABLE olap.category DROP COLUMN ref')
                 c.execute('ALTER TABLE olap.formdef DROP COLUMN ref')
         olap_cmd()
         with postgres_db.conn() as conn:
             with conn.cursor() as c:
                 c.execute('SELECT * FROM olap.category')
                 for line in c.fetchall():
                     assert len(line) == 3

                         'label': 'catégorie',
                         'join': ['formdef', 'category'],
                         'type': 'integer',
                         'value': 'category.id',
                         'value': 'category.ref',
                         'value_label': 'category.label',
                         'order_by': 'category.label',
                     },
-...
                         'label': 'formulaire',
                         'join': ['formdef'],
                         'type': 'integer',
                         'value': 'formdef.id',
                         'value': 'formdef.ref',
                         'value_label': 'formdef.label',
                         'order_by': 'formdef.label',
                     },
-...
             self.ex(query, vars=(name,))
             return self.cur.fetchone()[0]
         def update_table_sequence_number(self, name):
             self.ex("""SELECT setval(pg_get_serial_sequence('{name}', 'id'),
             (SELECT GREATEST(1, MAX(id)) FROM {name}))""", ctx={'name': quote(name)})
         def create_labeled_table_serial(self, name, comment):
             self.create_table(
                 name, [['id', 'serial primary key'], ['label', 'varchar']], comment=comment)
-...
                     'INSERT INTO {schema_temp}.{name} SELECT * FROM {schema}.{name}',
                     ctx={'name': quote(name)}
+                )
                 # Update sequence
                 self.ex("""SELECT setval(pg_get_serial_sequence('{name}', 'id'),
                 (SELECT GREATEST(1, MAX(id)) FROM {name}))""", ctx={'name': quote(name)})
                 self.update_table_sequence_number(name)
         def create_referenced_table(self, name, fields, comment):
             # add primary key and reference fields
             new_fields = [['id', 'serial primary key'], ['ref', 'varchar UNIQUE']] + fields
             self.create_table(name, new_fields, comment=comment)
         def do_referenced_data(self, name, data, result_column, update_column='label'):
             to_insert = []
             for item in data:
                 ref = item[0]
                 self.ex(
                     'SELECT ref, {column} FROM {name} WHERE ref = %s',
                     ctx={'name': quote(name), 'column': quote(update_column)},
                     vars=(ref,))
                 if self.cur.fetchall():
                     for item in self.cur.fetchall():
                         self.ex('UPDATE {name} SET {column}=%s WHERE ref=%s',
                                 ctx={'name': quote(name), 'column': quote(update_column)},
                                 vars=[item[1], ref])
                 else:
                     to_insert.append(item)
             if to_insert:
                 columns_values = ', '.join(['%s' for x in range(len(item))])
                 tmpl = ', '.join(['(DEFAULT, %s)' % columns_values] * len(data))
                 query = 'INSERT INTO {name} VALUES %s' % tmpl
                 self.ex(query, ctx={'name': quote(name)}, # 'column': quote(update_column)},
                         vars=list(itertools.chain(*to_insert)))
             result = {}
             self.ex('SELECT id, {column} FROM {name}', ctx={'name': quote(name),
                                                             'column': result_column})
             for _id, column  in self.cur.fetchall():
                 result[column] = _id
             return result
         def create_labeled_table(self, name, labels, comment=None):
             self.create_table(
-...
                 res[label] = id_
             return res
         def do_category_table(self):
             fields = [['label', 'varchar']]
             table_name = self.default_ctx['category_table']
             self.create_referenced_table(table_name, fields, 'catégorie')
             categories_data = [(c.slug, c.title) for c in self.categories]
             tmp_cat_map = self.do_referenced_data(table_name, categories_data, 'label')
             self.update_table_sequence_number(table_name)
             # remap categories ids to ids in the table
             return dict((c.title, tmp_cat_map[c.title]) for c in self.categories)
         def do_formdef_table(self):
             categories_mapping = self.do_category_table()
             formdef_fields = [['category_id', 'integer REFERENCES {category_table} (id)'],
                               ['label', 'varchar']
+            ]
             table_name = self.default_ctx['form_table']
             self.create_referenced_table(table_name, formdef_fields, 'types de formulaire')
             formdefs = [(form.slug, categories_mapping.get(form.schema.category),
                          form.schema.name) for form in self.formdefs if form.count]
             self.formdefs_mapping = self.do_referenced_data(table_name, formdefs, 'ref')
             self.update_table_sequence_number(table_name)
         def do_base_table(self):
             # channels
             self.create_labeled_table('channel', [[c[0], c[2]] for c in self.channels],
-...
             self.role_mapping = dict(
                 (role.id, tmp_role_map[role.name]) for role in self.roles)
             # categories
             tmp_cat_map = self.create_labeled_table(
                 'category', enumerate(c.title for c in self.categories), comment='catégorie')
             self.categories_mapping = dict((c.slug, tmp_cat_map[c.title]) for c in self.categories)
             # forms
             self.do_formdef_table()
             self.create_labeled_table('hour', zip(range(0, 24), map(str, range(0, 24))),
                                       comment='heures')
             self.create_labeled_table('status', self.status,
                                       comment='statuts simplifiés')
             self.ex('CREATE TABLE {form_table} (id serial PRIMARY KEY,'
                     ' category_id integer REFERENCES {category_table} (id),'
                     ' label varchar)')
             self.ex('COMMENT ON TABLE {form_table} IS %s', vars=('types de formulaire',))
             # agents
             self.create_labeled_table_serial('agent', comment='agents')
-...
             self.status_mapping = dict((s.id, tmp_status_map[s.name]) for s in statuses)
         def do_data_table(self):
             self.ex('INSERT INTO {form_table} (category_id, label) VALUES (%s, %s) RETURNING (id)',
                     vars=[self.categories_mapping.get(self.formdef.schema.category_id),
                           self.formdef.schema.name])
             self.formdef_sql_id = self.cur.fetchone()[0]
             columns = OrderedDict()
             columns['status_id'] = {'sql_col_name': 'status_id', 'sql_col_def': 'smallint REFERENCES {status_table} (id)'}
-...
                 if channel == 'web' and data.submission.backoffice:
                     channel = 'backoffice'
                 row = {
                     'formdef_id': self.formdef_sql_id,
                     'formdef_id': self.formdefs_mapping[self.formdef.slug],
                     'receipt_time': data.receipt_time,
                     'hour_id': data.receipt_time.hour,
                     'channel_id': self.channel_to_id[channel],
+    -

Projet

Général

Profil

Produits Entr'ouvert » OLAP / Business Intelligence pour Publik

0001-feeder-preseve-categories-and-form-names-order-36930.patch