From 9ef993108b945a41d2f9d430c6820c6c90cae6b0 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 4 May 2020 14:04:32 +0200 Subject: [PATCH 3/3] misc: add migration to ensure jsonb type (#42312) --- .../migrations/0004_text_to_jsonb.py | 18 +++++++ .../migrations/0018_text_to_jsonb.py | 18 +++++++ .../migrations/0012_text_to_jsonb.py | 21 ++++++++ .../migrations/0018_text_to_jsonb.py | 19 +++++++ .../gesbac/migrations/0004_text_to_jsonb.py | 19 +++++++ .../migrations/0008_text_to_jsonb.py | 18 +++++++ .../migrations/0003_text_to_jsonb.py | 18 +++++++ .../opengis/migrations/0011_text_to_jsonb.py | 18 +++++++ .../pastell/migrations/0008_text_to_jsonb.py | 18 +++++++ .../migrations/0002_text_to_jsonb.py | 18 +++++++ .../sp_fr/migrations/0003_text_to_jsonb.py | 18 +++++++ .../base/migrations/0019_text_to_jsonb.py | 20 +++++++ .../migrations/0006_text_to_jsonb.py | 18 +++++++ .../migrations/0005_text_to_jsonb.py | 18 +++++++ .../migrations/0006_text_to_jsonb.py | 18 +++++++ .../migrations/0009_text_to_jsonb.py | 18 +++++++ passerelle/utils/db.py | 53 +++++++++++++++++++ tests/test_misc.py | 41 ++++++++++++++ 18 files changed, 389 insertions(+) create mode 100644 passerelle/apps/atos_genesys/migrations/0004_text_to_jsonb.py create mode 100644 passerelle/apps/base_adresse/migrations/0018_text_to_jsonb.py create mode 100644 passerelle/apps/cartads_cs/migrations/0012_text_to_jsonb.py create mode 100644 passerelle/apps/csvdatasource/migrations/0018_text_to_jsonb.py create mode 100644 passerelle/apps/gesbac/migrations/0004_text_to_jsonb.py create mode 100644 passerelle/apps/jsondatastore/migrations/0008_text_to_jsonb.py create mode 100644 passerelle/apps/mdel_ddpacs/migrations/0003_text_to_jsonb.py create mode 100644 passerelle/apps/opengis/migrations/0011_text_to_jsonb.py create mode 100644 passerelle/apps/pastell/migrations/0008_text_to_jsonb.py create mode 100644 passerelle/apps/phonecalls/migrations/0002_text_to_jsonb.py create mode 100644 passerelle/apps/sp_fr/migrations/0003_text_to_jsonb.py create mode 100644 passerelle/base/migrations/0019_text_to_jsonb.py create mode 100644 passerelle/contrib/fake_family/migrations/0006_text_to_jsonb.py create mode 100644 passerelle/contrib/meyzieu_newsletters/migrations/0005_text_to_jsonb.py create mode 100644 passerelle/contrib/planitech/migrations/0006_text_to_jsonb.py create mode 100644 passerelle/contrib/teamnet_axel/migrations/0009_text_to_jsonb.py create mode 100644 passerelle/utils/db.py diff --git a/passerelle/apps/atos_genesys/migrations/0004_text_to_jsonb.py b/passerelle/apps/atos_genesys/migrations/0004_text_to_jsonb.py new file mode 100644 index 00000000..b8aff751 --- /dev/null +++ b/passerelle/apps/atos_genesys/migrations/0004_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('atos_genesys', '0003_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='Link', field_name='extra'), + ] diff --git a/passerelle/apps/base_adresse/migrations/0018_text_to_jsonb.py b/passerelle/apps/base_adresse/migrations/0018_text_to_jsonb.py new file mode 100644 index 00000000..c043c8f9 --- /dev/null +++ b/passerelle/apps/base_adresse/migrations/0018_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('base_adresse', '0017_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='AddressCacheModel', field_name='data'), + ] diff --git a/passerelle/apps/cartads_cs/migrations/0012_text_to_jsonb.py b/passerelle/apps/cartads_cs/migrations/0012_text_to_jsonb.py new file mode 100644 index 00000000..449e5947 --- /dev/null +++ b/passerelle/apps/cartads_cs/migrations/0012_text_to_jsonb.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('cartads_cs', '0011_cartadsdossier_cartads_cache_infos'), + ] + + operations = [ + EnsureJsonbType(model_name='CartaDSDataCache', field_name='data_parameters'), + EnsureJsonbType(model_name='CartaDSDataCache', field_name='data_values'), + EnsureJsonbType(model_name='cartadsdossier', field_name='cartads_steps_cache'), + EnsureJsonbType(model_name='cartadsdossier', field_name='cartads_cache_infos'), + ] diff --git a/passerelle/apps/csvdatasource/migrations/0018_text_to_jsonb.py b/passerelle/apps/csvdatasource/migrations/0018_text_to_jsonb.py new file mode 100644 index 00000000..ea2f08f0 --- /dev/null +++ b/passerelle/apps/csvdatasource/migrations/0018_text_to_jsonb.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('csvdatasource', '0017_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='csvdatasource', field_name='_dialect_options'), + EnsureJsonbType(model_name='TableRow', field_name='data'), + ] diff --git a/passerelle/apps/gesbac/migrations/0004_text_to_jsonb.py b/passerelle/apps/gesbac/migrations/0004_text_to_jsonb.py new file mode 100644 index 00000000..aba86641 --- /dev/null +++ b/passerelle/apps/gesbac/migrations/0004_text_to_jsonb.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('gesbac', '0003_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='Form', field_name='demand_data'), + EnsureJsonbType(model_name='Form', field_name='card_data'), + ] diff --git a/passerelle/apps/jsondatastore/migrations/0008_text_to_jsonb.py b/passerelle/apps/jsondatastore/migrations/0008_text_to_jsonb.py new file mode 100644 index 00000000..ac9de4e8 --- /dev/null +++ b/passerelle/apps/jsondatastore/migrations/0008_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('jsondatastore', '0007_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='JsonData', field_name='content'), + ] diff --git a/passerelle/apps/mdel_ddpacs/migrations/0003_text_to_jsonb.py b/passerelle/apps/mdel_ddpacs/migrations/0003_text_to_jsonb.py new file mode 100644 index 00000000..c74345dc --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/migrations/0003_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('mdel_ddpacs', '0002_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='Demand', field_name='data'), + ] diff --git a/passerelle/apps/opengis/migrations/0011_text_to_jsonb.py b/passerelle/apps/opengis/migrations/0011_text_to_jsonb.py new file mode 100644 index 00000000..2e71226d --- /dev/null +++ b/passerelle/apps/opengis/migrations/0011_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('opengis', '0010_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='FeatureCache', field_name='data'), + ] diff --git a/passerelle/apps/pastell/migrations/0008_text_to_jsonb.py b/passerelle/apps/pastell/migrations/0008_text_to_jsonb.py new file mode 100644 index 00000000..d9fbe2eb --- /dev/null +++ b/passerelle/apps/pastell/migrations/0008_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('pastell', '0007_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='Pastell', field_name='document_fields'), + ] diff --git a/passerelle/apps/phonecalls/migrations/0002_text_to_jsonb.py b/passerelle/apps/phonecalls/migrations/0002_text_to_jsonb.py new file mode 100644 index 00000000..14d5b227 --- /dev/null +++ b/passerelle/apps/phonecalls/migrations/0002_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('phonecalls', '0001_initial'), + ] + + operations = [ + EnsureJsonbType(model_name='Call', field_name='details'), + ] diff --git a/passerelle/apps/sp_fr/migrations/0003_text_to_jsonb.py b/passerelle/apps/sp_fr/migrations/0003_text_to_jsonb.py new file mode 100644 index 00000000..cc62e0a5 --- /dev/null +++ b/passerelle/apps/sp_fr/migrations/0003_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('sp_fr', '0002_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='Mapping', field_name='rules'), + ] diff --git a/passerelle/base/migrations/0019_text_to_jsonb.py b/passerelle/base/migrations/0019_text_to_jsonb.py new file mode 100644 index 00000000..03d519f1 --- /dev/null +++ b/passerelle/base/migrations/0019_text_to_jsonb.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0018_smslog'), + ] + + operations = [ + EnsureJsonbType(model_name='ResourceLog', field_name='extra'), + EnsureJsonbType(model_name='Job', field_name='parameters'), + EnsureJsonbType(model_name='Job', field_name='status_details'), + ] diff --git a/passerelle/contrib/fake_family/migrations/0006_text_to_jsonb.py b/passerelle/contrib/fake_family/migrations/0006_text_to_jsonb.py new file mode 100644 index 00000000..eefbe960 --- /dev/null +++ b/passerelle/contrib/fake_family/migrations/0006_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('fake_family', '0005_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='FakeFamily', field_name='jsondatabase'), + ] diff --git a/passerelle/contrib/meyzieu_newsletters/migrations/0005_text_to_jsonb.py b/passerelle/contrib/meyzieu_newsletters/migrations/0005_text_to_jsonb.py new file mode 100644 index 00000000..819644d2 --- /dev/null +++ b/passerelle/contrib/meyzieu_newsletters/migrations/0005_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('meyzieu_newsletters', '0004_remove_meyzieunewsletters_log_level'), + ] + + operations = [ + EnsureJsonbType(model_name='MeyzieuNewsletters', field_name='transport_titles_mapping'), + ] diff --git a/passerelle/contrib/planitech/migrations/0006_text_to_jsonb.py b/passerelle/contrib/planitech/migrations/0006_text_to_jsonb.py new file mode 100644 index 00000000..04533826 --- /dev/null +++ b/passerelle/contrib/planitech/migrations/0006_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('planitech', '0005_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='planitechconnector', field_name='custom_fields'), + ] diff --git a/passerelle/contrib/teamnet_axel/migrations/0009_text_to_jsonb.py b/passerelle/contrib/teamnet_axel/migrations/0009_text_to_jsonb.py new file mode 100644 index 00000000..2cdb51cc --- /dev/null +++ b/passerelle/contrib/teamnet_axel/migrations/0009_text_to_jsonb.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-05-04 12:06 +from __future__ import unicode_literals + +from django.db import migrations + +from passerelle.utils.db import EnsureJsonbType + + +class Migration(migrations.Migration): + + dependencies = [ + ('teamnet_axel', '0008_auto_20200504_1402'), + ] + + operations = [ + EnsureJsonbType(model_name='TeamnetAxel', field_name='billing_regies'), + ] diff --git a/passerelle/utils/db.py b/passerelle/utils/db.py new file mode 100644 index 00000000..4d956934 --- /dev/null +++ b/passerelle/utils/db.py @@ -0,0 +1,53 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2020 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.db.migrations.operations.base import Operation + + +class EnsureJsonbType(Operation): + + reversible = True + + def __init__(self, model_name, field_name): + self.model_name = model_name + self.field_name = field_name + + def state_forwards(self, app_label, state): + pass + + def database_forwards(self, app_label, schema_editor, from_state, to_state): + model = from_state.apps.get_model(app_label, self.model_name) + table_name = model._meta.db_table + field = model._meta.get_field(self.field_name) + _, column_name = field.get_attname_column() + with schema_editor.connection.cursor() as cursor: + cursor.execute( + 'SELECT data_type FROM information_schema.columns WHERE table_name = %s AND ' + 'column_name = %s;', (table_name, column_name) + ) + current_type = cursor.fetchone()[0].lower() + if current_type != 'jsonb': + assert current_type in ('json', 'text') + cursor.execute( + 'ALTER TABLE {table} ALTER COLUMN {col} TYPE jsonb USING {col}::jsonb;' + .format(table=table_name, col=column_name) + ) + + def database_backwards(self, app_label, schema_editor, from_state, to_state): + pass + + def describe(self): + return "Migrate to postgres jsonb type" diff --git a/tests/test_misc.py b/tests/test_misc.py index 1d6bcd09..ec14ff0d 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -2,7 +2,11 @@ import datetime import pytest from mock import patch +from django.core.files import File +from django.db import connection +from django.db.migrations.executor import MigrationExecutor from django.utils import timezone +from django.utils.six import StringIO from passerelle.base.models import ResourceLog from passerelle.apps.opengis.models import OpenGIS @@ -88,3 +92,40 @@ def test_trace_emails(app, settings, dummy_csv_datasource, email_handler, mailou assert len(mailoutbox) == 2 assert mailoutbox[0].to == ['admin@example.net'] assert mailoutbox[1].to == ['john.doe@example.net'] + + +def test_jsonb_migration(transactional_db): + app = 'csvdatasource' + + migrate_from = [(app, '0017_auto_20200504_1402')] + migrate_to = [(app, '0018_text_to_jsonb')] + executor = MigrationExecutor(connection) + old_apps = executor.loader.project_state(migrate_from).apps + # state of the db is not important + executor.migrate(migrate_from, fake=True) + + data = {'data': {'test': 1}} + CsvDataSource = old_apps.get_model(app, 'CsvDataSource') + connector = CsvDataSource.objects.create(csv_file=File(StringIO(''), 't.csv'), + _dialect_options=data) + pk = connector.pk + + field = CsvDataSource._meta.get_field('_dialect_options') + with connection.cursor() as cursor: + cursor.execute( + 'ALTER TABLE {table} ALTER COLUMN {col} TYPE text USING {col}::text;' + .format(table=CsvDataSource._meta.db_table, col=field.get_attname_column()[1]) + ) + connector = CsvDataSource.objects.get(pk=pk) + # db is in a broken state + assert connector._dialect_options != data + + # ensure migration fixes it + executor = MigrationExecutor(connection) + executor.migrate(migrate_to) + executor.loader.build_graph() + + apps = executor.loader.project_state(migrate_to).apps + CsvDataSource = apps.get_model(app, 'CsvDataSource') + connector = CsvDataSource.objects.get(pk=pk) + assert connector._dialect_options == data -- 2.20.1