Produits Entr'ouvert » OLAP / Business Intelligence pour Publik

Build-Depends: python3-setuptools, python3-all, debhelper (>= 9), dh-python

Standards-Version: 3.9.6

Homepage: http://dev.entrouvert.org/projects/wcs-olap/

Depends: ${python3:Depends}

isodate python3-isodate

psycopg2 python3-psycopg2

cached_property python3-cached-property

export PYBUILD_NAME=wcs-olap

	dh $@ --with python3 --buildsystem=pybuild

        with open('VERSION', 'w') as fd:

            fd.write(version)

       tag exists, take 0.0- and add the length of the commit log.

        p = subprocess.Popen(

            ['git', 'describe', '--dirty=.dirty', '--match=v*'],

            stdout=subprocess.PIPE, stderr=subprocess.PIPE)

            result = result.decode('ascii').strip()[1:]  # strip spaces/newlines and initial v

            if '-' in result:  # not a tagged version

                real_number, commit_count, commit_hash = result.split('-', 2)

                version = '%s.post%s+%s' % (real_number, commit_count, commit_hash)

            else:

                version = result

            return version

            return '0.0.post%s' % len(subprocess.check_output(['git', 'rev-list', 'HEAD']).splitlines())

    return '0.0'

      install_requires=[

          'requests',

          'psycopg2',

          'isodate',

          'six',

          'cached-property'

      ],

import pathlib

import httmock

            (pathlib.Path(__file__).parent / 'olap.model').open() as fd2:

        json_schema = json.load(fd)

        expected_json_schema = json.load(fd2)

        expected_json_schema['pg_dsn'] = postgres_db.dsn

        assert json_schema == expected_json_schema

    @httmock.urlmatch()

    def requests_raise(url, request):

        raise requests.RequestException('wat!')

    with httmock.HTTMock(requests_raise):

    @httmock.urlmatch()

    def return_401(url, request):

        return {'status_code': 401, 'content': {"err": 1, "err_desc": "invalid signature"}}

    with httmock.HTTMock(return_401):

    @httmock.urlmatch()

    def return_invalid_json(url, request):

        return 'x'

    with httmock.HTTMock(return_invalid_json):

envlist = py3-coverage

	quixote>=3

	httmock

	#./get_wcs.sh

[pytest]

junit_family=xunit2

import configparser

import locale

import os

import sys

from . import wcs_api, feeder

    config = configparser.ConfigParser()

    urls = [url for url in config.sections() if url.startswith('http://')

            or url.startswith('https://')]

            print('no url found')

        except KeyError as e:

                api = wcs_api.WcsApi(url=url, orig=orig, key=key,

                                     batch_size=batch_size,

                                     verify=(defaults.get('verify', 'True') == 'True'))

                olap_feeder = feeder.WcsOlapFeeder(

                    api=api, schema=schema, pg_dsn=pg_dsn, logger=logger,

                    config=defaults, do_feed=feed, fake=fake, slugs=slugs)

                olap_feeder.feed()

            except Exception:

from collections import OrderedDict

from .utils import Whatever

    def __init__(self, api, pg_dsn, schema, logger=None, config=None, do_feed=True, fake=False, slugs=None):

        self.slugs = slugs

        return [formdef for formdef in self.api.formdefs if not self.slugs or formdef.slug in self.slugs]

                return dict((k, helper(v)) for k, v in o.items())

            'category', enumerate(c.title for c in self.categories), comment='catégorie')

        self.categories_mapping = dict((c.slug, tmp_cat_map[c.title]) for c in self.categories)

        for at, comment in self.comments.items():

        for function, name in self.formdef.schema.workflow.functions.items():

        for data in self.formdef.formdatas.anonymized.full:

            for function, name in self.formdef.schema.workflow.functions.items():

        for function, name in self.formdef.schema.workflow.functions.items():

import urllib.parse as urlparse

# from http://repos.entrouvert.org/portail-citoyen.git/tree/portail_citoyen/apps/data_source_plugin/signature.py

        nonce = hex(random.getrandbits(128))[2:]

    new_query += urlparse.urlencode((

    new_query += '&signature=' + urlparse.quote(signature)

    if isinstance(key, str):

    if isinstance(s, str):

        s = s.encode('utf-8')

    if not ('signature' in parsed and 'algo' in parsed

            and 'timestamp' in parsed and 'nonce' in parsed):

        return False

    unsigned_query, signature_content = query.split('&signature=', 1)

    if '&' in signature_content:

        return False  # signature must be the last parameter

        res |= a ^ b

if __name__ == '__main__':

    key = '12345'

    signed_query = sign_query('NameId=_12345&orig=montpellier', key)

    assert check_query(signed_query, key, timedelta=0) is False

    assert check_query(signed_query, key) is True

# wcs_olap

# Copyright (C) 2020 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import collections

import base64

import copy

import logging

import datetime

import contextlib

import json

import urllib.parse as urlparse

class WcsApiError(Exception):

    pass

class JSONFile(object):

    def __init__(self, d):

        self.d = d

    @property

    def filename(self):

        return self.d.get('filename', '')

    @property

    def content_type(self):

        return self.d.get('content_type', 'application/octet-stream')

    @property

    def content(self):

        return base64.b64decode(self.d['content'])

def to_dict(o):

    if hasattr(o, 'to_dict'):

        return o.to_dict()

    elif isinstance(o, dict):

        return {k: to_dict(v) for k, v in o.items()}

    elif isinstance(o, (list, tuple)):

        return [to_dict(v) for v in o]

    else:

        return o

        self._wcs_api = wcs_api

    def to_dict(self):

        d = collections.OrderedDict()

        for key, value in self.__dict__.items():

            if key[0] == '_':

                continue

            d[key] = to_dict(value)

        return d

    submission = None

    workflow = None

    roles = None

    with_files = False

    def __init__(self, wcs_api, forms, **kwargs):

        self.forms = forms

        if self.submission:

            self.submission = BaseObject(wcs_api, **self.submission)

        if self.workflow:

            self.workflow = FormDataWorkflow(wcs_api, **self.workflow)

        for function in self.roles or []:

        if 'roles' in self.__dict__:

            del self.roles

    def __str__(self):

        return '{self.formdef} - {self.id}'.format(self=self)

    @property

    def full(self):

        if self.with_files:

            return self

        if not hasattr(self, '_full'):

            self._full = self.forms[self.id]

        return self._full

    @property

    def anonymized(self):

        return self.forms.anonymized[self.id]

    def __getitem__(self, key):

        value = self.full.fields.get(key)

        # unserialize files

        if isinstance(value, dict) and 'content' in value:

            return JSONFile(value)

        return value

        assert not hasattr(self.statuses[0], 'startpoint'), 'startpoint is exported by w.c.s. FIXME'

        for status in self.statuses:

            status.startpoint = False

        self.statuses[0].startpoint = True

class FormDatas(object):

    def __init__(self, wcs_api, formdef, full=False, anonymize=False, batch=1000):

        self.wcs_api = wcs_api

        self.formdef = formdef

        self._full = full

        self.anonymize = anonymize

        self.batch = batch

    def __getitem__(self, slice_or_id):

        # get batch of forms

        if isinstance(slice_or_id, slice):

            def helper():

                if slice_or_id.stop <= slice_or_id.start or slice_or_id.step:

                    raise ValueError('invalid slice %s' % slice_or_id)

                offset = slice_or_id.start

                limit = slice_or_id.stop - slice_or_id.start

                url_parts = ['api/forms/{self.formdef.slug}/list'.format(self=self)]

                query = {}

                query['full'] = 'on' if self._full else 'off'

                if offset:

                    query['offset'] = str(offset)

                if limit:

                    query['limit'] = str(limit)

                if self.anonymize:

                    query['anonymise'] = 'on'

                url_parts.append('?%s' % urlparse.urlencode(query))

                for d in self.wcs_api.get_json(*url_parts):

                    # w.c.s. had a bug where some formdata lost their draft status, skip them

                    if not d.get('receipt_time'):

                        continue

                    yield FormData(wcs_api=self.wcs_api, forms=self, formdef=self.formdef, **d)

            return helper()

        # or get one form

        else:

            url_parts = ['api/forms/{formdef.slug}/{id}/'.format(formdef=self.formdef, id=slice_or_id)]

            if self.anonymize:

                url_parts.append('?anonymise=true')

            d = self.wcs_api.get_json(*url_parts)

            return FormData(wcs_api=self.wcs_api, forms=self, formdef=self.formdef, with_files=True, **d)

    @property

    def full(self):

        forms = copy.copy(self)

        forms._full = True

        return forms

    @property

    def anonymized(self):

        forms = copy.copy(self)

        forms.anonymize = True

        return forms

    def batched(self, batch):

        forms = copy.copy(self)

        forms.batch = batch

        return forms

    def __iter__(self):

        start = 0

        while True:

            empty = True

            for formdef in self[start:start + self.batch]:

                empty = False

                yield formdef

            if empty:

                break

            start += self.batch

    def __len__(self):

        return len(list((o for o in self)))

class CancelSubmitError(Exception):

    pass

class FormDefSubmit(object):

    formdef = None

    data = None

    user_email = None

    user_name_id = None

    backoffice_submission = False

    submission_channel = None

    submission_context = None

    draft = False

    def __init__(self, wcs_api, formdef, **kwargs):

        self.wcs_api = wcs_api

        self.formdef = formdef

        self.data = {}

        self.__dict__.update(kwargs)

    def payload(self):

        d = {

            'data': self.data.copy(),

        }

        if self.draft:

            d.setdefault('meta', {})['draft'] = True

        if self.backoffice_submission:

            d.setdefault('meta', {})['backoffice-submission'] = True

        if self.submission_context:

            d['context'] = self.submission_context

        if self.submission_channel:

            d.setdefault('context', {})['channel'] = self.submission_channel

        if self.user_email:

            d.setdefault('user', {})['email'] = self.user_email

        if self.user_name_id:

            d.setdefault('user', {})['NameID'] = self.user_name_id

        return d

    def set(self, field, value, **kwargs):

        if isinstance(field, Field):

            varname = field.varname

            if not varname:

                raise ValueError('field has no varname, submit is impossible')

        else:

            varname = field

            try:

                field = [f for f in self.formdef.schema.fields if f.varname == varname][0]

            except IndexError:

                raise ValueError('no field for varname %s' % varname)

        if value is None or value == {} or value == []:

            self.data.pop(varname, None)

        elif hasattr(self, '_set_type_%s' % field.type):

            getattr(self, '_set_type_%s' % field.type)(

                varname=varname,

                field=field,

                value=value, **kwargs)

        else:

            self.data[varname] = value

    def _set_type_item(self, varname, field, value, **kwargs):

        if isinstance(value, dict):

            if not set(value).issuperset(set(['id', 'text'])):

                raise ValueError('item field value must have id and text value')

        # clean previous values

        self.data.pop(varname, None)

        self.data.pop(varname + '_raw', None)

        self.data.pop(varname + '_structured', None)

        if isinstance(value, dict):

            # structured & display values

            self.data[varname + '_raw'] = value['id']

            self.data[varname] = value['text']

            if len(value) > 2:

                self.data[varname + '_structured'] = value

        else:

            # raw id in varname

            self.data[varname] = value

    def _set_type_items(self, varname, field, value, **kwargs):

        if not isinstance(value, list):

            raise TypeError('%s is an ItemsField it needs a list as value' % varname)

        has_dict = False

        for choice in value:

            if isinstance(value, dict):

                if not set(value).issuperset(set(['id', 'text'])):

                    raise ValueError('items field values must have id and text value')

                has_dict = True

        if has_dict:

            if not all(isinstance(choice, dict) for choice in value):

                raise ValueError('ItemsField value must be all structured or none')

        # clean previous values

        self.data.pop(varname, None)

        self.data.pop(varname + '_raw', None)

        self.data.pop(varname + '_structured', None)

        if has_dict:

            raw = self.data[varname + '_raw'] = []

            display = self.data[varname] = []

            structured = self.data[varname + '_structured'] = []

            for choice in value:

                raw.append(choice['id'])

                display.append(choice['text'])

                structured.append(choice)

        else:

            self.data[varname] = value[:]

    def _set_type_file(self, varname, field, value, **kwargs):

        filename = kwargs.get('filename')

        content_type = kwargs.get('content_type', 'application/octet-stream')

        if hasattr(value, 'read'):

            content = base64.b64encode(value.read()).decode('ascii')

        elif isinstance(value, bytes):

            content = base64.b64encode(value).decode('ascii')

        elif isinstance(value, dict):

            if not set(value).issuperset(set(['filename', 'content'])):

                raise ValueError('file field needs a dict value with filename and content')

            content = value['content']

            filename = value['filename']

            content_type = value.get('content_type', content_type)

        if not filename:

            raise ValueError('missing filename')

        self.data[varname] = {

            'filename': filename,

            'content': content,

            'content_type': content_type,

        }

    def _set_type_date(self, varname, field, value):

        if isinstance(value, str):

            value = datetime.datetime.strptime(value, '%Y-%m-%d').date()

        if isinstance(value, datetime.datetime):

            value = value.date()

        if isinstance(value, datetime.date):

            value = value.strftime('%Y-%m-%d')

        self.data[varname] = value

    def _set_type_map(self, varname, field, value):

        if not isinstance(value, dict):

            raise TypeError('value must be a dict for a map field')

        if set(value) != set(['lat', 'lon']):

            raise ValueError('map field expect keys lat and lon')

        self.data[varname] = value

    def _set_type_bool(self, varname, field, value):

        if isinstance(value, str):

            value = value.lower().strip() in ['yes', 'true', 'on']

        if not isinstance(value, bool):

            raise TypeError('value must be a boolean or a string true, yes, on, false, no, off')

        self.data[varname] = value

    def cancel(self):

        raise CancelSubmitError

        self._wcs_api = wcs_api

    def __str__(self):

    def formdatas(self):

        return FormDatas(wcs_api=self._wcs_api, formdef=self)

        if not hasattr(self, '_schema'):

            d = self._wcs_api.get_json('api/formdefs/{self.slug}/schema'.format(self=self))

            self._schema = Schema(self._wcs_api, **d)

        return self._schema

    @contextlib.contextmanager

    def submit(self, **kwargs):

        submitter = FormDefSubmit(

            wcs_api=self._wcs_api,

            formdef=self,

            **kwargs)

        try:

            yield submitter

        except CancelSubmitError:

            return

        payload = submitter.payload()

        d = self._wcs_api.post_json(payload, 'api/formdefs/{self.slug}/submit'.format(self=self))

        if d['err'] != 0:

            raise WcsApiError('submited returned an error: %s' % d)

        submitter.result = BaseObject(self._wcs_api, **d['data'])

class WcsObjects(object):

    url = None

    object_class = None

    def __init__(self, wcs_api):

        self.wcs_api = wcs_api

    def __getitem__(self, slug):

        if isinstance(slug, self.object_class):

            slug = slug.slug

        for instance in self:

            if instance.slug == slug:

                return instance

        raise KeyError('no instance with slug %r' % slug)

    def __iter__(self):

        for d in self.wcs_api.get_json(self.url)['data']:

            yield self.object_class(wcs_api=self.wcs_api, **d)

    def __len__(self):

        return len(list((o for o in self)))

class Roles(WcsObjects):

    # Paths are not coherent :/

    url = 'api/roles'

    object_class = Role

class FormDefs(WcsObjects):

    url = 'api/formdefs/?include-count=on'

    object_class = FormDef

class Categories(WcsObjects):

    url = 'api/categories/'

    object_class = Category

    def __init__(self, url, email=None, name_id=None, batch_size=1000,

                 session=None, logger=None, orig=None, key=None, verify=True):

        self.batch_size = batch_size

        self.email = email

        self.name_id = name_id

        self.requests = session or requests.Session()

        self.logger = logger or logging.getLogger(__name__)

    def _build_url(self, url_parts):

        url = self.url

        for url_part in url_parts:

            url = urlparse.urljoin(url, url_part)

        return url

        url = self._build_url(url_parts)

        params = {}

        if self.email:

            params['email'] = self.email

        if self.name_id:

            params['NameID'] = self.name_id

        if self.orig:

            params['orig'] = self.orig

        query_string = urlparse.urlencode(params)

        complete_url = url + ('&' if '?' in url else '?') + query_string

        final_url = complete_url

        if self.key:

            final_url = signature.sign_url(final_url, self.key)

            response = self.requests.get(final_url, verify=self.verify)

            response.raise_for_status()

            content = getattr(getattr(e, 'response', None), 'content', None)

            raise WcsApiError('GET request failed', final_url, e, content)

        else:

            try:

                return response.json()

            except ValueError as e:

                raise WcsApiError('Invalid JSON content', final_url, e)

    def post_json(self, data, *url_parts):

        url = self._build_url(url_parts)

        params = {}

        if self.email:

            params['email'] = self.email

        if self.name_id:

            params['NameID'] = self.name_id

        if self.orig:

            params['orig'] = self.orig

        query_string = urlparse.urlencode(params)

        complete_url = url + ('&' if '?' in url else '?') + query_string

        final_url = complete_url

        if self.key:

            final_url = signature.sign_url(final_url, self.key)

        try:

            response = self.requests.post(

                final_url,

                data=json.dumps(data),

                headers={'content-type': 'application/json'},

                verify=self.verify)

            response.raise_for_status()

        except requests.RequestException as e:

            content = getattr(getattr(e, 'response', None), 'content', None)

            raise WcsApiError('POST request failed', final_url, e, content)

                return response.json()

                raise WcsApiError('Invalid JSON content', final_url, e)

        return Roles(self)

        return FormDefs(self)

        return Categories(self)