0001-wcs_api-use-WcsApi.batch_size-for-FormDatas-batch-si.patch
tests/test_wcs.py | ||
---|---|---|
19 | 19 |
import pytest |
20 | 20 |
import pathlib |
21 | 21 | |
22 |
import unittest.mock as mock |
|
23 | ||
22 | 24 |
import requests |
23 | 25 |
import httmock |
24 | 26 | |
... | ... | |
35 | 37 |
c.execute('CREATE TABLE olap_temp.coin()') |
36 | 38 |
c.execute('COMMIT') |
37 | 39 | |
38 |
olap_cmd() |
|
40 |
class MockSession(requests.Session): |
|
41 |
mocks = [] |
|
42 | ||
43 |
def __init__(self, *args, **kwargs): |
|
44 |
super().__init__(*args, **kwargs) |
|
45 |
self.get = mock.MagicMock(wraps=self.get) |
|
46 |
self.mocks.append(self.get) |
|
47 | ||
48 |
with mock.patch('requests.Session', MockSession) as Session: |
|
49 |
olap_cmd() |
|
50 |
call_args_list = MockSession.mocks[-1].call_args_list |
|
51 |
url_with_limits = [call_args[0][0] for call_args in MockSession.mocks[-1].call_args_list |
|
52 |
if 'limit' in call_args[0][0]] |
|
53 |
assert url_with_limits, call_args_list |
|
54 |
for url_with_limit in url_with_limits: |
|
55 |
assert '&limit=500&' in url_with_limit |
|
56 | ||
39 | 57 | |
40 | 58 |
expected_schema = [ |
41 | 59 |
('agent', 'id'), |
wcs_olap/wcs_api.py | ||
---|---|---|
218 | 218 | |
219 | 219 | |
220 | 220 |
class FormDatas(object): |
221 |
def __init__(self, wcs_api, formdef, full=False, anonymize=False, batch=1000):
|
|
221 |
def __init__(self, wcs_api, formdef, full=False, anonymize=False): |
|
222 | 222 |
self.wcs_api = wcs_api |
223 | 223 |
self.formdef = formdef |
224 | 224 |
self._full = full |
225 | 225 |
self.anonymize = anonymize |
226 |
self.batch = batch
|
|
226 |
self.batch_size = wcs_api.batch_size
|
|
227 | 227 | |
228 | 228 |
def __getitem__(self, slice_or_id): |
229 | 229 |
# get batch of forms |
... | ... | |
272 | 272 | |
273 | 273 |
def batched(self, batch): |
274 | 274 |
forms = copy.copy(self) |
275 |
forms.batch = batch |
|
275 |
forms.batch_size = batch
|
|
276 | 276 |
return forms |
277 | 277 | |
278 | 278 |
def __iter__(self): |
279 | 279 |
start = 0 |
280 | 280 |
while True: |
281 | 281 |
empty = True |
282 |
for formdef in self[start:start + self.batch]: |
|
282 |
for formdef in self[start:start + self.batch_size]:
|
|
283 | 283 |
empty = False |
284 | 284 |
yield formdef |
285 | 285 |
if empty: |
286 | 286 |
break |
287 |
start += self.batch |
|
287 |
start += self.batch_size
|
|
288 | 288 | |
289 | 289 |
def __len__(self): |
290 | 290 |
return len(list((o for o in self))) |
... | ... | |
533 | 533 |
def __init__(self, url, email=None, name_id=None, batch_size=1000, |
534 | 534 |
session=None, logger=None, orig=None, key=None, verify=True): |
535 | 535 |
self.url = url |
536 |
self.batch_size = batch_size |
|
537 | 536 |
self.email = email |
538 | 537 |
self.name_id = name_id |
539 | 538 |
self.requests = session or requests.Session() |
... | ... | |
541 | 540 |
self.orig = orig |
542 | 541 |
self.key = key |
543 | 542 |
self.verify = verify |
543 |
self.batch_size = batch_size |
|
544 | 544 | |
545 | 545 |
def _build_url(self, url_parts): |
546 | 546 |
url = self.url |
547 |
- |