Projet

Général

Profil

0001-misc-keep-data-source-calls-in-request-cache-26612.patch

Frédéric Péters, 21 septembre 2018 13:05

Télécharger (7 ko)

Voir les différences:

Subject: [PATCH] misc: keep data source calls in request cache (#26612)

 tests/test_datasource.py   | 15 +++++++++++++++
 wcs/data_sources.py        |  9 ++++++++-
 wcs/qommon/http_request.py |  1 +
 3 files changed, 24 insertions(+), 1 deletion(-)
tests/test_datasource.py
117 117
            ('foo', 'Foo', 'foo', {'id': 'foo', 'text': 'Foo', 'value': '2017-01-01'})]
118 118

  
119 119
def test_json_datasource(http_requests):
120
    get_request().datasources_cache = {}
120 121
    datasource = {'type': 'json', 'value': ''}
121 122
    assert data_sources.get_items(datasource) == []
122 123

  
123 124
    # missing file
125
    get_request().datasources_cache = {}
124 126
    json_file_path = os.path.join(pub.app_dir, 'test.json')
125 127
    datasource = {'type': 'json', 'value': 'file://%s' % json_file_path}
126 128
    assert data_sources.get_items(datasource) == []
127 129

  
128 130
    # invalid json file
131
    get_request().datasources_cache = {}
129 132
    json_file = open(json_file_path, 'w')
130 133
    json_file.write(u'foobar'.encode('zlib_codec'))
131 134
    json_file.close()
132 135
    assert data_sources.get_items(datasource) == []
133 136

  
134 137
    # empty json file
138
    get_request().datasources_cache = {}
135 139
    json_file = open(json_file_path, 'w')
136 140
    json.dump({}, json_file)
137 141
    json_file.close()
138 142
    assert data_sources.get_items(datasource) == []
139 143

  
140 144
    # unrelated json file
145
    get_request().datasources_cache = {}
141 146
    json_file = open(json_file_path, 'w')
142 147
    json.dump('foobar', json_file)
143 148
    json_file.close()
144 149
    assert data_sources.get_items(datasource) == []
145 150

  
146 151
    # another unrelated json file
152
    get_request().datasources_cache = {}
147 153
    json_file = open(json_file_path, 'w')
148 154
    json.dump({'data': 'foobar'}, json_file)
149 155
    json_file.close()
150 156
    assert data_sources.get_items(datasource) == []
151 157

  
152 158
    # a good json file
159
    get_request().datasources_cache = {}
153 160
    json_file = open(json_file_path, 'w')
154 161
    json.dump({'data': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}, json_file)
155 162
    json_file.close()
......
160 167
            {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
161 168

  
162 169
    # a json file with additional keys
170
    get_request().datasources_cache = {}
163 171
    json_file = open(json_file_path, 'w')
164 172
    json.dump({'data': [{'id': '1', 'text': 'foo', 'more': 'xxx'},
165 173
                        {'id': '2', 'text': 'bar', 'more': 'yyy'}]}, json_file)
......
172 180
            {'id': '2', 'text': 'bar', 'more': 'yyy'}]
173 181

  
174 182
    # json specified with a variadic url
183
    get_request().datasources_cache = {}
175 184
    class JsonUrlPath(object):
176 185
        def get_substitution_variables(self):
177 186
            return {'json_url': 'file://%s' % json_file_path}
......
182 191
            ('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
183 192

  
184 193
    # same with django templated url
194
    get_request().datasources_cache = {}
185 195
    class JsonUrlPath(object):
186 196
        def get_substitution_variables(self):
187 197
            return {'json_url': 'file://%s' % json_file_path}
......
192 202
            ('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
193 203

  
194 204
    # json specified with a variadic url with an erroneous space
205
    get_request().datasources_cache = {}
195 206
    class JsonUrlPath(object):
196 207
        def get_substitution_variables(self):
197 208
            return {'json_url': 'file://%s' % json_file_path}
......
202 213
            ('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
203 214

  
204 215
    # same with django templated url
216
    get_request().datasources_cache = {}
205 217
    class JsonUrlPath(object):
206 218
        def get_substitution_variables(self):
207 219
            return {'json_url': 'file://%s' % json_file_path}
......
212 224
            ('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
213 225

  
214 226
    # a json file with integer as 'id'
227
    get_request().datasources_cache = {}
215 228
    json_file = open(json_file_path, 'w')
216 229
    json.dump({'data': [{'id': 1, 'text': 'foo'}, {'id': 2, 'text': 'bar'}]}, json_file)
217 230
    json_file.close()
......
222 235
            {'id': 1, 'text': 'foo'}, {'id': 2, 'text': 'bar'}]
223 236

  
224 237
    # a json file with empty or no text values
238
    get_request().datasources_cache = {}
225 239
    json_file = open(json_file_path, 'w')
226 240
    json.dump({'data': [{'id': '1', 'text': ''}, {'id': '2'}]}, json_file)
227 241
    json_file.close()
......
233 247
            {'id': '2', 'text': '2'}]
234 248

  
235 249
    # a json file with empty or no id
250
    get_request().datasources_cache = {}
236 251
    json_file = open(json_file_path, 'w')
237 252
    json.dump({'data': [{'id': '', 'text': 'foo'}, {'text': 'bar'}, {'id': None}]}, json_file)
238 253
    json_file.close()
wcs/data_sources.py
19 19
import urlparse
20 20
import xml.etree.ElementTree as ET
21 21

  
22
from quixote import get_publisher
22
from quixote import get_publisher, get_request
23 23
from quixote.html import TemplateIO
24 24

  
25 25
from qommon import _
......
156 156
        if Template.is_template_string(url):
157 157
            vars = get_publisher().substitutions.get_context_variables()
158 158
            url = get_variadic_url(url, vars)
159

  
160
        request = get_request()
161
        if hasattr(request, 'datasources_cache') and url in request.datasources_cache:
162
            return request.datasources_cache[url]
163

  
159 164
        try:
160 165
            signature_key, orig = get_secret_and_orig(url)
161 166
        except MissingSecret:
......
181 186
                if 'text' not in item:
182 187
                    item['text'] = item['id']
183 188
                items.append(item)
189
            if hasattr(request, 'datasources_cache'):
190
                request.datasources_cache[url] = items
184 191
            return items
185 192
        except qommon.misc.ConnectionError as e:
186 193
            get_logger().warn('Error loading JSON data source (%s)' % str(e))
wcs/qommon/http_request.py
37 37
        self.is_json_marker = None
38 38
        self.ignore_session = False
39 39
        self.wscalls_cache = {}
40
        self.datasources_cache = {}
40 41
        # keep a copy of environment to make sure it's not reused along
41 42
        # uwsgi/gunicorn processes.
42 43
        self.environ = copy.copy(self.environ)
43
-