0001-general-use-uwsgi-spooler-to-run-afterjobs-48407.patch
debian/control | ||
---|---|---|
21 | 21 |
python3-psycopg2, |
22 | 22 |
python3-pyproj, |
23 | 23 |
python3-requests, |
24 |
python3-uwsgidecorators, |
|
24 | 25 |
python3-vobject, |
25 | 26 |
python3-xstatic-leaflet, |
26 | 27 |
python3-xstatic-leaflet-gesturehandling, |
debian/debian_config.py | ||
---|---|---|
3 | 3 |
import os |
4 | 4 | |
5 | 5 |
PROJECT_NAME = 'wcs' |
6 |
WCS_MANAGE_COMMAND = '/usr/bin/wcs-manage' |
|
6 | 7 | |
7 | 8 |
# |
8 | 9 |
# hobotization |
debian/uwsgi.ini | ||
---|---|---|
12 | 12 |
chmod-socket = 666 |
13 | 13 |
vacuum = true |
14 | 14 | |
15 |
spooler-processes = 3 |
|
16 |
import = wcs.qommon.spooler |
|
17 | ||
15 | 18 |
master = true |
16 | 19 |
enable-threads = true |
17 | 20 |
harakiri = 120 |
debian/wcs.dirs | ||
---|---|---|
3 | 3 |
usr/lib/wcs |
4 | 4 |
var/lib/wcs |
5 | 5 |
var/lib/wcs/collectstatic |
6 |
var/lib/wcs/spooler |
|
6 | 7 |
var/log/wcs |
debian/wcs.init | ||
---|---|---|
44 | 44 |
DAEMON_ARGS=${DAEMON_ARGS:-"--pidfile=$PIDFILE |
45 | 45 |
--uid $USER --gid $GROUP |
46 | 46 |
--ini /etc/$NAME/uwsgi.ini |
47 |
--spooler /var/lib/wcs/spooler/ |
|
47 | 48 |
--daemonize /var/log/uwsgi.$NAME.log"} |
48 | 49 | |
49 | 50 |
# Load the VERBOSE setting and other rcS variables |
debian/wcs.postinst | ||
---|---|---|
35 | 35 |
chown $USER:$GROUP /var/log/$NAME |
36 | 36 |
chown $USER:$GROUP /var/lib/$NAME |
37 | 37 |
chown $USER:$GROUP /var/lib/$NAME/collectstatic |
38 |
chown $USER:$GROUP /var/lib/$NAME/spooler |
|
38 | 39 | |
39 | 40 |
# create a secret file |
40 | 41 |
SECRET_FILE=$CONFIG_DIR/secret |
debian/wcs.service | ||
---|---|---|
10 | 10 |
Group=%p |
11 | 11 |
ExecStartPre=/usr/bin/wcs-manage migrate |
12 | 12 |
ExecStartPre=/usr/bin/wcs-manage collectstatic |
13 |
ExecStart=/usr/bin/uwsgi --ini /etc/%p/uwsgi.ini |
|
13 |
ExecStartPre=/bin/mkdir -p /var/lib/wcs/spooler/%m/ |
|
14 |
ExecStart=/usr/bin/uwsgi --ini /etc/%p/uwsgi.ini --spooler /var/lib/wcs/spooler/%m/ |
|
14 | 15 |
ExecReload=/bin/kill -HUP $MAINPID |
15 | 16 |
KillSignal=SIGQUIT |
16 | 17 |
TimeoutStartSec=0 |
tests/test_ctl.py | ||
---|---|---|
12 | 12 |
from wcs.wf.jump import JumpWorkflowStatusItem |
13 | 13 |
from wcs.fields import StringField, EmailField |
14 | 14 |
import wcs.qommon.ctl |
15 |
from wcs.qommon.afterjobs import AfterJob |
|
15 | 16 |
from wcs.qommon.management.commands.collectstatic import Command as CmdCollectStatic |
16 | 17 |
from wcs.qommon.management.commands.migrate import Command as CmdMigrate |
17 | 18 |
from wcs.qommon.management.commands.migrate_schemas import Command as CmdMigrateSchemas |
... | ... | |
404 | 405 |
def test_shell(): |
405 | 406 |
with pytest.raises(CommandError): |
406 | 407 |
call_command('shell') # missing tenant name |
408 | ||
409 | ||
410 |
class TestAfterJob(AfterJob): |
|
411 |
def execute(self): |
|
412 |
pass |
|
413 | ||
414 |
def test_runjob(pub): |
|
415 |
with pytest.raises(CommandError): |
|
416 |
call_command('runjob') |
|
417 |
with pytest.raises(CommandError): |
|
418 |
call_command('runjob', '--domain=example.net', '--job-id=%s' % 'invalid') |
|
419 | ||
420 |
job = TestAfterJob(label='test') |
|
421 |
job.store() |
|
422 |
assert AfterJob.get(job.id).status == 'registered' |
|
423 |
call_command('runjob', '--domain=example.net', '--job-id=%s' % job.id) |
|
424 |
assert AfterJob.get(job.id).status == 'completed' |
tests/test_publisher.py | ||
---|---|---|
272 | 272 |
def test_clean_afterjobs(): |
273 | 273 |
pub = create_temporary_pub() |
274 | 274 | |
275 |
job = AfterJob(id='a')
|
|
276 |
job.status = 'completed' |
|
277 |
job.creation_time = time.time() - 3 * 3600 |
|
278 |
job.completion_time = time.time() - 3 * 3600 |
|
279 |
job.store() |
|
280 | ||
281 |
job = AfterJob(id='b')
|
|
282 |
job.status = 'completed' |
|
283 |
job.creation_time = time.time() |
|
284 |
job.completion_time = time.time() |
|
285 |
job.store() |
|
286 | ||
287 |
job = AfterJob(id='c')
|
|
288 |
job.status = 'running' |
|
289 |
job.creation_time = time.time() - 3 * 86400 |
|
290 |
job.store() |
|
275 |
job1 = AfterJob()
|
|
276 |
job1.status = 'completed'
|
|
277 |
job1.creation_time = time.time() - 3 * 3600
|
|
278 |
job1.completion_time = time.time() - 3 * 3600
|
|
279 |
job1.store()
|
|
280 | ||
281 |
job2 = AfterJob()
|
|
282 |
job2.status = 'completed'
|
|
283 |
job2.creation_time = time.time()
|
|
284 |
job2.completion_time = time.time()
|
|
285 |
job2.store()
|
|
286 | ||
287 |
job3 = AfterJob()
|
|
288 |
job3.status = 'running'
|
|
289 |
job3.creation_time = time.time() - 3 * 86400
|
|
290 |
job3.store()
|
|
291 | 291 | |
292 | 292 |
pub.clean_afterjobs() |
293 | 293 |
assert AfterJob.count() == 1 |
294 |
assert AfterJob.select()[0].id == 'b'
|
|
294 |
assert AfterJob.select()[0].id == job2.id
|
|
295 | 295 | |
296 | 296 | |
297 | 297 |
def test_clean_tempfiles(): |
wcs/ctl/management/commands/runjob.py | ||
---|---|---|
1 |
# w.c.s. - web application for online forms |
|
2 |
# Copyright (C) 2005-2020 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software; you can redistribute it and/or modify |
|
5 |
# it under the terms of the GNU General Public License as published by |
|
6 |
# the Free Software Foundation; either version 2 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU General Public License |
|
15 |
# along with this program; if not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
import sys |
|
18 | ||
19 |
from django.core.management import CommandError |
|
20 | ||
21 |
from wcs.qommon.http_response import AfterJob |
|
22 |
from . import TenantCommand |
|
23 | ||
24 | ||
25 |
class Command(TenantCommand): |
|
26 |
'''Run an afterjob (internal command)''' |
|
27 | ||
28 |
def add_arguments(self, parser): |
|
29 |
parser.add_argument('--domain', action='store', required=True) |
|
30 |
parser.add_argument('--job-id', action='store', required=True) |
|
31 | ||
32 |
def handle(self, *args, **options): |
|
33 |
domain = options.pop('domain') |
|
34 |
self.init_tenant_publisher(domain) |
|
35 |
try: |
|
36 |
job = AfterJob.get(options['job_id']) |
|
37 |
except KeyError: |
|
38 |
raise CommandError('missing job') |
|
39 |
job.run() |
wcs/qommon/afterjobs.py | ||
---|---|---|
14 | 14 |
# You should have received a copy of the GNU General Public License |
15 | 15 |
# along with this program; if not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 |
import sys |
|
18 |
import time |
|
19 |
import traceback |
|
20 |
import uuid |
|
21 | ||
22 |
from quixote import get_publisher, get_response |
|
17 | 23 |
from quixote.directory import Directory |
18 |
from quixote import get_response |
|
19 | 24 | |
20 |
from . import errors |
|
21 |
from .http_response import AfterJob |
|
22 |
from . import _ |
|
25 |
from . import _, N_, errors |
|
26 |
from .storage import StorableObject |
|
23 | 27 | |
24 | 28 | |
25 | 29 |
class AfterJobStatusDirectory(Directory): |
... | ... | |
34 | 38 |
if not job.completion_status: |
35 | 39 |
return job.status + '|' + _(job.status) |
36 | 40 |
return job.status + '|' + _(job.status) + ' ' + job.completion_status |
41 | ||
42 | ||
43 |
class AfterJob(StorableObject): |
|
44 |
_names = 'afterjobs' |
|
45 |
_reset_class = False |
|
46 | ||
47 |
label = None |
|
48 |
status = None |
|
49 |
creation_time = None |
|
50 |
completion_time = None |
|
51 |
completion_status = None |
|
52 | ||
53 |
execute = None |
|
54 | ||
55 |
def __init__(self, label=None, cmd=None, **kwargs): |
|
56 |
super().__init__(id=str(uuid.uuid4())) |
|
57 |
if label: |
|
58 |
self.label = label |
|
59 |
self.creation_time = time.time() |
|
60 |
self.job_cmd = cmd |
|
61 |
self.status = N_('registered') |
|
62 |
self.kwargs = kwargs |
|
63 | ||
64 |
def run(self, spool=False): |
|
65 |
if self.completion_time: |
|
66 |
return |
|
67 | ||
68 |
if spool and self.id and self.execute: |
|
69 |
from django.conf import settings |
|
70 |
if 'uwsgi' in sys.modules and settings.WCS_MANAGE_COMMAND: |
|
71 |
from .spooler import run_after_job |
|
72 |
self.store() |
|
73 |
run_after_job.spool(tenant_dir=get_publisher().app_dir, job_id=self.id) |
|
74 |
return |
|
75 | ||
76 |
self.status = N_('running') |
|
77 |
if self.id: |
|
78 |
self.store() |
|
79 |
try: |
|
80 |
if self.execute: |
|
81 |
self.execute() |
|
82 |
else: |
|
83 |
self.job_cmd(job=self) |
|
84 |
except Exception: |
|
85 |
get_publisher().notify_of_exception(sys.exc_info()) |
|
86 |
self.exception = traceback.format_exc() |
|
87 |
self.status = N_('failed') |
|
88 |
else: |
|
89 |
self.status = N_('completed') |
|
90 |
self.completion_time = time.time() |
|
91 |
if self.id: |
|
92 |
self.store() |
|
93 | ||
94 |
def __getstate__(self): |
|
95 |
if not isinstance(self.job_cmd, str): |
|
96 |
obj_dict = self.__dict__.copy() |
|
97 |
obj_dict['job_cmd'] = None |
|
98 |
return obj_dict |
|
99 |
return self.__dict__ |
wcs/qommon/http_response.py | ||
---|---|---|
21 | 21 | |
22 | 22 |
from django.utils.encoding import force_bytes |
23 | 23 |
from quixote import get_publisher |
24 |
from quixote.util import randbytes |
|
25 | 24 |
import quixote.http_response |
26 | 25 |
from quixote import get_publisher, get_request |
27 | 26 | |
28 | 27 |
from . import N_ |
29 |
from .storage import StorableObject |
|
30 | ||
31 | ||
32 |
class AfterJob(StorableObject): |
|
33 |
_names = 'afterjobs' |
|
34 | ||
35 |
label = None |
|
36 |
status = None |
|
37 |
creation_time = None |
|
38 |
completion_time = None |
|
39 |
completion_status = None |
|
28 |
from .afterjobs import AfterJob |
|
40 | 29 | |
41 | 30 | |
42 | 31 |
class HTTPResponse(quixote.http_response.HTTPResponse): |
... | ... | |
150 | 139 |
return '\n'.join(['<link rel="stylesheet" type="text/css" href="%scss/%s?%s" />' % ( |
151 | 140 |
root_url, x, version_hash) for x in self.css_includes]) |
152 | 141 | |
153 |
def add_after_job(self, label, cmd, fire_and_forget = False):
|
|
142 |
def add_after_job(self, label_or_instance, cmd=None, fire_and_forget=False):
|
|
154 | 143 |
if not self.after_jobs: |
155 | 144 |
self.after_jobs = [] |
156 |
job_id = randbytes(8) |
|
157 |
job = AfterJob(id = job_id) |
|
158 |
job.label = label |
|
159 |
job.creation_time = time.time() |
|
160 |
job.status = N_('registered') |
|
145 |
if isinstance(label_or_instance, AfterJob): |
|
146 |
job = label_or_instance |
|
147 |
else: |
|
148 |
job = AfterJob(label=label_or_instance, cmd=cmd) |
|
161 | 149 |
if fire_and_forget: |
162 | 150 |
job.id = None |
163 |
else: |
|
164 |
job.store() |
|
165 |
self.after_jobs.append((job, cmd)) |
|
151 |
self.after_jobs.append(job) |
|
166 | 152 |
return job |
167 | 153 | |
168 | 154 |
def process_after_jobs(self): |
169 |
if not self.after_jobs: |
|
170 |
return |
|
171 | ||
172 |
for job, job_function in self.after_jobs: |
|
173 |
if job.completion_time: |
|
174 |
continue |
|
175 |
job.status = N_('running') |
|
176 |
if job.id: |
|
177 |
job.store() |
|
178 |
try: |
|
179 |
job_function(job=job) |
|
180 |
except: |
|
181 |
get_publisher().notify_of_exception(sys.exc_info()) |
|
182 |
job.exception = traceback.format_exc() |
|
183 |
job.status = N_('failed') |
|
184 |
else: |
|
185 |
job.status = N_('completed') |
|
186 |
job.completion_time = time.time() |
|
187 |
if job.id: |
|
188 |
job.store() |
|
155 |
for job in self.after_jobs or []: |
|
156 |
job.run(spool=True) |
wcs/qommon/spooler.py | ||
---|---|---|
1 |
# w.c.s. - web application for online forms |
|
2 |
# Copyright (C) 2005-2020 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software; you can redistribute it and/or modify |
|
5 |
# it under the terms of the GNU General Public License as published by |
|
6 |
# the Free Software Foundation; either version 2 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU General Public License |
|
15 |
# along with this program; if not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
import subprocess |
|
18 | ||
19 |
from uwsgidecorators import spool |
|
20 | ||
21 | ||
22 |
@spool |
|
23 |
def run_after_job(args): |
|
24 |
from django.conf import settings |
|
25 |
subprocess.run([ |
|
26 |
settings.WCS_MANAGE_COMMAND, |
|
27 |
'runjob', |
|
28 |
'--domain', args['tenant_dir'].strip('/').split('/')[-1], |
|
29 |
'--job-id', args['job_id'] |
|
30 |
]) |
wcs/qommon/storage.py | ||
---|---|---|
291 | 291 |
_indexes = None |
292 | 292 |
_hashed_indexes = None |
293 | 293 |
_filename = None # None, unless must be saved to a specific location |
294 |
_reset_class = True # reset loaded object class |
|
294 | 295 | |
295 | 296 |
def __init__(self, id = None): |
296 | 297 |
self.id = id |
... | ... | |
506 | 507 |
finally: |
507 | 508 |
if fd: |
508 | 509 |
fd.close() |
509 |
o.__class__ = cls |
|
510 |
if cls._reset_class: |
|
511 |
o.__class__ = cls |
|
510 | 512 |
if any((isinstance(k, bytes) for k in o.__dict__)): |
511 | 513 |
pickle_2to3_conversion(o) |
512 | 514 |
if not ignore_migration: |
wcs/settings.py | ||
---|---|---|
173 | 173 | |
174 | 174 |
WCS_LEGACY_CONFIG_FILE = None |
175 | 175 | |
176 |
# management command, used to run afterjobs in uwsgi mode |
|
177 |
WCS_MANAGE_COMMAND = None |
|
178 | ||
176 | 179 |
# proxies=REQUESTS_PROXIES is used in python-requests call |
177 | 180 |
# http://docs.python-requests.org/en/master/user/advanced/?highlight=proxy#proxies |
178 | 181 |
REQUESTS_PROXIES = None |
179 |
- |