Projet

Général

Profil

0001-general-use-uwsgi-spooler-to-run-afterjobs-48407.patch

Frédéric Péters, 08 décembre 2020 13:40

Télécharger (15,3 ko)

Voir les différences:

Subject: [PATCH 1/5] general: use uwsgi spooler to run afterjobs (#48407)

 debian/control                        |  1 +
 debian/debian_config.py               |  1 +
 debian/uwsgi.ini                      |  3 ++
 debian/wcs.dirs                       |  1 +
 debian/wcs.init                       |  1 +
 debian/wcs.postinst                   |  1 +
 debian/wcs.service                    |  3 +-
 tests/test_ctl.py                     | 18 +++++++
 tests/test_publisher.py               | 34 ++++++-------
 wcs/ctl/management/commands/runjob.py | 39 +++++++++++++++
 wcs/qommon/afterjobs.py               | 71 +++++++++++++++++++++++++--
 wcs/qommon/http_response.py           | 50 ++++---------------
 wcs/qommon/spooler.py                 | 30 +++++++++++
 wcs/qommon/storage.py                 |  4 +-
 wcs/settings.py                       |  3 ++
 15 files changed, 196 insertions(+), 64 deletions(-)
 create mode 100644 wcs/ctl/management/commands/runjob.py
 create mode 100644 wcs/qommon/spooler.py
debian/control
21 21
    python3-psycopg2,
22 22
    python3-pyproj,
23 23
    python3-requests,
24
    python3-uwsgidecorators,
24 25
    python3-vobject,
25 26
    python3-xstatic-leaflet,
26 27
    python3-xstatic-leaflet-gesturehandling,
debian/debian_config.py
3 3
import os
4 4

  
5 5
PROJECT_NAME = 'wcs'
6
WCS_MANAGE_COMMAND = '/usr/bin/wcs-manage'
6 7

  
7 8
#
8 9
# hobotization
debian/uwsgi.ini
12 12
chmod-socket = 666
13 13
vacuum = true
14 14

  
15
spooler-processes = 3
16
import = wcs.qommon.spooler
17

  
15 18
master = true
16 19
enable-threads = true
17 20
harakiri = 120
debian/wcs.dirs
3 3
usr/lib/wcs
4 4
var/lib/wcs
5 5
var/lib/wcs/collectstatic
6
var/lib/wcs/spooler
6 7
var/log/wcs
debian/wcs.init
44 44
DAEMON_ARGS=${DAEMON_ARGS:-"--pidfile=$PIDFILE
45 45
--uid $USER --gid $GROUP
46 46
--ini /etc/$NAME/uwsgi.ini
47
--spooler /var/lib/wcs/spooler/
47 48
--daemonize /var/log/uwsgi.$NAME.log"}
48 49

  
49 50
# Load the VERBOSE setting and other rcS variables
debian/wcs.postinst
35 35
    chown $USER:$GROUP /var/log/$NAME
36 36
    chown $USER:$GROUP /var/lib/$NAME
37 37
    chown $USER:$GROUP /var/lib/$NAME/collectstatic
38
    chown $USER:$GROUP /var/lib/$NAME/spooler
38 39

  
39 40
    # create a secret file
40 41
    SECRET_FILE=$CONFIG_DIR/secret
debian/wcs.service
10 10
Group=%p
11 11
ExecStartPre=/usr/bin/wcs-manage migrate
12 12
ExecStartPre=/usr/bin/wcs-manage collectstatic
13
ExecStart=/usr/bin/uwsgi --ini /etc/%p/uwsgi.ini
13
ExecStartPre=/bin/mkdir -p /var/lib/wcs/spooler/%m/
14
ExecStart=/usr/bin/uwsgi --ini /etc/%p/uwsgi.ini --spooler /var/lib/wcs/spooler/%m/
14 15
ExecReload=/bin/kill -HUP $MAINPID
15 16
KillSignal=SIGQUIT
16 17
TimeoutStartSec=0
tests/test_ctl.py
12 12
from wcs.wf.jump import JumpWorkflowStatusItem
13 13
from wcs.fields import StringField, EmailField
14 14
import wcs.qommon.ctl
15
from wcs.qommon.afterjobs import AfterJob
15 16
from wcs.qommon.management.commands.collectstatic import Command as CmdCollectStatic
16 17
from wcs.qommon.management.commands.migrate import Command as CmdMigrate
17 18
from wcs.qommon.management.commands.migrate_schemas import Command as CmdMigrateSchemas
......
404 405
def test_shell():
405 406
    with pytest.raises(CommandError):
406 407
        call_command('shell')  # missing tenant name
408

  
409

  
410
class TestAfterJob(AfterJob):
411
    def execute(self):
412
        pass
413

  
414
def test_runjob(pub):
415
    with pytest.raises(CommandError):
416
        call_command('runjob')
417
    with pytest.raises(CommandError):
418
        call_command('runjob', '--domain=example.net', '--job-id=%s' % 'invalid')
419

  
420
    job = TestAfterJob(label='test')
421
    job.store()
422
    assert AfterJob.get(job.id).status == 'registered'
423
    call_command('runjob', '--domain=example.net', '--job-id=%s' % job.id)
424
    assert AfterJob.get(job.id).status == 'completed'
tests/test_publisher.py
272 272
def test_clean_afterjobs():
273 273
    pub = create_temporary_pub()
274 274

  
275
    job = AfterJob(id='a')
276
    job.status = 'completed'
277
    job.creation_time = time.time() - 3 * 3600
278
    job.completion_time = time.time() - 3 * 3600
279
    job.store()
280

  
281
    job = AfterJob(id='b')
282
    job.status = 'completed'
283
    job.creation_time = time.time()
284
    job.completion_time = time.time()
285
    job.store()
286

  
287
    job = AfterJob(id='c')
288
    job.status = 'running'
289
    job.creation_time = time.time() - 3 * 86400
290
    job.store()
275
    job1 = AfterJob()
276
    job1.status = 'completed'
277
    job1.creation_time = time.time() - 3 * 3600
278
    job1.completion_time = time.time() - 3 * 3600
279
    job1.store()
280

  
281
    job2 = AfterJob()
282
    job2.status = 'completed'
283
    job2.creation_time = time.time()
284
    job2.completion_time = time.time()
285
    job2.store()
286

  
287
    job3 = AfterJob()
288
    job3.status = 'running'
289
    job3.creation_time = time.time() - 3 * 86400
290
    job3.store()
291 291

  
292 292
    pub.clean_afterjobs()
293 293
    assert AfterJob.count() == 1
294
    assert AfterJob.select()[0].id == 'b'
294
    assert AfterJob.select()[0].id == job2.id
295 295

  
296 296

  
297 297
def test_clean_tempfiles():
wcs/ctl/management/commands/runjob.py
1
# w.c.s. - web application for online forms
2
# Copyright (C) 2005-2020  Entr'ouvert
3
#
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
8
#
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
# GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, see <http://www.gnu.org/licenses/>.
16

  
17
import sys
18

  
19
from django.core.management import CommandError
20

  
21
from wcs.qommon.http_response import AfterJob
22
from . import TenantCommand
23

  
24

  
25
class Command(TenantCommand):
26
    '''Run an afterjob (internal command)'''
27

  
28
    def add_arguments(self, parser):
29
        parser.add_argument('--domain', action='store', required=True)
30
        parser.add_argument('--job-id', action='store', required=True)
31

  
32
    def handle(self, *args, **options):
33
        domain = options.pop('domain')
34
        self.init_tenant_publisher(domain)
35
        try:
36
            job = AfterJob.get(options['job_id'])
37
        except KeyError:
38
            raise CommandError('missing job')
39
        job.run()
wcs/qommon/afterjobs.py
14 14
# You should have received a copy of the GNU General Public License
15 15
# along with this program; if not, see <http://www.gnu.org/licenses/>.
16 16

  
17
import sys
18
import time
19
import traceback
20
import uuid
21

  
22
from quixote import get_publisher, get_response
17 23
from quixote.directory import Directory
18
from quixote import get_response
19 24

  
20
from . import errors
21
from .http_response import AfterJob
22
from . import _
25
from . import _, N_, errors
26
from .storage import StorableObject
23 27

  
24 28

  
25 29
class AfterJobStatusDirectory(Directory):
......
34 38
        if not job.completion_status:
35 39
            return job.status + '|' + _(job.status)
36 40
        return job.status + '|' + _(job.status) + ' ' + job.completion_status
41

  
42

  
43
class AfterJob(StorableObject):
44
    _names = 'afterjobs'
45
    _reset_class = False
46

  
47
    label = None
48
    status = None
49
    creation_time = None
50
    completion_time = None
51
    completion_status = None
52

  
53
    execute = None
54

  
55
    def __init__(self, label=None, cmd=None, **kwargs):
56
        super().__init__(id=str(uuid.uuid4()))
57
        if label:
58
            self.label = label
59
        self.creation_time = time.time()
60
        self.job_cmd = cmd
61
        self.status = N_('registered')
62
        self.kwargs = kwargs
63

  
64
    def run(self, spool=False):
65
        if self.completion_time:
66
            return
67

  
68
        if spool and self.id and self.execute:
69
            from django.conf import settings
70
            if 'uwsgi' in sys.modules and settings.WCS_MANAGE_COMMAND:
71
                from .spooler import run_after_job
72
                self.store()
73
                run_after_job.spool(tenant_dir=get_publisher().app_dir, job_id=self.id)
74
                return
75

  
76
        self.status = N_('running')
77
        if self.id:
78
            self.store()
79
        try:
80
            if self.execute:
81
                self.execute()
82
            else:
83
                self.job_cmd(job=self)
84
        except Exception:
85
            get_publisher().notify_of_exception(sys.exc_info())
86
            self.exception = traceback.format_exc()
87
            self.status = N_('failed')
88
        else:
89
            self.status = N_('completed')
90
        self.completion_time = time.time()
91
        if self.id:
92
            self.store()
93

  
94
    def __getstate__(self):
95
        if not isinstance(self.job_cmd, str):
96
            obj_dict = self.__dict__.copy()
97
            obj_dict['job_cmd'] = None
98
            return obj_dict
99
        return self.__dict__
wcs/qommon/http_response.py
21 21

  
22 22
from django.utils.encoding import force_bytes
23 23
from quixote import get_publisher
24
from quixote.util import randbytes
25 24
import quixote.http_response
26 25
from quixote import get_publisher, get_request
27 26

  
28 27
from . import N_
29
from .storage import StorableObject
30

  
31

  
32
class AfterJob(StorableObject):
33
    _names = 'afterjobs'
34

  
35
    label = None
36
    status = None
37
    creation_time = None
38
    completion_time = None
39
    completion_status = None
28
from .afterjobs import AfterJob
40 29

  
41 30

  
42 31
class HTTPResponse(quixote.http_response.HTTPResponse):
......
150 139
        return '\n'.join(['<link rel="stylesheet" type="text/css" href="%scss/%s?%s" />' % (
151 140
                    root_url, x, version_hash) for x in self.css_includes])
152 141

  
153
    def add_after_job(self, label, cmd, fire_and_forget = False):
142
    def add_after_job(self, label_or_instance, cmd=None, fire_and_forget=False):
154 143
        if not self.after_jobs:
155 144
            self.after_jobs = []
156
        job_id = randbytes(8)
157
        job = AfterJob(id = job_id)
158
        job.label = label
159
        job.creation_time = time.time()
160
        job.status = N_('registered')
145
        if isinstance(label_or_instance, AfterJob):
146
            job = label_or_instance
147
        else:
148
            job = AfterJob(label=label_or_instance, cmd=cmd)
161 149
        if fire_and_forget:
162 150
            job.id = None
163
        else:
164
            job.store()
165
        self.after_jobs.append((job, cmd))
151
        self.after_jobs.append(job)
166 152
        return job
167 153

  
168 154
    def process_after_jobs(self):
169
        if not self.after_jobs:
170
            return
171

  
172
        for job, job_function in self.after_jobs:
173
            if job.completion_time:
174
                continue
175
            job.status = N_('running')
176
            if job.id:
177
                job.store()
178
            try:
179
                job_function(job=job)
180
            except:
181
                get_publisher().notify_of_exception(sys.exc_info())
182
                job.exception = traceback.format_exc()
183
                job.status = N_('failed')
184
            else:
185
                job.status = N_('completed')
186
            job.completion_time = time.time()
187
            if job.id:
188
                job.store()
155
        for job in self.after_jobs or []:
156
            job.run(spool=True)
wcs/qommon/spooler.py
1
# w.c.s. - web application for online forms
2
# Copyright (C) 2005-2020  Entr'ouvert
3
#
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
8
#
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
# GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, see <http://www.gnu.org/licenses/>.
16

  
17
import subprocess
18

  
19
from uwsgidecorators import spool
20

  
21

  
22
@spool
23
def run_after_job(args):
24
    from django.conf import settings
25
    subprocess.run([
26
        settings.WCS_MANAGE_COMMAND,
27
        'runjob',
28
        '--domain', args['tenant_dir'].strip('/').split('/')[-1],
29
        '--job-id', args['job_id']
30
    ])
wcs/qommon/storage.py
291 291
    _indexes = None
292 292
    _hashed_indexes = None
293 293
    _filename = None # None, unless must be saved to a specific location
294
    _reset_class = True  # reset loaded object class
294 295

  
295 296
    def __init__(self, id = None):
296 297
        self.id = id
......
506 507
        finally:
507 508
            if fd:
508 509
                fd.close()
509
        o.__class__ = cls
510
        if cls._reset_class:
511
            o.__class__ = cls
510 512
        if any((isinstance(k, bytes) for k in o.__dict__)):
511 513
            pickle_2to3_conversion(o)
512 514
        if not ignore_migration:
wcs/settings.py
173 173

  
174 174
WCS_LEGACY_CONFIG_FILE = None
175 175

  
176
# management command, used to run afterjobs in uwsgi mode
177
WCS_MANAGE_COMMAND = None
178

  
176 179
# proxies=REQUESTS_PROXIES is used in python-requests call
177 180
# http://docs.python-requests.org/en/master/user/advanced/?highlight=proxy#proxies
178 181
REQUESTS_PROXIES = None
179
-