0001-misc-add-log-mecanism-for-cron-jobs-57604.patch

Lauréline Guérin, 21 octobre 2021 09:55

Voir les différences: en ligne côte à côte

Subject: [PATCH 1/2] misc: add log mecanism for cron jobs (#57604)

 tests/settings.py                      |  1 +
 tests/test_publisher.py                | 68 +++++++++++++++++++++++++-
 wcs/qommon/cron.py                     | 35 ++++++++++++-
 wcs/qommon/management/commands/cron.py |  5 +-
 wcs/settings.py                        |  1 +
 5 files changed, 106 insertions(+), 4 deletions(-)

tests/settings.py
1	1	TIME_ZONE = 'Europe/Paris'
2	2	LANGUAGE_CODE = 'en-us'
3	3	ALLOWED_HOSTS = ['*']
	4	CRON_LOG_FILENAME = '/var/lib/wcs/cron-test.log'

         assert pub.cfg['sp'] == {'what': 'ever'}
     def test_cron_command():
     def test_cron_command(settings):
         def clear_log_file():
             utcnow = datetime.datetime.utcnow()
             with open('%s-%s' % (settings.CRON_LOG_FILENAME, utcnow.strftime('%Y%m%d')), 'w'):
                 pass
         def get_logs():
             utcnow = datetime.datetime.utcnow()
             with open('%s-%s' % (settings.CRON_LOG_FILENAME, utcnow.strftime('%Y%m%d'))) as fd:
                 lines = fd.readlines()
             lines = [line[27:].strip() for line in lines]  # 27 chars for date & time
             return lines
         offset = ord(settings.SECRET_KEY[-1]) % 60
         pub = create_temporary_pub()
         with mock.patch('tempfile.gettempdir') as gettempdir:
             gettempdir.side_effect = lambda: pub.app_dir
-...
                     mock_tenants.return_value = [
                         Tenant(os.path.join(pub.app_dir, x)) for x in ('example.net', 'foo.bar', 'something.com')
+                    ]
                     clear_log_file()
                     call_command('cron')
                     assert cron_worker.call_count == 3
                     assert get_logs() == [
                         'starting cron (minutes offset is %s)' % offset,
                         '[tenant example.net] start',
                         '[tenant foo.bar] start',
                         '[tenant something.com] start',
+                    ]
                     cron_worker.reset_mock()
                     clear_log_file()
                     call_command('cron', domain='example.net')
                     assert cron_worker.call_count == 1
                     assert get_logs() == [
                         'starting cron (minutes offset is %s)' % offset,
                         '[tenant example.net] start',
+                    ]
                     cron_worker.reset_mock()
                     # disable cron on something.com
-...
                             '''
+                        )
                     clear_log_file()
                     call_command('cron')
                     assert cron_worker.call_count == 2
                     assert get_logs() == [
                         'starting cron (minutes offset is %s)' % offset,
                         '[tenant example.net] start',
                         '[tenant foo.bar] start',
+                    ]
                     cron_worker.reset_mock()
                     os.unlink(site_options_path)
-...
             def job2(pub):
                 jobs.append('job2')
             def job3(pub):
             def job3(pub, job):
                 jobs.append('job3')
                 for key in ['foo', 'bar', 'blah']:
                     with job.log_long_job(key):
                         pass
             @classmethod
             def register_test_cronjobs(cls):
-...
                 call_command('cron', job_name='job1', domain='example.net')
                 assert jobs == []
                 get_publisher_class().cronjobs = []
                 clear_log_file()
                 call_command('cron', job_name='job2', domain='example.net')
                 assert jobs == ['job2']
                 assert get_logs() == [
                     'starting cron (minutes offset is %s)' % offset,
                     '[tenant example.net] start',
+                ]
                 get_publisher_class().cronjobs = []
                 jobs = []
                 clear_log_file()
                 with mock.patch('wcs.qommon.cron.CronJob.LONG_JOB_DURATION', 0):
                     call_command('cron', job_name='job2', domain='example.net')
                 assert get_logs() == [
                     'starting cron (minutes offset is %s)' % offset,
                     '[tenant example.net] start',
                     '[tenant example.net] long job: job2 (took 0 minutes)',
+                ]
                 assert jobs == ['job2']
                 get_publisher_class().cronjobs = []
                 jobs = []
                 clear_log_file()
                 with mock.patch('wcs.qommon.cron.CronJob.LONG_JOB_DURATION', 0):
                     call_command('cron', job_name='job3', domain='example.net')
                 assert get_logs() == [
                     'starting cron (minutes offset is %s)' % offset,
                     '[tenant example.net] start',
                     '[tenant example.net] job3: running on "foo" took 0 minutes',
                     '[tenant example.net] job3: running on "bar" took 0 minutes',
                     '[tenant example.net] job3: running on "blah" took 0 minutes',
                     '[tenant example.net] long job: job3 (took 0 minutes)',
+                ]
                 assert jobs == ['job3']
     def test_clean_afterjobs():

     # You should have received a copy of the GNU General Public License
     # along with this program; if not, see <http://www.gnu.org/licenses/>.
     import datetime
     import inspect
     from contextlib import contextmanager
     from time import perf_counter
     from django.conf import settings
     from quixote import get_publisher
     class CronJob:
-...
         days = None
         function = None
         LONG_JOB_DURATION = 2 * 60  # 2 minutes
         def __init__(self, function, name=None, hours=None, minutes=None, weekdays=None, days=None):
             self.function = function
             self.name = name
-...
             self.weekdays = weekdays
             self.days = days
         @contextmanager
         def log_long_job(self, obj_description=None):
             start = perf_counter()
             yield
             duration = perf_counter() - start
             if duration > self.LONG_JOB_DURATION:
                 minutes = int(duration / 60)
                 if obj_description:
                     self.log('%s: running on "%s" took %d minutes' % (self.name, obj_description, minutes))
                 else:
                     self.log('long job: %s (took %s minutes)' % (self.name, minutes))
         @staticmethod
         def log(message, with_tenant=True):
             utcnow = datetime.datetime.utcnow()
             tenant_prefix = ''
             if with_tenant:
                 tenant_prefix = '[tenant %s] ' % get_publisher().tenant.hostname
             with open('%s-%s' % (settings.CRON_LOG_FILENAME, utcnow.strftime('%Y%m%d')), 'a+') as fd:
                 fd.write('%s %s%s\n' % (utcnow.isoformat(), tenant_prefix, message))
     def cron_worker(publisher, now, job_name=None):
         # reindex user and formdata if needed (should only be run once)
-...
             for extra_source in publisher.extra_sources:
                 publisher.substitutions.feed(extra_source(publisher, None))
             try:
                 job.function(publisher)
                 with job.log_long_job():
                     if 'job' in inspect.getfullargspec(job.function).args:
                         job.function(publisher, job=job)
                     else:
                         job.function(publisher)
             except Exception as e:
                 publisher.record_error(exception=e, context='[CRON]', notify=True)

     from django.conf import settings
     from django.core.management.base import BaseCommand, CommandError
     from wcs.qommon.cron import cron_worker
     from wcs.qommon.cron import CronJob, cron_worker
     from wcs.qommon.publisher import get_publisher_class
     from wcs.qommon.vendor import locket
     from wcs.wf.jump import JUMP_TIMEOUT_INTERVAL
-...
                 if verbosity > 1:
                     print('Command is ignored because DISABLE_CRON_JOBS is set in settings')
                 return
             offset = ord(settings.SECRET_KEY[-1]) % 60
             CronJob.log('starting cron (minutes offset is %s)' % offset, with_tenant=False)
             lockfile = os.path.join(tempfile.gettempdir(), 'wcs-cron-in-progress.lock')
             if options.get('domain'):
                 domains = [options.get('domain')]
-...
                             continue
                         if verbosity > 1:
                             print('cron work on %s' % hostname)
                         CronJob.log('start')
                         cron_worker(publisher, now, job_name=options.get('job_name'))
                 if verbosity > 2:
                     print('cron end (release lock %s)' % lockfile)

     # components, one should disable cron jobs execution on secondary servers;
     # set the following variable True disables "cron" management command
     DISABLE_CRON_JOBS = False
     CRON_LOG_FILENAME = '/var/lib/wcs/cron.log'
     # w.c.s. can have very large forms, in backoffice and frontoffice
     DATA_UPLOAD_MAX_NUMBER_FIELDS = 2000  # Django default is 1000
+    -

Projet

Général

Profil

Produits Entr'ouvert » w.c.s.

0001-misc-add-log-mecanism-for-cron-jobs-57604.patch