From ee3eea047c10d0032efc5a8b98d9bbdb54fe0a3a Mon Sep 17 00:00:00 2001 From: Tom Ritter Date: Sat, 11 Nov 2017 23:25:15 -0500 Subject: Fix executions that stomped on each other Every hour I would fire off two executions: the minute execution and the hour execution. Each would read the statefile on start. Whichever execution took longer would overwrite the state file with the data that was present for the other's execution upon start. This would manifest as extra notifications if the starts aligned - because the 'I already sent a notification' member update wasn't being persisted. This fixes it by only performing one execution at any given time, but knowing to run all the relevant jobs for that type of execution. AKA When I run the hourly job I also run the minute jobs. Another problem related to this one is logged in #1 - minute executions that take longer than a minute will also cause this type of error. --- README.md | 4 ++-- jobmanager.py | 4 ++-- jobs/JobBase.py | 4 ++-- main.py | 20 ++++++++++++++++++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a64837c..cd93862 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,8 @@ Checker relies on the system cron to run at every interval. You need one cron jo First edit the ... to your path, then enter the following into your cron - * * * * * .../checker/main.py -m cron -c minute >/dev/null 2>&1 - 0 * * * * .../checker/main.py -m cron -c hour >/dev/null 2>&1 + 1-59 * * * * .../checker/main.py -m cron -c minute >/dev/null 2>&1 + 0 1-11,13-23 * * * .../checker/main.py -m cron -c hour >/dev/null 2>&1 0 0 * * * .../checker/main.py -m cron -c day >/dev/null 2>&1 0 12 * * * .../checker/main.py -m cron -c day_noon >/dev/null 2>&1 diff --git a/jobmanager.py b/jobmanager.py index 5b3505f..5ba58b5 100755 --- a/jobmanager.py +++ b/jobmanager.py @@ -39,11 +39,11 @@ class JobManager: def list_jobs(self): return self.jobs - def execute_jobs(self, cronmode): + def execute_jobs(self, cronmodes): logging.info("Executing jobs...") emailWorks = True for thisJob in self.jobs: - if thisJob.shouldExecute(cronmode): + if thisJob.shouldExecute(cronmodes): logging.info("Executing " + thisJob.getName() + "(" + thisJob.getStateName() + ")") try: lastRunStatus = self.state[thisJob.getStateName()] diff --git a/jobs/JobBase.py b/jobs/JobBase.py index c5be180..3561992 100755 --- a/jobs/JobBase.py +++ b/jobs/JobBase.py @@ -44,9 +44,9 @@ class JobBase(object): return self.stateName """Returns True if the job should execute this cron-run""" - def shouldExecute(self, cronmode): + def shouldExecute(self, cronmodes): frequency = self.executeEvery() - if cronmode == frequency: + if frequency in cronmodes: return True return False diff --git a/main.py b/main.py index 49bb5bb..7ee5a16 100755 --- a/main.py +++ b/main.py @@ -18,6 +18,22 @@ from twisted.web import server from jobmanager import JobManager from statustracker import StatusTracker from servers import StatusSite, PingSite + +# We only run one command on the hour and day marks, but we need to execute all the jobs in that instance, +# including our minute jobs and hour jobs +def convert_crontimes(crontime): + ret = ["minute"] + if crontime == "minute": + pass + elif crontime == "hour": + ret.append("hour") + elif crontime == "day": + ret.append("hour") + ret.append("day") + elif crontime == "day_noon": + ret.append("hour") + ret.append("day_noon") + return ret if __name__ == "__main__": parser = argparse.ArgumentParser(description="Check your stuff.") @@ -78,9 +94,9 @@ if __name__ == "__main__": parser.print_help() sys.exit(-1) else: - log.info("Running cron at frequency " + args.crontime) + log.info("Running cron at frequencies " + str(convert_crontimes(args.crontime))) try: - if jobManager.execute_jobs(args.crontime): + if jobManager.execute_jobs(convert_crontimes(args.crontime)): jobManager.mark_jobs_ran() else: jobManager.mark_jobs_ran_with_error() -- cgit v1.2.3