diff options
| -rwxr-xr-x | jobmanager.py | 4 | ||||
| -rwxr-xr-x | jobs/JobBase.py | 16 | ||||
| -rw-r--r-- | jobstate.py | 15 |
3 files changed, 23 insertions, 12 deletions
diff --git a/jobmanager.py b/jobmanager.py index d77bfb0..e4cb72a 100755 --- a/jobmanager.py +++ b/jobmanager.py @@ -65,7 +65,9 @@ class JobManager: else: #Successful Run logging.info("Execution of " + thisJob.getName() + " succeeded") - if lastRunStatus.CurrentStateSuccess == False and thisJob.notifyOnFailureEvery() == JobBase.JobFailureNotificationFrequency.ONSTATECHANGE: + if lastRunStatus.CurrentStateSuccess == False and \ + thisJob.notifyOnFailureEvery() == JobBase.JobFailureNotificationFrequency.ONSTATECHANGE and \ + lastRunStatus.NumFailures >= thisJob.numberFailuresBeforeNotification(): if not thisJob.onStateChangeSuccess(): emailWorks = False lastRunStatus.markSuccessful() diff --git a/jobs/JobBase.py b/jobs/JobBase.py index d7ec2a1..1882443 100755 --- a/jobs/JobBase.py +++ b/jobs/JobBase.py @@ -23,8 +23,8 @@ class JobFailureNotificationFrequency: ONSTATECHANGE = "state_change" class JobFailureCountMinimumBeforeNotification: - ONE = "one" - TWO = "two" + ONE = 1 + TWO = 2 class JobBase(object): def __init__(self, config, *args): @@ -48,16 +48,16 @@ class JobBase(object): return True return False - """Returns True if the jobmanager should call 'onFailure' to alert the admin""" + """Returns True if the jobmanager should call 'onFailure' to alert the admin after a job failed""" def shouldNotifyFailure(self, jobState): notifyFrequency = self.notifyOnFailureEvery() minFailureCount = self.numberFailuresBeforeNotification() + currentFailureCount = jobState.NumFailures - if minFailureCount == JobFailureCountMinimumBeforeNotification.ONE: - pass - elif minFailureCount == JobFailureCountMinimumBeforeNotification.TWO: - if jobState.CurrentStateSuccess: - return False + if 1 + currentFailureCount >= minFailureCount: + pass #keep evaluating + else: + return False #Do not notify if notifyFrequency == JobFailureNotificationFrequency.EVERYTIME: return True diff --git a/jobstate.py b/jobstate.py index ba4113f..a8e6542 100644 --- a/jobstate.py +++ b/jobstate.py @@ -11,14 +11,17 @@ class JobState: self.CurrentStateSuccess = True self.FirstFailureTime = 0 self.LastNotifyTime = 0 + self.NumFailures = 0 def markFailedAndNotify(self): if self.CurrentStateSuccess: self.CurrentStateSuccess = False self.FirstFailureTime = time.time() self.LastNotifyTime = self.FirstFailureTime + self.NumFailures = 1 else: self.LastNotifyTime = time.time() + self.NumFailures += 1 def markFailedNoNotify(self): if self.CurrentStateSuccess: @@ -26,8 +29,9 @@ class JobState: self.CurrentStateSuccess = False self.FirstFailureTime = time.time() self.LastNotifyTime = 0 + self.NumFailures = 1 else: - pass + self.NumFailures += 1 def markSuccessful(self): if self.CurrentStateSuccess: @@ -36,13 +40,15 @@ class JobState: self.CurrentStateSuccess = True self.FirstFailureTime = 0 self.LastNotifyTime = 0 + self.NumFailures = 0 def serialize(self): ret = self.name + "|" ret += "Succeeding" if self.CurrentStateSuccess else "Failing" ret += "|" + str(self.FirstFailureTime) ret += "|" + str(self.LastNotifyTime) + "|" - ret += self.friendlyname + "\n" + ret += self.friendlyname.replace("|", "#") #Why yes, this is ugly! + ret += "|" + str(self.NumFailures) + "\n" return ret @staticmethod @@ -56,7 +62,10 @@ class JobState: s.CurrentStateSuccess = True if parts[1] == "Succeeding" else False s.FirstFailureTime = float(parts[2]) s.LastNotifyTime = float(parts[3]) - s.friendlyname = parts[4] + s.friendlyname = parts[4].replace("#", "|") + + if len(parts) > 5: + s.NumFailures = int(parts[5]) return s |