Only notify on failures and successes after the number of failures required

3 files changed, 23 insertions, 12 deletions

diff --git a/jobmanager.py b/jobmanager.py
index d77bfb0..e4cb72a 100755
--- a/jobmanager.py
+++ b/jobmanager.py
@@ -65,7 +65,9 @@ class JobManager:
                 else:
                     #Successful Run
                     logging.info("Execution of " + thisJob.getName() + " succeeded")
-                    if lastRunStatus.CurrentStateSuccess == False and thisJob.notifyOnFailureEvery() == JobBase.JobFailureNotificationFrequency.ONSTATECHANGE:
+                    if lastRunStatus.CurrentStateSuccess == False and \
+                        thisJob.notifyOnFailureEvery() == JobBase.JobFailureNotificationFrequency.ONSTATECHANGE and \
+                        lastRunStatus.NumFailures >= thisJob.numberFailuresBeforeNotification():
                         if not thisJob.onStateChangeSuccess():
                             emailWorks = False
                     lastRunStatus.markSuccessful()
diff --git a/jobs/JobBase.py b/jobs/JobBase.py
index d7ec2a1..1882443 100755
--- a/jobs/JobBase.py
+++ b/jobs/JobBase.py
@@ -23,8 +23,8 @@ class JobFailureNotificationFrequency:
     ONSTATECHANGE = "state_change"
 
 class JobFailureCountMinimumBeforeNotification:
-    ONE = "one"
-    TWO = "two"
+    ONE = 1
+    TWO = 2
 
 class JobBase(object):
     def __init__(self, config, *args):
@@ -48,16 +48,16 @@ class JobBase(object):
             return True
         return False
 
-    """Returns True if the jobmanager should call 'onFailure' to alert the admin"""
+    """Returns True if the jobmanager should call 'onFailure' to alert the admin after a job failed"""
     def shouldNotifyFailure(self, jobState):
         notifyFrequency = self.notifyOnFailureEvery()
         minFailureCount = self.numberFailuresBeforeNotification()
+        currentFailureCount = jobState.NumFailures
 
-        if minFailureCount == JobFailureCountMinimumBeforeNotification.ONE:
-            pass
-        elif minFailureCount == JobFailureCountMinimumBeforeNotification.TWO:
-            if jobState.CurrentStateSuccess:
-                return False
+        if 1 + currentFailureCount >= minFailureCount:
+            pass #keep evaluating
+        else:
+            return False #Do not notify
 
         if notifyFrequency == JobFailureNotificationFrequency.EVERYTIME:
             return True
diff --git a/jobstate.py b/jobstate.py
index ba4113f..a8e6542 100644
--- a/jobstate.py
+++ b/jobstate.py
@@ -11,14 +11,17 @@ class JobState:
 		self.CurrentStateSuccess = True
 		self.FirstFailureTime = 0
 		self.LastNotifyTime = 0
+		self.NumFailures = 0
 
 	def markFailedAndNotify(self):
 		if self.CurrentStateSuccess:
 			self.CurrentStateSuccess = False
 			self.FirstFailureTime = time.time()
 			self.LastNotifyTime = self.FirstFailureTime
+			self.NumFailures = 1
 		else:
 			self.LastNotifyTime = time.time()
+			self.NumFailures += 1
 
 	def markFailedNoNotify(self):
 		if self.CurrentStateSuccess:
@@ -26,8 +29,9 @@ class JobState:
 			self.CurrentStateSuccess = False
 			self.FirstFailureTime = time.time()
 			self.LastNotifyTime = 0
+			self.NumFailures = 1
 		else:
-			pass
+			self.NumFailures += 1
 
 	def markSuccessful(self):
 		if self.CurrentStateSuccess:
@@ -36,13 +40,15 @@ class JobState:
 			self.CurrentStateSuccess = True
 			self.FirstFailureTime = 0
 			self.LastNotifyTime = 0
+			self.NumFailures = 0
 
 	def serialize(self):
 		ret  = self.name + "|" 
 		ret += "Succeeding" if self.CurrentStateSuccess else "Failing"
 		ret += "|" + str(self.FirstFailureTime)
 		ret += "|" + str(self.LastNotifyTime) + "|"
-		ret += self.friendlyname + "\n"
+		ret += self.friendlyname.replace("|", "#") #Why yes, this is ugly!
+		ret += "|" + str(self.NumFailures) + "\n"
 		return ret
 
 	@staticmethod
@@ -56,7 +62,10 @@ class JobState:
 		s.CurrentStateSuccess = True if parts[1] == "Succeeding" else False
 		s.FirstFailureTime = float(parts[2])
 		s.LastNotifyTime = float(parts[3])
-		s.friendlyname = parts[4] 
+		s.friendlyname = parts[4].replace("#", "|")
+
+		if len(parts) > 5:
+			s.NumFailures = int(parts[5])
 
 		return s