Create an option that lets you require two failures before alerting.

8 files changed, 52 insertions, 19 deletions

diff --git a/README.md b/README.md
index adaeff2..3be415d 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,8 @@ JobBase is the base for a job, and should be used when you have a single, custom
  * This should return a JobFrequency constant indicating how often you want the job to run
 * notifyOnFailureEvery
  * This should return a JobFailureNotificationFrequency constant indicating how often you want to be notified about a (continually) failing job
+* numberFailuresBeforeNotification
+ * This should return a JobFailureCountMinimumBeforeNotification constant indicating how many failures should occur in a row before notifying on failure
 * execute
  * This does the work. It returns True if the job succeeded or False if it didn't
 * onFailure
diff --git a/jobs/EmailChecker.py b/jobs/EmailChecker.py
index 818600d..74b41d9 100755
--- a/jobs/EmailChecker.py
+++ b/jobs/EmailChecker.py
@@ -14,6 +14,8 @@ class EmailChecker(JobBase.JobBase):
         return JobBase.JobFrequency.HOUR
     def notifyOnFailureEvery(self):
         return JobBase.JobFailureNotificationFrequency.EVERYTIME
+    def numberFailuresBeforeNotification(self):
+        return JobBase.JobFailureCountMinimumBeforeNotification.ONE
     def execute(self):
         if self.config.getboolean('email', 'nomail'):
             logging.debug("Pretending the EmailChecker job succeeded.")
diff --git a/jobs/HTTPServerChecker.py b/jobs/HTTPServerChecker.py
index 9336588..8f0e88f 100755
--- a/jobs/HTTPServerChecker.py
+++ b/jobs/HTTPServerChecker.py
@@ -8,17 +8,18 @@ import JobSpawner
 

 class HTTPServerChecker(JobSpawner.JobSpawner):

     servers = [ 

-                #("http://example.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME),

-                #("https://exampletwo.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME)

+                #("http://example.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME, JobBase.JobFailureCountMinimumBeforeNotification.ONE),

+                #("https://exampletwo.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME, JobBase.JobFailureCountMinimumBeforeNotification.ONE)

               ]

 

     class ServerChecker(JobBase.JobBase):

-        def __init__(self, config, url, frequency, failureNotificationFrequency):

+        def __init__(self, config, url, frequency, failureNotificationFrequency, failuresBeforeNotification):

             self.config = config

             self.url = url

             self.frequency = frequency

             self.failureNotificationFrequency = failureNotificationFrequency

-            super(HTTPServerChecker.ServerChecker, self).__init__(config, url, frequency, failureNotificationFrequency)

+            self.failuresBeforeNotification = failuresBeforeNotification

+            super(HTTPServerChecker.ServerChecker, self).__init__(config, url, frequency, failureNotificationFrequency, failuresBeforeNotification)

 

         def getName(self):

             return str(self.__class__) + " for " + self.url

@@ -26,6 +27,8 @@ class HTTPServerChecker(JobSpawner.JobSpawner):
             return self.frequency

         def notifyOnFailureEvery(self):

             return self.failureNotificationFrequency

+        def numberFailuresBeforeNotification(self):

+            return self.failuresBeforeNotification

         def execute(self):

             try:

                 i = requests.get(self.url)

@@ -44,5 +47,5 @@ class HTTPServerChecker(JobSpawner.JobSpawner):
 

     def get_sub_jobs(self, config):

         for s in self.servers:

-            yield self.ServerChecker(config, s[0], s[1], s[2])

+            yield self.ServerChecker(config, s[0], s[1], s[2], s[3])

 

diff --git a/jobs/JobBase.py b/jobs/JobBase.py
index 78c8c71..6b47eb7 100755
--- a/jobs/JobBase.py
+++ b/jobs/JobBase.py
@@ -22,6 +22,10 @@ class JobFailureNotificationFrequency:
     EVERYDAY = "day"
     ONSTATECHANGE = "state_change"
 
+class JobFailureCountMinimumBeforeNotification:
+    ONE = "one"
+    TWO = "two"
+
 class JobBase(object):
     def __init__(self, config, *args):
         self.config = config
@@ -47,6 +51,14 @@ class JobBase(object):
     """Returns True if the jobmanager should call 'onFailure' to alert the admin"""
     def shouldNotifyFailure(self, jobState):
         notifyFrequency = self.notifyOnFailureEvery()
+        minFailureCount = self.numberFailuresBeforeNotification()
+
+        if minFailureCount == JobFailureCountMinimumBeforeNotification.ONE:
+            pass
+        elif minFailureCount == JobFailureCountMinimumBeforeNotification.TWO:
+            if jobState.CurrentStateSuccess:
+                return False
+
         if notifyFrequency == JobFailureNotificationFrequency.EVERYTIME:
             return True
         elif notifyFrequency == JobFailureNotificationFrequency.EVERYFIVEMINUTES:
@@ -95,6 +107,12 @@ class JobBase(object):
         pass
 
     """OVERRIDE ME
+        Returns a JobFailureCountMinimumBeforeNotification indicating how many 
+        failures should occur before a notification email should be sent"""
+    def numberFailuresBeforeNotification(self):
+        pass
+
+    """OVERRIDE ME
        Executes the job's actions, and returns true to indicate the job succeeded."""
     def execute(self):
         pass
diff --git a/jobs/JobSpawner.py b/jobs/JobSpawner.py
index 0765339..ff9744f 100755
--- a/jobs/JobSpawner.py
+++ b/jobs/JobSpawner.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python

-

-class JobSpawner:

-    """OVERRIDE ME

-       Returns an array (or using 'yield') of Job objects to run"""

-    def get_sub_jobs(self, config):

-        pass

+#!/usr/bin/env python
+
+class JobSpawner:
+    """OVERRIDE ME
+       Returns an array (or using 'yield') of Job objects to run"""
+    def get_sub_jobs(self, config):
+        pass
diff --git a/jobs/PeerChecker.py b/jobs/PeerChecker.py
index 1214b0f..7832b25 100755
--- a/jobs/PeerChecker.py
+++ b/jobs/PeerChecker.py
@@ -23,6 +23,8 @@ class PeerChecker(JobSpawner.JobSpawner):
             return JobBase.JobFrequency.HOUR

         def notifyOnFailureEvery(self):

             return JobBase.JobFailureNotificationFrequency.EVERYTIME

+        def numberFailuresBeforeNotification(self):

+            return JobBase.JobFailureCountMinimumBeforeNotification.ONE

         def execute(self):

             peerOK = False

 

diff --git a/jobs/TCPServerChecker.py b/jobs/TCPServerChecker.py
index 4b5460c..d02cb58 100755
--- a/jobs/TCPServerChecker.py
+++ b/jobs/TCPServerChecker.py
@@ -9,18 +9,19 @@ import JobSpawner
 

 class TCPServerChecker(JobSpawner.JobSpawner):

     servers = [ 

-                #("example.com", 53, "example.com:tcpdns", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME)

+                #("example.com", 53, "example.com:tcpdns", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME, JobBase.JobFailureCountMinimumBeforeNotification.ONE)

               ]

 

     class ServerChecker(JobBase.JobBase):

-        def __init__(self, config, ip, port, friendlyName, frequency, failureNotificationFrequency):

+        def __init__(self, config, ip, port, friendlyName, frequency, failureNotificationFrequency, failuresBeforeNotification):

             self.config = config

             self.ip = ip

             self.port = port

             self.friendlyName = friendlyName + "(" + self.ip + ":" + str(self.port) + ")"

             self.frequency = frequency

             self.failureNotificationFrequency = failureNotificationFrequency

-            super(TCPServerChecker.ServerChecker, self).__init__(config, ip, port, friendlyName, frequency, failureNotificationFrequency)

+            self.failuresBeforeNotification = failuresBeforeNotification

+            super(TCPServerChecker.ServerChecker, self).__init__(config, ip, port, friendlyName, frequency, failureNotificationFrequency, failuresBeforeNotification)

 

         def getName(self):

             return str(self.__class__) + " for " + self.friendlyName

@@ -28,6 +29,8 @@ class TCPServerChecker(JobSpawner.JobSpawner):
             return self.frequency

         def notifyOnFailureEvery(self):

             return self.failureNotificationFrequency

+        def numberFailuresBeforeNotification(self):

+            return self.failuresBeforeNotification

         def execute(self):

             try:

                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

@@ -46,6 +49,6 @@ class TCPServerChecker(JobSpawner.JobSpawner):
 

     def get_sub_jobs(self, config):

         for s in self.servers:

-            yield self.ServerChecker(config, s[0], s[1], s[2], s[3], s[4])

+            yield self.ServerChecker(config, s[0], s[1], s[2], s[3], s[4], s[5])

 

             

diff --git a/jobs/TLSCertExpiration.py b/jobs/TLSCertExpiration.py
index 873a53e..d5cfa48 100755
--- a/jobs/TLSCertExpiration.py
+++ b/jobs/TLSCertExpiration.py
@@ -16,12 +16,13 @@ class TLSCertExpiration(JobSpawner.JobSpawner):
               ]
 
     class CertChecker(JobBase.JobBase):
-        def __init__(self, config, url, frequency, failureNotificationFrequency):
+        def __init__(self, config, url, frequency, failureNotificationFrequency, failuresBeforeNotification):
             self.config = config
             self.url = url
             self.frequency = frequency
             self.failureNotificationFrequency = failureNotificationFrequency
-            super(TLSCertExpiration.CertChecker, self).__init__(config, url, frequency, failureNotificationFrequency)
+            self.failuresBeforeNotification = failuresBeforeNotification
+            super(TLSCertExpiration.CertChecker, self).__init__(config, url, frequency, failureNotificationFrequency, failuresBeforeNotification)
 
         def getName(self):
             return str(self.__class__) + " for " + self.url
@@ -29,6 +30,8 @@ class TLSCertExpiration(JobSpawner.JobSpawner):
             return self.frequency
         def notifyOnFailureEvery(self):
             return self.failureNotificationFrequency
+        def numberFailuresBeforeNotification(self):
+            return self.failuresBeforeNotification
         def execute(self):
             try:
                 context = ssl._create_unverified_context()
@@ -54,5 +57,5 @@ class TLSCertExpiration(JobSpawner.JobSpawner):
 
     def get_sub_jobs(self, config):
         for s in self.servers:
-            yield self.CertChecker(config, s[0], s[1], s[2])
+            yield self.CertChecker(config, s[0], s[1], s[2], s[3])