From 3e85f1d0d48f0fb65a83af6ce3fb4b6a08f0bb1a Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 28 Dec 2016 13:59:43 -0600 Subject: Create an option that lets you require two failures before alerting. --- README.md | 2 ++ jobs/EmailChecker.py | 2 ++ jobs/HTTPServerChecker.py | 13 ++++++++----- jobs/JobBase.py | 18 ++++++++++++++++++ jobs/JobSpawner.py | 14 +++++++------- jobs/PeerChecker.py | 2 ++ jobs/TCPServerChecker.py | 11 +++++++---- jobs/TLSCertExpiration.py | 9 ++++++--- 8 files changed, 52 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index adaeff2..3be415d 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ JobBase is the base for a job, and should be used when you have a single, custom * This should return a JobFrequency constant indicating how often you want the job to run * notifyOnFailureEvery * This should return a JobFailureNotificationFrequency constant indicating how often you want to be notified about a (continually) failing job +* numberFailuresBeforeNotification + * This should return a JobFailureCountMinimumBeforeNotification constant indicating how many failures should occur in a row before notifying on failure * execute * This does the work. It returns True if the job succeeded or False if it didn't * onFailure diff --git a/jobs/EmailChecker.py b/jobs/EmailChecker.py index 818600d..74b41d9 100755 --- a/jobs/EmailChecker.py +++ b/jobs/EmailChecker.py @@ -14,6 +14,8 @@ class EmailChecker(JobBase.JobBase): return JobBase.JobFrequency.HOUR def notifyOnFailureEvery(self): return JobBase.JobFailureNotificationFrequency.EVERYTIME + def numberFailuresBeforeNotification(self): + return JobBase.JobFailureCountMinimumBeforeNotification.ONE def execute(self): if self.config.getboolean('email', 'nomail'): logging.debug("Pretending the EmailChecker job succeeded.") diff --git a/jobs/HTTPServerChecker.py b/jobs/HTTPServerChecker.py index 9336588..8f0e88f 100755 --- a/jobs/HTTPServerChecker.py +++ b/jobs/HTTPServerChecker.py @@ -8,17 +8,18 @@ import JobSpawner class HTTPServerChecker(JobSpawner.JobSpawner): servers = [ - #("http://example.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME), - #("https://exampletwo.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME) + #("http://example.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME, JobBase.JobFailureCountMinimumBeforeNotification.ONE), + #("https://exampletwo.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME, JobBase.JobFailureCountMinimumBeforeNotification.ONE) ] class ServerChecker(JobBase.JobBase): - def __init__(self, config, url, frequency, failureNotificationFrequency): + def __init__(self, config, url, frequency, failureNotificationFrequency, failuresBeforeNotification): self.config = config self.url = url self.frequency = frequency self.failureNotificationFrequency = failureNotificationFrequency - super(HTTPServerChecker.ServerChecker, self).__init__(config, url, frequency, failureNotificationFrequency) + self.failuresBeforeNotification = failuresBeforeNotification + super(HTTPServerChecker.ServerChecker, self).__init__(config, url, frequency, failureNotificationFrequency, failuresBeforeNotification) def getName(self): return str(self.__class__) + " for " + self.url @@ -26,6 +27,8 @@ class HTTPServerChecker(JobSpawner.JobSpawner): return self.frequency def notifyOnFailureEvery(self): return self.failureNotificationFrequency + def numberFailuresBeforeNotification(self): + return self.failuresBeforeNotification def execute(self): try: i = requests.get(self.url) @@ -44,5 +47,5 @@ class HTTPServerChecker(JobSpawner.JobSpawner): def get_sub_jobs(self, config): for s in self.servers: - yield self.ServerChecker(config, s[0], s[1], s[2]) + yield self.ServerChecker(config, s[0], s[1], s[2], s[3]) diff --git a/jobs/JobBase.py b/jobs/JobBase.py index 78c8c71..6b47eb7 100755 --- a/jobs/JobBase.py +++ b/jobs/JobBase.py @@ -22,6 +22,10 @@ class JobFailureNotificationFrequency: EVERYDAY = "day" ONSTATECHANGE = "state_change" +class JobFailureCountMinimumBeforeNotification: + ONE = "one" + TWO = "two" + class JobBase(object): def __init__(self, config, *args): self.config = config @@ -47,6 +51,14 @@ class JobBase(object): """Returns True if the jobmanager should call 'onFailure' to alert the admin""" def shouldNotifyFailure(self, jobState): notifyFrequency = self.notifyOnFailureEvery() + minFailureCount = self.numberFailuresBeforeNotification() + + if minFailureCount == JobFailureCountMinimumBeforeNotification.ONE: + pass + elif minFailureCount == JobFailureCountMinimumBeforeNotification.TWO: + if jobState.CurrentStateSuccess: + return False + if notifyFrequency == JobFailureNotificationFrequency.EVERYTIME: return True elif notifyFrequency == JobFailureNotificationFrequency.EVERYFIVEMINUTES: @@ -94,6 +106,12 @@ class JobBase(object): def notifyOnFailureEvery(self): pass + """OVERRIDE ME + Returns a JobFailureCountMinimumBeforeNotification indicating how many + failures should occur before a notification email should be sent""" + def numberFailuresBeforeNotification(self): + pass + """OVERRIDE ME Executes the job's actions, and returns true to indicate the job succeeded.""" def execute(self): diff --git a/jobs/JobSpawner.py b/jobs/JobSpawner.py index 0765339..ff9744f 100755 --- a/jobs/JobSpawner.py +++ b/jobs/JobSpawner.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python - -class JobSpawner: - """OVERRIDE ME - Returns an array (or using 'yield') of Job objects to run""" - def get_sub_jobs(self, config): - pass +#!/usr/bin/env python + +class JobSpawner: + """OVERRIDE ME + Returns an array (or using 'yield') of Job objects to run""" + def get_sub_jobs(self, config): + pass diff --git a/jobs/PeerChecker.py b/jobs/PeerChecker.py index 1214b0f..7832b25 100755 --- a/jobs/PeerChecker.py +++ b/jobs/PeerChecker.py @@ -23,6 +23,8 @@ class PeerChecker(JobSpawner.JobSpawner): return JobBase.JobFrequency.HOUR def notifyOnFailureEvery(self): return JobBase.JobFailureNotificationFrequency.EVERYTIME + def numberFailuresBeforeNotification(self): + return JobBase.JobFailureCountMinimumBeforeNotification.ONE def execute(self): peerOK = False diff --git a/jobs/TCPServerChecker.py b/jobs/TCPServerChecker.py index 4b5460c..d02cb58 100755 --- a/jobs/TCPServerChecker.py +++ b/jobs/TCPServerChecker.py @@ -9,18 +9,19 @@ import JobSpawner class TCPServerChecker(JobSpawner.JobSpawner): servers = [ - #("example.com", 53, "example.com:tcpdns", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME) + #("example.com", 53, "example.com:tcpdns", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME, JobBase.JobFailureCountMinimumBeforeNotification.ONE) ] class ServerChecker(JobBase.JobBase): - def __init__(self, config, ip, port, friendlyName, frequency, failureNotificationFrequency): + def __init__(self, config, ip, port, friendlyName, frequency, failureNotificationFrequency, failuresBeforeNotification): self.config = config self.ip = ip self.port = port self.friendlyName = friendlyName + "(" + self.ip + ":" + str(self.port) + ")" self.frequency = frequency self.failureNotificationFrequency = failureNotificationFrequency - super(TCPServerChecker.ServerChecker, self).__init__(config, ip, port, friendlyName, frequency, failureNotificationFrequency) + self.failuresBeforeNotification = failuresBeforeNotification + super(TCPServerChecker.ServerChecker, self).__init__(config, ip, port, friendlyName, frequency, failureNotificationFrequency, failuresBeforeNotification) def getName(self): return str(self.__class__) + " for " + self.friendlyName @@ -28,6 +29,8 @@ class TCPServerChecker(JobSpawner.JobSpawner): return self.frequency def notifyOnFailureEvery(self): return self.failureNotificationFrequency + def numberFailuresBeforeNotification(self): + return self.failuresBeforeNotification def execute(self): try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -46,6 +49,6 @@ class TCPServerChecker(JobSpawner.JobSpawner): def get_sub_jobs(self, config): for s in self.servers: - yield self.ServerChecker(config, s[0], s[1], s[2], s[3], s[4]) + yield self.ServerChecker(config, s[0], s[1], s[2], s[3], s[4], s[5]) diff --git a/jobs/TLSCertExpiration.py b/jobs/TLSCertExpiration.py index 873a53e..d5cfa48 100755 --- a/jobs/TLSCertExpiration.py +++ b/jobs/TLSCertExpiration.py @@ -16,12 +16,13 @@ class TLSCertExpiration(JobSpawner.JobSpawner): ] class CertChecker(JobBase.JobBase): - def __init__(self, config, url, frequency, failureNotificationFrequency): + def __init__(self, config, url, frequency, failureNotificationFrequency, failuresBeforeNotification): self.config = config self.url = url self.frequency = frequency self.failureNotificationFrequency = failureNotificationFrequency - super(TLSCertExpiration.CertChecker, self).__init__(config, url, frequency, failureNotificationFrequency) + self.failuresBeforeNotification = failuresBeforeNotification + super(TLSCertExpiration.CertChecker, self).__init__(config, url, frequency, failureNotificationFrequency, failuresBeforeNotification) def getName(self): return str(self.__class__) + " for " + self.url @@ -29,6 +30,8 @@ class TLSCertExpiration(JobSpawner.JobSpawner): return self.frequency def notifyOnFailureEvery(self): return self.failureNotificationFrequency + def numberFailuresBeforeNotification(self): + return self.failuresBeforeNotification def execute(self): try: context = ssl._create_unverified_context() @@ -54,5 +57,5 @@ class TLSCertExpiration(JobSpawner.JobSpawner): def get_sub_jobs(self, config): for s in self.servers: - yield self.CertChecker(config, s[0], s[1], s[2]) + yield self.CertChecker(config, s[0], s[1], s[2], s[3]) -- cgit v1.2.3