Refactor lots of things to allow you to be notified every so often, instead of every single time.

7 files changed, 160 insertions, 54 deletions

diff --git a/jobs/EmailChecker.py b/jobs/EmailChecker.py
index 925e0db..939d5b8 100755
--- a/jobs/EmailChecker.py
+++ b/jobs/EmailChecker.py
@@ -12,6 +12,8 @@ import JobBase
 class EmailChecker(JobBase.JobBase):
     def executeEvery(self):
         return JobBase.JobFrequency.HOUR
+    def notifyOnFailureEvery(self):
+        return JobBase.JobFailureNotificationFrequency.EVERYTIME
     def execute(self):
         USER = self.config.get('email', 'user')
         PASS = self.config.get('email', 'pass')
@@ -47,9 +49,9 @@ class EmailChecker(JobBase.JobBase):
                 foundSubject = True
         M.close()
         M.logout()
-        if not foundSubject:
-            #This may not work, but try anyway
-            self.sendEmail("Email Fetch Failure", logdetails)
-            return False
-        else:
-            return True
+
+        self.logdetails = logdetails
+        return foundSubject
+    def onFailure(self):
+        return self.sendEmail("Email Fetch Failure", self.logdetails)
+            
diff --git a/jobs/HTTPServerChecker.py b/jobs/HTTPServerChecker.py
index ec8eda1..ec2a9d6 100755
--- a/jobs/HTTPServerChecker.py
+++ b/jobs/HTTPServerChecker.py
@@ -8,29 +8,37 @@ import JobSpawner
 

 class HTTPServerChecker(JobSpawner.JobSpawner):

     servers = [ 

-                #("http://example.com", JobBase.JobFrequency.MINUTE),

-                #("https://exampletwo.com", JobBase.JobFrequency.MINUTE)

+                #("http://example.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME),

+                #("https://exampletwo.com", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME)

               ]

 

     class ServerChecker(JobBase.JobBase):

-        def __init__(self, url, frequency):

+        def __init__(self, config, url, frequency, failureNotificationFrequency):

+            self.config = config

             self.url = url

             self.frequency = frequency

+            self.failureNotificationFrequency = failureNotificationFrequency

 

         def getName(self):

             return str(self.__class__) + " for " + self.url

         def executeEvery(self):

             return self.frequency

+        def notifyOnFailureEvery(self):

+            return self.failureNotificationFrequency

         def execute(self):

             try:

                 requests.get(self.url)

                 return True

             except:

-                msg = "Could not hit server " + self.url

-                logging.warn(msg)

-                return self.sendEmail(msg, "")

+                self.failuremsg = "Could not hit server " + self.url

+                logging.warn(self.failuremsg)

+                return False

+        def onFailure(self):

+            return self.sendEmail(self.failuremsg, "")

+        def onStateChangeSuccess(self):

+            return self.sendEmail("Successfully hit " + self.url, "")

 

-    def get_sub_jobs(self):

+    def get_sub_jobs(self, config):

         for s in self.servers:

-            yield self.ServerChecker(s[0], s[1])

+            yield self.ServerChecker(config, s[0], s[1], s[2])

 

diff --git a/jobs/JobBase.py b/jobs/JobBase.py
index a7f02c9..29ca443 100755
--- a/jobs/JobBase.py
+++ b/jobs/JobBase.py
@@ -1,7 +1,9 @@
 #!/usr/bin/env python
 
+import time
 import random
 import logging
+import datetime
 
 import smtplib
 
@@ -11,27 +13,100 @@ class JobFrequency:
     DAY = "day"
     DAY_NOON = "day_noon"
 
+class JobFailureNotificationFrequency:
+    EVERYTIME = "every"
+    EVERYFIVEMINUTES = "5min"
+    EVERYTENMINUTES = "10min"
+    EVERYHOUR = "hour"
+    ONSTATECHANGE = "state_change"
+
 class JobBase:
-    def __init__(self):
-        self.config = None
+    def __init__(self, config):
+        self.config = config
+
+    """ Return a friendly name to identify this Job"""
     def getName(self):
         return str(self.__class__)
+
+    """Return a non-friendly, guarenteed-unique name to identify this Job
+       Needed to keep track of the job's run history. 
+       Takes into account the contructor arguments to uniquely identify JobSpawner-jobs"""
+    def getStateName(self):
+        return self.getName()
+
+    """Returns True if the job should execute this cron-run"""
     def shouldExecute(self, cronmode):
         frequency = self.executeEvery()
         if cronmode == frequency:
             return True
         return False
-    def setConfig(self, config):
-        self.config = config
-       
+
+    """Returns True if the jobmanager should call 'onFailure' to alert the admin"""
+    def shouldNotifyFailure(self, jobState):
+        notifyFrequency = self.notifyOnFailureEvery()
+        if notifyFrequency == JobFailureNotificationFrequency.EVERYTIME:
+            return True
+        elif notifyFrequency == JobFailureNotificationFrequency.EVERYFIVEMINUTES:
+            now = time.time()
+            lastNotify = jobState.LastNotifyTime
+            if now - lastNotify > datetime.timedelta(minutes=4, seconds=30):
+                return True
+            return False
+        elif notifyFrequency == JobFailureNotificationFrequency.EVERYTENMINUTES:
+            now = time.time()
+            lastNotify = jobState.LastNotifyTime
+            if now - lastNotify > datetime.timedelta(minutes=9, seconds=15):
+                return True
+            return False
+        elif notifyFrequency == JobFailureNotificationFrequency.EVERYHOUR:
+            now = time.time()
+            lastNotify = jobState.LastNotifyTime
+            if now - lastNotify > datetime.timedelta(minutes=59, seconds=0):
+                return True
+            return False
+        elif notifyFrequency == JobFailureNotificationFrequency.ONSTATECHANGE:
+            #Only notify if the last JobState was a Success
+            return jobState.CurrentStateSuccess
+        return True
+
+    """Helper method to send email"""
     def sendEmail(self, subject, body, to=""):
         return sendEmail(self.config, subject, body, to)
 
+
+    """OVERRIDE ME
+        Returns a JobFrequency indicating how often the job should be run."""
     def executeEvery(self):
         pass
+
+    """OVERRIDE ME
+        Returns a JobFailureNotificationFrequency indicating how often a failure 
+        notification email should be sent"""
+    def notifyOnFailureEvery(self):
+        pass
+
+    """OVERRIDE ME
+       Executes the job's actions, and returns true to indicate the job succeeded."""
     def execute(self):
         pass
 
+    """OVERRIDE ME
+       Notify the admin the job failed. Returns True if the email could be 
+       successfully sent.
+       Example: return self.sendEmail(self.subject, self.body, self.notificationAddress)"""
+    def onFailure(self):
+        pass
+
+    """OVERRIDE ME
+       Notify the admin the job succeeded (when it was previously failing). Only used for
+       JobFailureNotificationFrequency.ONSTATECHANGE
+
+       Returns True if the email could be successfully sent.
+       Example: return self.sendEmail(self.subject, self.body, self.notificationAddress)"""
+    def onStateChangeSuccess(self):
+        log.warn(self.getName() + " did not override onStateChangeSuccess")
+        return True
+
 def sendEmail(config, subject, body, to=""):
     FROM = config.get('email', 'user')
     PASS = config.get('email', 'pass')
diff --git a/jobs/JobSpawner.py b/jobs/JobSpawner.py
index 3d09693..50f3043 100755
--- a/jobs/JobSpawner.py
+++ b/jobs/JobSpawner.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python

 

 class JobSpawner:

-    def get_sub_jobs(self):

+	"""OVERRIDE ME

+	   Returns an array (or using 'yield') of Job objects to run"""

+    def get_sub_jobs(self, config):

         pass

diff --git a/jobs/PeerChecker.py b/jobs/PeerChecker.py
index f17da53..8211472 100755
--- a/jobs/PeerChecker.py
+++ b/jobs/PeerChecker.py
@@ -9,46 +9,57 @@ import requests
 

 import JobBase

 

-class PeerChecker(JobBase.JobBase):

-    def executeEvery(self):

-        return JobBase.JobFrequency.HOUR

-    def execute(self):

-        testSuccess = True

-        peers = self.config.items('peers')

-        for p in peers:

-            peer = p[1].split(',')

+class PeerChecker(JobSpawner.JobSpawner):

+    class IndividualPeerChecker(JobBase.JobBase):

+        def __init__(self, config, checkurl, notificationAddress):

+            self.checkurl = checkurl

+            self.notificationAddress = notificationAddress

+

+        def executeEvery(self):

+            return JobBase.JobFrequency.HOUR

+        def notifyOnFailureEvery(self):

+            return JobBase.JobFailureNotificationFrequency.EVERYTIME

+        def execute(self):

             peerOK = False

 

-            subject = ""

-            body = ""

+            self.subject = ""

+            self.body = ""

 

             try:

-                response = requests.get(peer[0])

+                response = requests.get(self.checkurl)

                 if response.status_code != 200:

                     peerOK = False

-                    subject = peer[0] + " returned a non-standard status code."

-                    body = str(response.status_code) + "\n" + response.content

+                    self.subject = self.checkurl + " returned a non-standard status code."

+                    self.body = str(response.status_code) + "\n" + response.content

                 else:

                     if "True" in response.content:

                         peerOK = True

                     elif "MailProblem" in response.content:

                         peerOK = False

-                        subject = peer[0] + " reports it cannot send email."

-                        body = str(response.status_code) + "\n" + response.content

+                        self.subject = self.checkurl + " reports it cannot send email."

+                        self.body = str(response.status_code) + "\n" + response.content

                     elif "JobProblem" in response.content:

                         peerOK = False

-                        subject = peer[0] + " reports its jobs are not running."

-                        body = str(response.status_code) + "\n" + response.content

+                        self.subject = self.checkurl + " reports its jobs are not running."

+                        self.body = str(response.status_code) + "\n" + response.content

                     else:

                         peerOK = False

-                        subject = peer[0] + " had an unexpected response."

-                        body = str(response.status_code) + "\n" + response.content

+                        self.subject = self.checkurl + " had an unexpected response."

+                        self.body = str(response.status_code) + "\n" + response.content

             except Exception as e:

                 peerOK = False

-                subject = peer[0] + " is not responding."

-                body = str(e)

+                self.subject = self.checkurl + " is not responding."

+                self.body = str(e)

+            return peerOK

             

-            if not peerOK:

-                if not self.sendEmail(subject, body, peer[1]):

-                    testSuccess = False

-        return testSuccess

+            return peerOK:

+        def onFailure(self):

+            return self.sendEmail(self.subject, self.body, self.notificationAddress)

+        def onStateChangeSuccess(self):

+            return self.sendEmail("Successfully hit " + self.checkurl, "", self.notificationAddress)

+

+    def get_sub_jobs(self, config):

+        peers = config.items('peers')

+        for p in peers:

+            (address, email) = p[1].split(',')

+            yield self.IndividualPeerChecker(config, address, email)

diff --git a/jobs/TCPServerChecker.py b/jobs/TCPServerChecker.py
index 711047b..642e188 100755
--- a/jobs/TCPServerChecker.py
+++ b/jobs/TCPServerChecker.py
@@ -9,20 +9,24 @@ import JobSpawner
 

 class TCPServerChecker(JobSpawner.JobSpawner):

     servers = [ 

-                #("example.com", 53, "example.com:tcpdns", JobBase.JobFrequency.MINUTE),

+                #("example.com", 53, "example.com:tcpdns", JobBase.JobFrequency.MINUTE, JobBase.JobFailureNotificationFrequency.EVERYTIME)

               ]

 

     class ServerChecker(JobBase.JobBase):

-        def __init__(self, ip, port, friendlyName, frequency):

+        def __init__(self, config, ip, port, friendlyName, frequency, failureNotificationFrequency):

+            self.config = config

             self.ip = ip

             self.port = port

             self.friendlyName = friendlyName + "(" + self.ip + ":" + str(self.port) + ")"

             self.frequency = frequency

+            self.failureNotificationFrequency = failureNotificationFrequency

 

         def getName(self):

             return str(self.__class__) + " for " + self.friendlyName

         def executeEvery(self):

             return self.frequency

+        def notifyOnFailureEvery(self):

+            return self.failureNotificationFrequency

         def execute(self):

             try:

                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

@@ -30,12 +34,16 @@ class TCPServerChecker(JobSpawner.JobSpawner):
                 s.close()

                 return True

             except:

-                msg = "Could not hit server " + self.friendlyName

-                logging.warn(msg)

-                return self.sendEmail(msg, "")

+                self.failuremsg = "Could not hit server " + self.friendlyName

+                logging.warn(self.failuremsg)

+                return False

+        def onFailure(self):

+            return self.sendEmail(self.failuremsg, "")

+        def onStateChangeSuccess(self):

+            return self.sendEmail("Successfully hit " + self.friendlyName, "")

 

-    def get_sub_jobs(self):

+    def get_sub_jobs(self, config):

         for s in self.servers:

-            yield self.ServerChecker(s[0], s[1], s[2], s[3])

+            yield self.ServerChecker(config, s[0], s[1], s[2], s[3], s[4])

 

             

diff --git a/jobs/__init__.py b/jobs/__init__.py
index 9955164..0e780eb 100755
--- a/jobs/__init__.py
+++ b/jobs/__init__.py
@@ -44,10 +44,10 @@ class JobFinder:
                             # It has to do with JobBase being imported multiple times (within jobs) or something
                             if base.__name__ == 'JobBase':
                                 # A job was found, keep it
-                                self._jobs.add(obj())
+                                self._jobs.add(obj(self.config))
                             elif base.__name__ == 'JobSpawner':
                                 spawner = obj()
-                                for j in spawner.get_sub_jobs():
+                                for j in spawner.get_sub_jobs(self.config):
                                     self._jobs.add(j)