From 9b25f65ca655a567873c66c2b015884a3e013276 Mon Sep 17 00:00:00 2001 From: Tom Ritter Date: Mon, 25 Jan 2016 21:24:41 -0500 Subject: Initial commit of checker --- .gitignore | 2 ++ jobmanager.py | 35 +++++++++++++++++++ jobs/EmailChecker.py | 46 +++++++++++++++++++++++++ jobs/HTTPServerChecker.py | 36 ++++++++++++++++++++ jobs/JobBase.py | 53 +++++++++++++++++++++++++++++ jobs/JobSpawner.py | 5 +++ jobs/PeerChecker.py | 39 +++++++++++++++++++++ jobs/TCPServerChecker.py | 41 ++++++++++++++++++++++ jobs/__init__.py | 86 +++++++++++++++++++++++++++++++++++++++++++++++ main.py | 70 ++++++++++++++++++++++++++++++++++++++ servers.py | 43 ++++++++++++++++++++++++ settings.cfg.example | 16 +++++++++ statustracker.py | 22 ++++++++++++ 13 files changed, 494 insertions(+) create mode 100644 .gitignore create mode 100755 jobmanager.py create mode 100755 jobs/EmailChecker.py create mode 100755 jobs/HTTPServerChecker.py create mode 100755 jobs/JobBase.py create mode 100755 jobs/JobSpawner.py create mode 100755 jobs/PeerChecker.py create mode 100755 jobs/TCPServerChecker.py create mode 100755 jobs/__init__.py create mode 100755 main.py create mode 100755 servers.py create mode 100755 settings.cfg.example create mode 100755 statustracker.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d0a4d6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +settings.cfg diff --git a/jobmanager.py b/jobmanager.py new file mode 100755 index 0000000..74327a7 --- /dev/null +++ b/jobmanager.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +import time +import logging +import requests + +from jobs import JobFinder + +class JobManager: + def __init__(self, config): + jobsFinder = JobFinder(config) + self.jobs = jobsFinder.get_jobs() + self.config = config + + def list_jobs(self): + return self.jobs + + def execute_jobs(self, cronmode): + logging.info("Executing jobs...") + success = True + for thisJob in self.jobs: + thisJob.setConfig(self.config) + if thisJob.shouldExecute(cronmode): + logging.info("Executing " + thisJob.getName()) + if not thisJob.execute(): + success = False + return success + + def mark_jobs_ran(self): + logging.debug("Marking jobs as run successfully.") + requests.post("http://localhost:5001/", data="True") + + def mark_jobs_ran_with_error(self): + logging.warning("Marking jobs as run unsuccessfully.") + requests.post("http://localhost:5001/", data="False") \ No newline at end of file diff --git a/jobs/EmailChecker.py b/jobs/EmailChecker.py new file mode 100755 index 0000000..51992bf --- /dev/null +++ b/jobs/EmailChecker.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +import os +import base64 +import datetime + +import imaplib + +import JobBase + +class EmailChecker(JobBase.JobBase): + def executeEvery(self): + return JobBase.JobFrequency.HOUR + def execute(self): + USER = self.config.get('email', 'user') + PASS = self.config.get('email', 'pass') + + #Generate a random subject + subj = base64.b64encode(os.urandom(20)) + + if not self.sendEmail(subj, "", USER): + return False + + M = imaplib.IMAP4_SSL(self.config.get('email', 'imapserver')) + M.login(USER, PASS) + + #If we have set up a filter to auto-delete messages from ourself + if self.config.get('email', 'ideletesentmessagesautomatically'): + M.select("[Gmail]/Trash") + + criteria = '(FROM "'+USER+'" SINCE "'+datetime.date.today().strftime("%d-%b-%Y")+'")' + typ, data = M.search(None, criteria) + + foundSubject = False + for num in data[0].split(): + typ, data = M.fetch(num, '(BODY.PEEK[HEADER.FIELDS (Subject)])') + if subj in data[0][1]: + foundSubject = True + M.close() + M.logout() + if not foundSubject: + #This may not work, but try anyway + self.sendEmail("Email Fetch Failure", "Body") + return False + else: + return True \ No newline at end of file diff --git a/jobs/HTTPServerChecker.py b/jobs/HTTPServerChecker.py new file mode 100755 index 0000000..ec8eda1 --- /dev/null +++ b/jobs/HTTPServerChecker.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +import logging +import requests + +import JobBase +import JobSpawner + +class HTTPServerChecker(JobSpawner.JobSpawner): + servers = [ + #("http://example.com", JobBase.JobFrequency.MINUTE), + #("https://exampletwo.com", JobBase.JobFrequency.MINUTE) + ] + + class ServerChecker(JobBase.JobBase): + def __init__(self, url, frequency): + self.url = url + self.frequency = frequency + + def getName(self): + return str(self.__class__) + " for " + self.url + def executeEvery(self): + return self.frequency + def execute(self): + try: + requests.get(self.url) + return True + except: + msg = "Could not hit server " + self.url + logging.warn(msg) + return self.sendEmail(msg, "") + + def get_sub_jobs(self): + for s in self.servers: + yield self.ServerChecker(s[0], s[1]) + diff --git a/jobs/JobBase.py b/jobs/JobBase.py new file mode 100755 index 0000000..330b6a9 --- /dev/null +++ b/jobs/JobBase.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import random +import logging + +import smtplib + +class JobFrequency: + MINUTE = "minute" + HOUR = "hour" + +class JobBase: + def __init__(self): + self.config = None + def getName(self): + return str(self.__class__) + def shouldExecute(self, cronmode): + frequency = self.executeEvery() + if cronmode == frequency: + return True + return False + def setConfig(self, config): + self.config = config + + def sendEmail(self, subject, body, to=""): + return sendEmail(self.config, subject, body, to) + + def executeEvery(self): + pass + def execute(self): + pass + +def sendEmail(config, subject, body, to=""): + FROM = config.get('email', 'user') + PASS = config.get('email', 'pass') + if not to: + to = config.get('alertcontact', 'default') + + # Prepare actual message + # Avoid gmail threading + subject = subject + " " + str(random.random()) + message = """\From: %s\nTo: %s\nSubject: %s\n\n%s""" \ + % (FROM, ", ".join(to), subject, body) + try: + server = smtplib.SMTP(config.get('email', 'smtpserver'), config.get('email', 'smtpport')) + server.ehlo() + server.starttls() + server.login(FROM, PASS) + server.sendmail(FROM, to, message) + server.close() + return True + except: + return False \ No newline at end of file diff --git a/jobs/JobSpawner.py b/jobs/JobSpawner.py new file mode 100755 index 0000000..3d09693 --- /dev/null +++ b/jobs/JobSpawner.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +class JobSpawner: + def get_sub_jobs(self): + pass diff --git a/jobs/PeerChecker.py b/jobs/PeerChecker.py new file mode 100755 index 0000000..c8cca38 --- /dev/null +++ b/jobs/PeerChecker.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import os +import base64 +import datetime + +import imaplib + +import JobBase + +class PeerChecker(JobBase.JobBase): + def executeEvery(self): + return JobBase.JobFrequency.HOUR + def execute(self): + testSuccess = True + peers = self.config.items('peers') + for p in peers: + peer = p[1].split(',') + peerOK = False + + try: + response = requests.get(peer[0]) + if response.status_code != 200: + peerOK = False + subject = peer[0] + " returned a non-standard status code." + else: + if "True" in response.content: + peerOK = True + elif "False" in response.content: + peerOK = False + subject = peer[0] + " reports it cannot send email." + except: + peerOK = False + subject = peer[0] + " is not responding." + + if not peerOK: + if not self.sendEmail(subject, "", peer[1]): + testSuccess = False + return testSuccess \ No newline at end of file diff --git a/jobs/TCPServerChecker.py b/jobs/TCPServerChecker.py new file mode 100755 index 0000000..711047b --- /dev/null +++ b/jobs/TCPServerChecker.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +import os +import socket +import logging + +import JobBase +import JobSpawner + +class TCPServerChecker(JobSpawner.JobSpawner): + servers = [ + #("example.com", 53, "example.com:tcpdns", JobBase.JobFrequency.MINUTE), + ] + + class ServerChecker(JobBase.JobBase): + def __init__(self, ip, port, friendlyName, frequency): + self.ip = ip + self.port = port + self.friendlyName = friendlyName + "(" + self.ip + ":" + str(self.port) + ")" + self.frequency = frequency + + def getName(self): + return str(self.__class__) + " for " + self.friendlyName + def executeEvery(self): + return self.frequency + def execute(self): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((self.ip, self.port)) + s.close() + return True + except: + msg = "Could not hit server " + self.friendlyName + logging.warn(msg) + return self.sendEmail(msg, "") + + def get_sub_jobs(self): + for s in self.servers: + yield self.ServerChecker(s[0], s[1], s[2], s[3]) + + diff --git a/jobs/__init__.py b/jobs/__init__.py new file mode 100755 index 0000000..9955164 --- /dev/null +++ b/jobs/__init__.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +import os +import sys +import inspect +import logging +from imp import load_module, find_module +import importlib + +import jobs +import jobs.JobBase +import jobs.JobSpawner + +class JobFinder: + def __init__(self, config): + """ + Opens the jobs folder and looks at every .py module in that directory. + Finds available jobs by looking at any class defined in those modules + that implements the JobBase abstract class. + Returns a list of job classes. + """ + self._jobs = set([]) + self.config = config + + job_modules = self.get_job_modules_dynamic() + + for module in job_modules: + # Check every declaration in that module + for name in dir(module): + obj = getattr(module, name) + if name not in module.__name__: + # Jobs have to have the same class name as their module name + # This prevents Job B from being detected twice when there is a Job A that imports Job B + continue + + if inspect.isclass(obj): + # A class declaration was found in that module + # Checking if it's a subclass of JobBase + # Discarding JobBase as a subclass of JobBase + if obj != jobs.JobBase.JobBase and obj != jobs.JobSpawner.JobSpawner: + logging.info("Found " + str(obj)) + for base in obj.__bases__: + # H4ck because issubclass() doesn't seem to work as expected on Linux + # It has to do with JobBase being imported multiple times (within jobs) or something + if base.__name__ == 'JobBase': + # A job was found, keep it + self._jobs.add(obj()) + elif base.__name__ == 'JobSpawner': + spawner = obj() + for j in spawner.get_sub_jobs(): + self._jobs.add(j) + + + def get_job_modules_dynamic(self): + job_modules = [] + + job_dir = jobs.__path__[0] + full_job_dir = os.path.join(sys.path[0], job_dir) + if os.path.exists(full_job_dir): + for (root, dirs, files) in os.walk(full_job_dir): + del dirs[:] # Do not walk into subfolders of the job directory + # Checking every .py module in the job directory + jobs_loaded = [] + for source in (s for s in files if s.endswith((".py"))): + module_name = os.path.splitext(os.path.basename(source))[0] + if module_name in jobs_loaded: + continue + jobs_loaded.append(module_name) + full_name = os.path.splitext(source)[0].replace(os.path.sep,'.') + + try: # Try to import the job package + # The job package HAS to be imported as a submodule + # of module 'jobs' or it will break windows compatibility + (file, pathname, description) = \ + find_module(full_name, jobs.__path__) + module = load_module('jobs.' + full_name, file, + pathname, description) + except Exception as e: + logging.critical('Import Error on ' + module_name + ': ' + str(e)) + jobs.JobBase.sendEmail(self.config, 'Import Error on ' + module_name, str(e)) + continue + job_modules.append(module) + return job_modules + + def get_jobs(self): + return self._jobs \ No newline at end of file diff --git a/main.py b/main.py new file mode 100755 index 0000000..d9404b5 --- /dev/null +++ b/main.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +import os +import sys +import json +import pickle +import hashlib +import logging +import argparse +import binascii +import ConfigParser + +import requests + +from twisted.internet import reactor, ssl +from twisted.web import server + +from jobmanager import JobManager +from statustracker import StatusTracker +from servers import StatusSite, PingSite + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Check your stuff.") + parser.add_argument('-m', '--mode', choices=['daemon', 'cron'], required=True, help='The mode the application will run it.') + parser.add_argument('-c', '--crontime', choices=['minute', 'hour'], help='When in cron mode, the increment of cron.') + parser.add_argument('-v', action="store_true", help="Print verbose debugging information to stderr") + + args = parser.parse_args() + + config = ConfigParser.ConfigParser() + config.read('settings.cfg') + if not config.get('email', 'user') or \ + not config.get('email', 'pass') or \ + not config.get('email', 'smtpserver') or \ + not config.get('email', 'smtpport') or \ + not config.get('email', 'imapserver'): + print "Sending email address is not configured" + sys.exit(1) + if not config.get('alertcontact', 'default'): + print "Default alert contact is not configured" + sys.exit(1) + + + + requests_log = logging.getLogger("requests.packages.urllib3") + requests_log.setLevel(logging.CRITICAL) + logging.basicConfig(format="%(asctime)s:%(levelname)s: %(message)s") + log = logging.getLogger() + log.setLevel(logging.DEBUG) + if args.v: + log.setLevel(logging.DEBUG) + + if args.mode == 'daemon': + log.info("Starting up daemon") + statusTracker = StatusTracker(config) + reactor.listenTCP(5000, server.Site(StatusSite(statusTracker))) + reactor.listenTCP(5001, server.Site(PingSite(statusTracker)), interface='127.0.0.1') + reactor.run() + elif args.mode == 'cron': + jobManager = JobManager(config) + if not args.crontime: + log.warn("Did not run cron, no crontime specified") + parser.print_help() + sys.exit(-1) + else: + log.info("Running cron at frequency " + args.crontime) + if jobManager.execute_jobs(args.crontime): + jobManager.mark_jobs_ran() + else: + jobManager.mark_jobs_ran_with_error() diff --git a/servers.py b/servers.py new file mode 100755 index 0000000..207d4f1 --- /dev/null +++ b/servers.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +import logging + +from twisted.python.filepath import FilePath +from twisted.web import server, resource, http + +class StatusSite(resource.Resource): + isLeaf = True + def __init__(self, statusTracker): + resource.Resource.__init__(self) + self.statusTracker = statusTracker + def render_GET(self, request): + if self.statusTracker.isAllGood(): + logging.debug("Indicating that everything seems to be okay") + s = "True" + else: + logging.warn("Indicating that everything does not seem to be okay") + s = "False" + + request.setResponseCode(200) + return s + +class PingSite(resource.Resource): + isLeaf = True + def __init__(self, statusTracker): + resource.Resource.__init__(self) + self.statusTracker = statusTracker + def render_POST(self, request): + self.statusTracker.markJobRan() + emailStatus = request.content.read() + emailStatus = "True" in emailStatus + + logging.debug("Got notification of jobs ran") + if emailStatus: + logging.debug("Email is working") + else: + logging.warn("Email is _not_ working") + + self.statusTracker.markEmailStatus(emailStatus) + request.setResponseCode(200) + return "OK" + diff --git a/settings.cfg.example b/settings.cfg.example new file mode 100755 index 0000000..2dd8695 --- /dev/null +++ b/settings.cfg.example @@ -0,0 +1,16 @@ +[alertcontact] +default=youremail@example.com + +[email] +user=agmailaccountyoucreate@gmail.com +pass=yourpassword +smtpserver=smtp.gmail.com +smtpport=587 +imapserver=imap.gmail.com +#I create a filter that automatically deletes my sent messages. +# This way, anyone who hacks the account only sees the last 30 days of messages I've sent +ideletesentmessagesautomatically=True + +[peers] +peer1=http://someserver.com:5000,admin@someserverbackup.com +peer2=http://someotherserver.com:5000,admin@someotherserver.com \ No newline at end of file diff --git a/statustracker.py b/statustracker.py new file mode 100755 index 0000000..2c3350c --- /dev/null +++ b/statustracker.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +import time +import logging + +class StatusTracker: + emailNotificationsAreWorking = False + lastRunJob = 0 + def __init__(self, config): + self.emailNotificationsAreWorking = False + self.lastRunJob = 0 + self.config = config + + def isAllGood(self): + return self.emailNotificationsAreWorking and \ + time.time() - self.lastRunJob < 120 + + def markJobRan(self): + self.lastRunJob = time.time() + + def markEmailStatus(self, working): + self.emailNotificationsAreWorking = working \ No newline at end of file -- cgit v1.2.3