From c69fd6ca6190ff5c76be3504a2bb5235ec5a20ee Mon Sep 17 00:00:00 2001 From: Philip Colmer Date: Sat, 1 Feb 2014 11:48:57 +0000 Subject: Changed starting up delay from literal count to timed delay to make it more accurate. Added log count to rolling logs. --- healthcheck.py | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/healthcheck.py b/healthcheck.py index e1661ca..db900ae 100644 --- a/healthcheck.py +++ b/healthcheck.py @@ -16,7 +16,7 @@ class HealthCheck(object): self.last_response = prev_response self.last_address = prev_addr self.state_process = None - self.startingup_countdown = 0 + self.startingup_time = -1 self.logged_message = "" def silentremove(self, filename): @@ -215,24 +215,27 @@ class HealthCheck(object): except Exception,e: self.logmsg("Got exception trying to save service IP: %s" % str(e)) - # Set the starting up count - we won't run the change state script until - # the count reaches 0. Since the script hasn't run, we'll stay in StartingUp - # because we can't switch to Active until monit (or whatever) detects the service - # as actually running. - if (new_state == States.StartingUp and self.last_state != States.StartingUp): - self.startingup_countdown = 10 - # Override last state in order to prevent the state change script from - # running - self.last_state = States.StartingUp - self.logmsg("Holding start up for %s cycles" % str(self.startingup_countdown)) - elif (new_state == States.StartingUp and self.startingup_countdown > 0): - # Decrement the countdown - if we reach zero, switch last state to Passive - # to trigger the state change script - self.startingup_countdown -= 1 - if (self.startingup_countdown == 0): - self.last_state = States.Passive - else: - self.logmsg("Holding start up for %s cycles" % str(self.startingup_countdown)) + # Set the starting up time - we won't run the change state script until + # the elapsed time is more than 30. Since the script hasn't run, we'll stay + # in StartingUp because we can't switch to Active until monit (or whatever) + # detects the service as actually running. + if (new_state == States.StartingUp): + if (self.last_state != States.StartingUp): + self.startingup_time = time.time() + # Override last state in order to prevent the state change script from + # running + self.last_state = States.StartingUp + self.logmsg("Holding start up for 30 seconds") + elif (self.startingup_time != -1): + duration = time.time() - self.startingup_time + # If we've reached the desired duration, switch last state to Passive + # to trigger the state change script and reset the starting up time + # so that we don't keep on switching to the passive state. + if (duration > 30): + self.last_state = States.Passive + self.startingup_time = -1 + else: + self.logmsg("Holding start up; duration is now %s" % str(duration)) # See if a script exists for one of the supported state changes if (new_state != self.last_state): @@ -365,7 +368,7 @@ def main(): # Make a new RotatingFileHandler for the error log. fname = getattr(logscope, "rot_error_file", "%s/error.log" % script_dir) - h = handlers.TimedRotatingFileHandler(fname, when='midnight') + h = handlers.TimedRotatingFileHandler(fname, when='midnight', backupCount=7) h.setLevel(logging.DEBUG) h.setFormatter(_cplogging.logfmt) logscope.error_file = "" @@ -373,7 +376,7 @@ def main(): # Make a new RotatingFileHandler for the access log. fname = getattr(logscope, "rot_access_file", "%s/access.log" % script_dir) - h = handlers.TimedRotatingFileHandler(fname, when='midnight') + h = handlers.TimedRotatingFileHandler(fname, when='midnight', backupCount=7) h.setLevel(logging.DEBUG) h.setFormatter(_cplogging.logfmt) logscope.access_file = "" -- cgit v1.2.3