Changed starting up delay from literal count to timed delay to make it more accurate.HEADmaster

Added log count to rolling logs.

1 files changed, 24 insertions, 21 deletions

diff --git a/healthcheck.py b/healthcheck.py
index e1661ca..db900ae 100644
--- a/healthcheck.py
+++ b/healthcheck.py
@@ -16,7 +16,7 @@ class HealthCheck(object):
         self.last_response = prev_response
         self.last_address = prev_addr
         self.state_process = None
-        self.startingup_countdown = 0
+        self.startingup_time = -1
         self.logged_message = ""
 
     def silentremove(self, filename):
@@ -215,24 +215,27 @@ class HealthCheck(object):
         except Exception,e:
             self.logmsg("Got exception trying to save service IP: %s" % str(e))
 
-        # Set the starting up count - we won't run the change state script until
-        # the count reaches 0. Since the script hasn't run, we'll stay in StartingUp
-        # because we can't switch to Active until monit (or whatever) detects the service
-        # as actually running.
-        if (new_state == States.StartingUp and self.last_state != States.StartingUp):
-            self.startingup_countdown = 10
-            # Override last state in order to prevent the state change script from
-            # running
-            self.last_state = States.StartingUp
-            self.logmsg("Holding start up for %s cycles" % str(self.startingup_countdown))
-        elif (new_state == States.StartingUp and self.startingup_countdown > 0):
-            # Decrement the countdown - if we reach zero, switch last state to Passive
-            # to trigger the state change script
-            self.startingup_countdown -= 1
-            if (self.startingup_countdown == 0):
-                self.last_state = States.Passive
-            else:
-                self.logmsg("Holding start up for %s cycles" % str(self.startingup_countdown))
+        # Set the starting up time - we won't run the change state script until
+        # the elapsed time is more than 30. Since the script hasn't run, we'll stay
+        # in StartingUp because we can't switch to Active until monit (or whatever)
+        # detects the service as actually running.
+        if (new_state == States.StartingUp):
+            if (self.last_state != States.StartingUp):
+                self.startingup_time = time.time()
+                # Override last state in order to prevent the state change script from
+                # running
+                self.last_state = States.StartingUp
+                self.logmsg("Holding start up for 30 seconds")
+            elif (self.startingup_time != -1):
+                duration = time.time() - self.startingup_time
+                # If we've reached the desired duration, switch last state to Passive
+                # to trigger the state change script and reset the starting up time
+                # so that we don't keep on switching to the passive state.
+                if (duration > 30):
+                    self.last_state = States.Passive
+                    self.startingup_time = -1
+                else:
+                    self.logmsg("Holding start up; duration is now %s" % str(duration))
 
         # See if a script exists for one of the supported state changes
         if (new_state != self.last_state):
@@ -365,7 +368,7 @@ def main():
 
     # Make a new RotatingFileHandler for the error log.
     fname = getattr(logscope, "rot_error_file", "%s/error.log" % script_dir)
-    h = handlers.TimedRotatingFileHandler(fname, when='midnight')
+    h = handlers.TimedRotatingFileHandler(fname, when='midnight', backupCount=7)
     h.setLevel(logging.DEBUG)
     h.setFormatter(_cplogging.logfmt)
     logscope.error_file = ""
@@ -373,7 +376,7 @@ def main():
 
     # Make a new RotatingFileHandler for the access log.
     fname = getattr(logscope, "rot_access_file", "%s/access.log" % script_dir)
-    h = handlers.TimedRotatingFileHandler(fname, when='midnight')
+    h = handlers.TimedRotatingFileHandler(fname, when='midnight', backupCount=7)
     h.setLevel(logging.DEBUG)
     h.setFormatter(_cplogging.logfmt)
     logscope.access_file = ""