Changeset 29

Timestamp:

27/11/07 17:40:17 (2 years ago)

Author:

daedalus

Message:

* Manual bailout when in authoritarian mode now works correctly.
* Fixed up some of the log level and stats reporting.
* Added 'backout' mode, that will only run the backout portion of changes. More

useful for testing, but can also be used to undo provisioning by repurposing
the same set of templates used for provisioning.

Location:

trunk

Files:

• confloader.py (modified) (11 diffs)
• etc/netapp-create-volume.change-template.xml (added)
• etc/templating.xml (added)
• options.py (modified) (1 diff)
• provisioner.py (modified) (9 diffs)

trunk/confloader.py

@@ -14 +14 @@
 
 from device import Device
-from change import CHANGE_STATE
+from change import CHANGE_STATE, ChangeConditionFailure
+from provisioner import UserBailout
 import util
 
@@ -186 +187 @@
             self.add_namespace(nsnode)
         except IndexError:
-            log.warn("Cannot find global namespace")
             self.global_namespace = {}
 
@@ -604 +604 @@
                     if prereq not in self.change_success:
 
+                        # If we're in backout mode, a prereq in 'backout_ok' state
+                        # is treated as complete.
+                        if prereq.state in [ CHANGE_STATE['backout_ok'], ]:
+                            continue
+
                         # If a prereq has failed, doesn't need to retry, and is
                         # marked as 'onfail:continue', then we treat it as if
                         # this prereq has been met.
-                        if prereq.state not in [ CHANGE_STATE['pending'],
+                        elif prereq.state not in [ CHANGE_STATE['pending'],
                                                  CHANGE_STATE['retry'],
                                                  ] and prereq.on_fail_continue:
@@ -631 +636 @@
         Move a change from pending to complete.
         """
-        log.debug("Change completed.")
+        log.info("Change %s completed.", change.name)
         if change.state == CHANGE_STATE['success']:
             self.change_success.append(change)
@@ -657 +662 @@
             # Change will be retried
             pass
-        
+
         else:
             log.error("Unknown/unhandled change state '%s'", change.state)
@@ -703 +708 @@
 
         # deal with errors in stats
-        #self.alldone.addErrback(self.stats_error)
+        #self.alldone.addErrback(self.print_stats)
                                 
         return self.alldone
 
-    def get_next_changes(self, ignored):
+    def get_next_changes(self, results):
         """
         Fetch the next set of changes to run.
@@ -716 +721 @@
         have no interdependencies on one another.
         """
+        log.debug("Results in get_next_changes are: %s", results)
         log.debug("Fetching outstanding changes...")
         self.current_changelist = self.cfgldr.get_available_changes()
@@ -747 +753 @@
             log.debug("adding change '%s' to implementation queue", change.name)
                 
-            d = change.provisioner.perform_change(None, change, self.cfgldr.global_namespace)
+            d = change.provisioner.perform_change(None, change, self.cfgldr.global_namespace, backout=self.cfgldr.options.backout)
             d.addCallback(self.change_complete, change)
             d.addErrback(self.change_failure, change)
@@ -753 +759 @@
             pass
 
-        dl = defer.DeferredList(dlist)
+        dl = defer.DeferredList(dlist, fireOnOneErrback=True, consumeErrors=True)
         # After the current crop of changes has been completed, find if there are more
         dl.addCallback( self.get_next_changes )
+        dl.addErrback( self.bailout )
         return dl
 
+    def bailout(self, failure):
+        """
+        Bailout detects the first error from the deferred list,
+        and bails out.
+        """
+        log.debug("Running bailout...")
+        if failure.value.subFailure.type == UserBailout:
+            self.alldone.callback('Bailout')
+
+        else:
+            log.debug("Unhandled failure: %s", failure)
+            self.alldone.errback(failure)
+
     def change_complete(self, ignored, change):
         """
@@ -765 +785 @@
 
     def change_failure(self, failure, change):
-        log.error("Major change failure!")
-        tlog.err(failure)
+        if failure.type == UserBailout:
+            log.debug("Controller detected UserBailout.")
+            return failure
+
+        elif failure.type == ChangeConditionFailure:
+            pass
+
+        else:
+            log.error("Major change failure!")
+            tlog.err(failure)
+            pass
+        
         self.change_complete(failure, change)
 
@@ -780 +810 @@
 
         log.info("--- Final results ---")
-        
-        if not (success_count > 0 or failure_count > 0):
-            log.error("no changes succeeded or failed!")
-        else:
-            log.info("%d ok, %d failed (%d%% success rate)" % (success_count, failure_count, 100 * success_count / (success_count+failure_count) ) )
-
-        if success_count > 0:
-            log.info( "Successful changes: %s" % ', '.join([ x.name for x in self.cfgldr.change_success]))
-
-        if failure_count > 0:
-            log.info( "Failed changes: %s" % ', '.join([ x.name for x in self.cfgldr.change_failure]) )
+
+        # Don't print some stats when in backout mode
+        if not self.cfgldr.options.backout:
+        
+            if not (success_count > 0 or failure_count > 0):
+                log.error("no changes succeeded or failed!")
+            else:
+                log.info("%d ok, %d failed (%d%% success rate)" % (success_count, failure_count, 100 * success_count / (success_count+failure_count) ) )
+
+            if success_count > 0:
+                log.info( "Successful changes: %s" % ', '.join([ x.name for x in self.cfgldr.change_success]))
+
+            if failure_count > 0:
+                log.info( "Failed changes: %s" % ', '.join([ x.name for x in self.cfgldr.change_failure]) )
+                pass
+            pass
+
+        # These stats are always printed
 
         if backout_success_count > 0 or backout_failure_count > 0:
-            log.error( "%d backed out ok, %d failed (%d%% backout success rate)" % (backout_success_count, backout_failure_count, 100 * backout_success_count / (backout_success_count+backout_failure_count) ))
+            log.info( "%d backed out ok, %d failed (%d%% backout success rate)" % (backout_success_count, backout_failure_count, 100 * backout_success_count / (backout_success_count+backout_failure_count) ))
             
         if backout_success_count > 0:

trunk/options.py

@@ -79 +79 @@
         help_configfile = "The configuration file to load"
         help_loadonly = "Load the configuration file and exit. Used to test parsing."
-        
+        help_backout = "Run the backout portion of the changes."
 
         self.add_option('-a', '--authoritarian',  dest='authoritarian', action='store_true', default=False, help=help_authoritarian)        
+        self.add_option('-b', '--backout',  dest='backout', action='store_true', default=False, help=help_backout)        
         self.add_option('-c', '--configfile', dest='configfile', type='string', help=help_configfile)
         self.add_option('', '--loadonly', dest='loadonly', action='store_true', default=False, help=help_loadonly)

trunk/provisioner.py

@@ -80 +80 @@
         pass
 
-    def perform_change(self, ignored, change, namespace={}):
+    def perform_change(self, ignored, change, namespace={}, backout=False):
         """
         Perform a change on one or more (potentially) remote entities.
@@ -120 +120 @@
                 namespace.update(device.namespace)
 
-                log.debug("applying change with namespace: %s", namespace)
-                d.addCallback(self.apply_change, device, change, namespace)
-
-                d.addCallback(self.change_complete_success, change, namespace)
-                d.addErrback(self.change_failure, change, namespace)
+                if backout:
+                    # Just do the backout portion
+                    d.addCallback(self.backout_change, device, change, namespace)
+
+                else:
+                    log.debug("applying change with namespace: %s", namespace)
+                    d.addCallback(self.apply_change, device, change, namespace)
+                    
+                    d.addCallback(self.change_complete_success, change, namespace)
+                    d.addErrback(self.change_failure, change, namespace)
+                    pass
                 pass
             pass
@@ -158 +164 @@
         e = failure.check( ChangeConditionFailure, UserBailout )
         if e:
-            log.error("  failure was: %s", failure.value )
-            if isinstance(e, UserBailout):
-                log.info("User bailout detected.")
-                return defer.succeed('bailout')
+            log.error("  failure was: %s, %s", failure.type, failure.value )
+
+            # If we want to bail out, bail now
+            if failure.type == UserBailout:
+                return defer.fail(failure)
         else:
+            log.error("Unhandled failure in provisioner.change_apply_failed()")
             tlog.err(failure)
         
@@ -193 +201 @@
                 change.state = CHANGE_STATE['total_failure']
         else:
-            change.state = CHANGE_STATE['backout_failed']
-            pass
+            log.error("change_failure unhandled change state: %s", change.state)
+            raise ValueError("Invalid change state: %s" % change.state)
+
+        # Propogate a UserBailout failure
+        if failure.type == UserBailout:
+            return failure
+        
         #tlog.err(failure)
 
@@ -202 +215 @@
             log.info("Change '%s' was a success!", change.name)
         else:
+            log.error("change_complete_success in weird state: %s", change.state)
+            raise ValueError("wrong change state: %s" % change.state)
             change.state = CHANGE_STATE['backout_ok']
             log.info("Change '%s' backed out successfully", change.name)
@@ -241 +256 @@
     def backout_success(self, ignored, device, change, namespace):
         log.info("Successfully backed out change: '%s' from device '%s'", change.name, device)
+        change.state = CHANGE_STATE['backout_ok']
         self.backout_ok[device] = change
 
     def backout_failure(self, failure, device, change, namespace):
         log.critical("Backout of change '%s' failed for device '%s'!", change.name, device)
+        change.state = CHANGE_STATE['backout_failed']
         self.backout_failed[device] = change
-        tlog.err(failure)
+        #tlog.err(failure)
         return failure
 
@@ -611 +628 @@
             pass
 
-        d.addCallback(self.all_commands_done)
-        
+        d.addCallbacks(self.all_commands_done, self.all_commands_failure)
         return self.all_commands_defer
 
@@ -662 +678 @@
         log.error("Command failed: %s", errorstr)
         #self.all_commands_defer.errback( Exception(errorstr) )
-
+        return failure
+    
     def all_commands_done(self, result):
         """
@@ -669 +686 @@
         log.debug("All commands have finished.")
         self.all_commands_defer.callback( (self.exitcode, self.cmdoutput) )
+
+    def all_commands_failure(self, failure):
+        """
+        Some kind of failure in the commands occurred.
+        """
+        e = failure.check( UserBailout )
+        if e:
+            #log.error("User bailout detected.")
+            self.all_commands_defer.errback( failure )
 
 class CommandFailure(Exception):