Add error-handling to create_vlan

Re-order slightly, and work out roll-back support

If all changes worked ok on all switches, then *and only then* save
config on all of them. Otherwise, we'll restart all the switches
that *did* work after a failure, to reset their configurations back to
that before we started. It's a nuclear and slow path, but it should
work.

Change-Id: Ic6008ae9cf65482671be2c40cc8d33b6b346c16c

1 files changed, 28 insertions, 5 deletions

diff --git a/util.py b/util.py
index c24f564..1d0c1e9 100644
--- a/util.py
+++ b/util.py
@@ -195,7 +195,8 @@ class VlanUtil:
     #   b. Add the VLAN to all trunk ports (if needed)
     #
     # The VLAN may already exist on some of the switches, that's
-    # fine.
+    # fine. If things fail, we attempt to roll back by rebooting
+    # switches then removing the VLAN in the DB.
     def create_vlan(self, state, command, data):
 
         print 'create_vlan'
@@ -215,6 +216,9 @@ class VlanUtil:
             print 'DB creation failed'
             raise
 
+        # Keep track of which switches we've configured, for later use
+        switches_done = []
+
         # 2. Now the switches
         try:
             for switch in sorted(config.switches):
@@ -225,6 +229,10 @@ class VlanUtil:
                     s = self.get_switch_driver(switch, config)
                     s.switch_connect(config.switches[switch].username, config.switches[switch].password)
 
+                    # Mark this switch as one we've touched, for
+                    # either config saving or rollback below
+                    switches_done.append(switch)
+
                     # 2a. Create the VLAN on the switch
                     s.vlan_create(tag)
                     s.vlan_set_name(tag, name)
@@ -248,9 +256,6 @@ class VlanUtil:
                         print 'Added VLAN tag %d, name %s to switch %s' % (tag, name, switch)
                         s.port_add_trunk_to_vlan(port, tag)
 
-                    # Save switch config so it will stay across reboots
-                    s.switch_save_running_config()
-
                     # And now we're done with this switch
                     s.switch_disconnect()
                     del s
@@ -259,11 +264,29 @@ class VlanUtil:
                     raise
         
         except IOError:
-            # Bugger. Looks like one of the switch calls above failed.
+            # Bugger. Looks like one of the switch calls above
+            # failed. To undo the changes safely, we'll need to reset
+            # all the switches we managed to configure. This could
+            # take some time!
+            for switch in switches_done:
+                s = self.get_switch_driver(switch, config)
+                s.switch_connect(config.switches[switch].username, config.switches[switch].password)
+                s.switch_restart() # Will implicitly also close the connection
+                del s
+
             # Undo the database change
             print 'Switch access failed. Deleting the new VLAN entry in the database'
             db.delete_vlan(vlan_id)
             raise
 
+        # If we've got this far, things were successful. Save config
+        # on all the switches so it will persist across reboots
+        for switch in switches_done
+            s = self.get_switch_driver(switch, config)
+            s.switch_connect(config.switches[switch].username, config.switches[switch].password)
+            s.switch_save_running_config()
+            s.switch_disconnect()
+            del s
+
         return vlan_id # If we're successful