viona: reset device more completely

lib/propolis/src/hw/virtio/pci.rs

@@ -1098,6 +1098,25 @@ impl PciVirtioState {
         }
     }
 
+    /// Reset all non-control queues as part of a device reset (or shutdown).
+    pub fn reset_queues(&self, dev: &dyn VirtioDevice) {
+        let mut state = self.state.lock().unwrap();
+        self.reset_queues_locked(dev, &mut state);
+    }
+
+    fn reset_queues_locked(
+        &self,
+        dev: &dyn VirtioDevice,
+        state: &mut MutexGuard<VirtioState>,
+    ) {
+        for queue in self.queues.iter_all() {
+            queue.reset();
+            if dev.queue_change(queue, VqChange::Reset).is_err() {
+                self.needs_reset_locked(dev, state);
+            }
+        }
+    }
+
     /// Reset the virtio portion of the device
     ///
     /// This leaves PCI state (such as configured BARs) unchanged
@@ -1106,12 +1125,7 @@ impl PciVirtioState {
         dev: &dyn VirtioDevice,
         mut state: MutexGuard<VirtioState>,
     ) {
-        for queue in self.queues.iter() {
-            queue.reset();
-            if dev.queue_change(queue, VqChange::Reset).is_err() {
-                self.needs_reset_locked(dev, &mut state);
-            }
-        }
+        self.reset_queues_locked(dev, &mut state);
         state.reset();
         let _ = self.isr_state.read_clear();
     }

lib/propolis/src/hw/virtio/queue.rs

@@ -1005,6 +1005,15 @@ impl VirtQueues {
         self.queues[..len].iter().chain([self.get_control()])
     }
 
+    /// Iterate all queues, regardless of the device's current configuration
+    /// happening to use them or not.
+    ///
+    /// This is primarily useful for operations like device reset and teardown
+    /// where we need to manage all *possible* device state.
+    pub fn iter_all(&self) -> impl std::iter::Iterator<Item = &Arc<VirtQueue>> {
+        self.queues.iter()
+    }
+
     pub fn export(&self) -> migrate::VirtQueuesV1 {
         let len = self.len() as u64;
         let queues = self.queues.iter().map(|q| q.export()).collect();

lib/propolis/src/hw/virtio/viona.rs

@@ -642,25 +642,7 @@ impl PciVirtioViona {
 
     /// Make sure all in-kernel virtqueue processing is stopped
     fn queues_kill(&self) {
-        let mut inner = self.inner.lock().unwrap();
-        for vq in self.virtio_state.queues.iter() {
-            let rs = inner.for_vq(vq);
-            match *rs {
-                VRingState::Init => {
-                    // Already at rest
-                }
-                VRingState::Fatal => {
-                    // No sense in attempting a reset
-                }
-                _ => {
-                    if self.hdl.ring_reset(vq).is_err() {
-                        *rs = VRingState::Fatal;
-                    } else {
-                        *rs = VRingState::Init;
-                    }
-                }
-            }
-        }
+        self.virtio_state.reset_queues(self);
     }
 
     fn poller_start(&self) {
@@ -826,6 +808,8 @@ impl Lifecycle for PciVirtioViona {
     }
     fn reset(&self) {
         self.virtio_state.reset(self);
+        self.set_use_pairs(1).expect("can set viona back to one queue pair");
+        self.hdl.set_pairs(1).expect("can set viona back to one queue pair");
     }
     fn start(&self) -> anyhow::Result<()> {
         self.run();