Reduce singifincantly the amount of tx interrupts

Instead of enabling interrupts for tx by the host when we have
a single used pkt in the ring, wait until we have 1/2 ring.
This improves the amount of tx irqs from one per pkt to practically
zero (note that we actively call tx_gc if there is not place on the
ring when doing tx). There was a 40% performance boost on the
netperf rx test

drivers/virtio-net.cc

@@ -214,7 +214,7 @@ namespace virtio {
         while (1) {
 
             // Wait for rx queue (used elements)
-            virtio_driver::wait_for_queue(_rx_queue);
+            virtio_driver::wait_for_queue(_rx_queue, &vring::used_ring_not_empty);
             trace_virtio_net_rx_wake();
 
             u32 len;
@@ -357,7 +357,7 @@ namespace virtio {
 
         while (1) {
             // Wait for tx queue (used elements)
-            virtio_driver::wait_for_queue(_tx_queue);
+            virtio_driver::wait_for_queue(_tx_queue, &vring::used_ring_is_half_empty);
             trace_virtio_net_tx_wake();
             tx_gc();
         }

drivers/virtio-vring.cc

@@ -218,11 +218,16 @@ namespace virtio {
         return (_avail_count >= descriptors);
     }
 
-    bool vring::used_ring_not_empty()
+    bool vring::used_ring_not_empty() const
     {
         return (_used_guest_head != _used->_idx.load(std::memory_order_relaxed));
     }
 
+    bool vring::used_ring_is_half_empty() const
+        {
+            return (_used_guest_head - _used->_idx.load(std::memory_order_relaxed) > (u16)_num/2);
+        }
+
     bool
     vring::kick() {
         bool kicked = true;

drivers/virtio-vring.hh

@@ -118,7 +118,8 @@ class virtio_driver;
         // Ring operations
         bool add_buf(sglist* sg, u16 out, u16 in, void* cookie);
         void* get_buf(u32 *len);
-        bool used_ring_not_empty();
+        bool used_ring_not_empty() const;
+        bool used_ring_is_half_empty() const;
         bool avail_ring_not_empty();
         // when the available ring has x descriptors as room it means that
         // x descriptors can be allocated while _num-x are available for the host

drivers/virtio.cc

@@ -3,9 +3,12 @@
 #include "drivers/virtio.hh"
 #include "virtio-vring.hh"
 #include <osv/debug.h>
+#include "osv/trace.hh"
 
 using namespace pci;
 
+TRACEPOINT(trace_virtio_wait_for_queue, "queue(%p) have_elements=%d", void*, int);
+
 namespace virtio {
 
 virtio_driver::virtio_driver(pci::device& dev)
@@ -187,22 +190,24 @@ vring* virtio_driver::get_virt_queue(unsigned idx)
     return (_queues[idx]);
 }
 
-void virtio_driver::wait_for_queue(vring* queue)
+void virtio_driver::wait_for_queue(vring* queue, bool (vring::*pred)() const)
 {
-    sched::thread::wait_until([queue] {
-        bool have_elements = queue->used_ring_not_empty();
+    sched::thread::wait_until([queue,pred] {
+        bool have_elements = (queue->*pred)();
         if (!have_elements) {
             queue->enable_interrupts();
 
             // we must check that the ring is not empty *after*
             // we enable interrupts to avoid a race where a packet
             // may have been delivered between queue->used_ring_not_empty()
             // and queue->enable_interrupts() above
-            have_elements = queue->used_ring_not_empty();
+            have_elements = (queue->*pred)();
             if (have_elements) {
                 queue->disable_interrupts();
             }
         }
+
+        trace_virtio_wait_for_queue(queue, have_elements);
         return have_elements;
     });
 }

drivers/virtio.hh

@@ -115,7 +115,7 @@ public:
     vring* get_virt_queue(unsigned idx);
 
     // block the calling thread until the queue has some used elements in it.
-    void wait_for_queue(vring* queue);
+    void wait_for_queue(vring* queue, bool (vring::*pred)() const);
 
     // guest/host features physical access
     u32 get_device_features(void);