diff options
Diffstat (limited to 'hw/net/virtio-net.c')
| -rw-r--r-- | hw/net/virtio-net.c | 387 |
1 files changed, 348 insertions, 39 deletions
diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c index b7f3d1b2eb..aff67a92df 100644 --- a/hw/net/virtio-net.c +++ b/hw/net/virtio-net.c @@ -42,6 +42,7 @@ #include "trace.h" #include "monitor/qdev.h" #include "hw/pci/pci.h" +#include "net_rx_pkt.h" #define VIRTIO_NET_VM_VERSION 11 @@ -77,25 +78,15 @@ tso/gso/gro 'off'. */ #define VIRTIO_NET_RSC_DEFAULT_INTERVAL 300000 -/* temporary until standard header include it */ -#if !defined(VIRTIO_NET_HDR_F_RSC_INFO) - -#define VIRTIO_NET_HDR_F_RSC_INFO 4 /* rsc_ext data in csum_ fields */ -#define VIRTIO_NET_F_RSC_EXT 61 - -#endif - -static inline __virtio16 *virtio_net_rsc_ext_num_packets( - struct virtio_net_hdr *hdr) -{ - return &hdr->csum_start; -} - -static inline __virtio16 *virtio_net_rsc_ext_num_dupacks( - struct virtio_net_hdr *hdr) -{ - return &hdr->csum_offset; -} +#define VIRTIO_NET_RSS_SUPPORTED_HASHES (VIRTIO_NET_RSS_HASH_TYPE_IPv4 | \ + VIRTIO_NET_RSS_HASH_TYPE_TCPv4 | \ + VIRTIO_NET_RSS_HASH_TYPE_UDPv4 | \ + VIRTIO_NET_RSS_HASH_TYPE_IPv6 | \ + VIRTIO_NET_RSS_HASH_TYPE_TCPv6 | \ + VIRTIO_NET_RSS_HASH_TYPE_UDPv6 | \ + VIRTIO_NET_RSS_HASH_TYPE_IP_EX | \ + VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | \ + VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) static VirtIOFeature feature_sizes[] = { {.flags = 1ULL << VIRTIO_NET_F_MAC, @@ -108,6 +99,8 @@ static VirtIOFeature feature_sizes[] = { .end = endof(struct virtio_net_config, mtu)}, {.flags = 1ULL << VIRTIO_NET_F_SPEED_DUPLEX, .end = endof(struct virtio_net_config, duplex)}, + {.flags = (1ULL << VIRTIO_NET_F_RSS) | (1ULL << VIRTIO_NET_F_HASH_REPORT), + .end = endof(struct virtio_net_config, supported_hash_types)}, {} }; @@ -138,6 +131,12 @@ static void virtio_net_get_config(VirtIODevice *vdev, uint8_t *config) memcpy(netcfg.mac, n->mac, ETH_ALEN); virtio_stl_p(vdev, &netcfg.speed, n->net_conf.speed); netcfg.duplex = n->net_conf.duplex; + netcfg.rss_max_key_size = VIRTIO_NET_RSS_MAX_KEY_SIZE; + virtio_stw_p(vdev, &netcfg.rss_max_indirection_table_length, + virtio_host_has_feature(vdev, VIRTIO_NET_F_RSS) ? + VIRTIO_NET_RSS_MAX_TABLE_LEN : 1); + virtio_stl_p(vdev, &netcfg.supported_hash_types, + VIRTIO_NET_RSS_SUPPORTED_HASHES); memcpy(config, &netcfg, n->config_size); } @@ -561,7 +560,7 @@ static int peer_has_ufo(VirtIONet *n) } static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs, - int version_1) + int version_1, int hash_report) { int i; NetClientState *nc; @@ -569,7 +568,10 @@ static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs, n->mergeable_rx_bufs = mergeable_rx_bufs; if (version_1) { - n->guest_hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); + n->guest_hdr_len = hash_report ? + sizeof(struct virtio_net_hdr_v1_hash) : + sizeof(struct virtio_net_hdr_mrg_rxbuf); + n->rss_data.populate_hash = !!hash_report; } else { n->guest_hdr_len = n->mergeable_rx_bufs ? sizeof(struct virtio_net_hdr_mrg_rxbuf) : @@ -690,6 +692,8 @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO4); virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6); virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN); + + virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT); } if (!peer_has_vnet_hdr(n) || !peer_has_ufo(n)) { @@ -701,6 +705,8 @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, return features; } + virtio_clear_feature(&features, VIRTIO_NET_F_RSS); + virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT); features = vhost_net_get_features(get_vhost_net(nc->peer), features); vdev->backend_features = features; @@ -860,18 +866,22 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) } virtio_net_set_multiqueue(n, + virtio_has_feature(features, VIRTIO_NET_F_RSS) || virtio_has_feature(features, VIRTIO_NET_F_MQ)); virtio_net_set_mrg_rx_bufs(n, virtio_has_feature(features, VIRTIO_NET_F_MRG_RXBUF), virtio_has_feature(features, - VIRTIO_F_VERSION_1)); + VIRTIO_F_VERSION_1), + virtio_has_feature(features, + VIRTIO_NET_F_HASH_REPORT)); n->rsc4_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO4); n->rsc6_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO6); + n->rss_data.redirect = virtio_has_feature(features, VIRTIO_NET_F_RSS); if (n->has_vnet_hdr) { n->curr_guest_offloads = @@ -1136,25 +1146,165 @@ static int virtio_net_handle_announce(VirtIONet *n, uint8_t cmd, } } +static void virtio_net_disable_rss(VirtIONet *n) +{ + if (n->rss_data.enabled) { + trace_virtio_net_rss_disable(); + } + n->rss_data.enabled = false; +} + +static uint16_t virtio_net_handle_rss(VirtIONet *n, + struct iovec *iov, + unsigned int iov_cnt, + bool do_rss) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(n); + struct virtio_net_rss_config cfg; + size_t s, offset = 0, size_get; + uint16_t queues, i; + struct { + uint16_t us; + uint8_t b; + } QEMU_PACKED temp; + const char *err_msg = ""; + uint32_t err_value = 0; + + if (do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_RSS)) { + err_msg = "RSS is not negotiated"; + goto error; + } + if (!do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) { + err_msg = "Hash report is not negotiated"; + goto error; + } + size_get = offsetof(struct virtio_net_rss_config, indirection_table); + s = iov_to_buf(iov, iov_cnt, offset, &cfg, size_get); + if (s != size_get) { + err_msg = "Short command buffer"; + err_value = (uint32_t)s; + goto error; + } + n->rss_data.hash_types = virtio_ldl_p(vdev, &cfg.hash_types); + n->rss_data.indirections_len = + virtio_lduw_p(vdev, &cfg.indirection_table_mask); + n->rss_data.indirections_len++; + if (!do_rss) { + n->rss_data.indirections_len = 1; + } + if (!is_power_of_2(n->rss_data.indirections_len)) { + err_msg = "Invalid size of indirection table"; + err_value = n->rss_data.indirections_len; + goto error; + } + if (n->rss_data.indirections_len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { + err_msg = "Too large indirection table"; + err_value = n->rss_data.indirections_len; + goto error; + } + n->rss_data.default_queue = do_rss ? + virtio_lduw_p(vdev, &cfg.unclassified_queue) : 0; + if (n->rss_data.default_queue >= n->max_queues) { + err_msg = "Invalid default queue"; + err_value = n->rss_data.default_queue; + goto error; + } + offset += size_get; + size_get = sizeof(uint16_t) * n->rss_data.indirections_len; + g_free(n->rss_data.indirections_table); + n->rss_data.indirections_table = g_malloc(size_get); + if (!n->rss_data.indirections_table) { + err_msg = "Can't allocate indirections table"; + err_value = n->rss_data.indirections_len; + goto error; + } + s = iov_to_buf(iov, iov_cnt, offset, + n->rss_data.indirections_table, size_get); + if (s != size_get) { + err_msg = "Short indirection table buffer"; + err_value = (uint32_t)s; + goto error; + } + for (i = 0; i < n->rss_data.indirections_len; ++i) { + uint16_t val = n->rss_data.indirections_table[i]; + n->rss_data.indirections_table[i] = virtio_lduw_p(vdev, &val); + } + offset += size_get; + size_get = sizeof(temp); + s = iov_to_buf(iov, iov_cnt, offset, &temp, size_get); + if (s != size_get) { + err_msg = "Can't get queues"; + err_value = (uint32_t)s; + goto error; + } + queues = do_rss ? virtio_lduw_p(vdev, &temp.us) : n->curr_queues; + if (queues == 0 || queues > n->max_queues) { + err_msg = "Invalid number of queues"; + err_value = queues; + goto error; + } + if (temp.b > VIRTIO_NET_RSS_MAX_KEY_SIZE) { + err_msg = "Invalid key size"; + err_value = temp.b; + goto error; + } + if (!temp.b && n->rss_data.hash_types) { + err_msg = "No key provided"; + err_value = 0; + goto error; + } + if (!temp.b && !n->rss_data.hash_types) { + virtio_net_disable_rss(n); + return queues; + } + offset += size_get; + size_get = temp.b; + s = iov_to_buf(iov, iov_cnt, offset, n->rss_data.key, size_get); + if (s != size_get) { + err_msg = "Can get key buffer"; + err_value = (uint32_t)s; + goto error; + } + n->rss_data.enabled = true; + trace_virtio_net_rss_enable(n->rss_data.hash_types, + n->rss_data.indirections_len, + temp.b); + return queues; +error: + trace_virtio_net_rss_error(err_msg, err_value); + virtio_net_disable_rss(n); + return 0; +} + static int virtio_net_handle_mq(VirtIONet *n, uint8_t cmd, struct iovec *iov, unsigned int iov_cnt) { VirtIODevice *vdev = VIRTIO_DEVICE(n); - struct virtio_net_ctrl_mq mq; - size_t s; uint16_t queues; - s = iov_to_buf(iov, iov_cnt, 0, &mq, sizeof(mq)); - if (s != sizeof(mq)) { - return VIRTIO_NET_ERR; - } + virtio_net_disable_rss(n); + if (cmd == VIRTIO_NET_CTRL_MQ_HASH_CONFIG) { + queues = virtio_net_handle_rss(n, iov, iov_cnt, false); + return queues ? VIRTIO_NET_OK : VIRTIO_NET_ERR; + } + if (cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { + queues = virtio_net_handle_rss(n, iov, iov_cnt, true); + } else if (cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { + struct virtio_net_ctrl_mq mq; + size_t s; + if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ)) { + return VIRTIO_NET_ERR; + } + s = iov_to_buf(iov, iov_cnt, 0, &mq, sizeof(mq)); + if (s != sizeof(mq)) { + return VIRTIO_NET_ERR; + } + queues = virtio_lduw_p(vdev, &mq.virtqueue_pairs); - if (cmd != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { + } else { return VIRTIO_NET_ERR; } - queues = virtio_lduw_p(vdev, &mq.virtqueue_pairs); - if (queues < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || queues > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || queues > n->max_queues || @@ -1387,8 +1537,107 @@ static int receive_filter(VirtIONet *n, const uint8_t *buf, int size) return 0; } +static uint8_t virtio_net_get_hash_type(bool isip4, + bool isip6, + bool isudp, + bool istcp, + uint32_t types) +{ + if (isip4) { + if (istcp && (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4)) { + return NetPktRssIpV4Tcp; + } + if (isudp && (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4)) { + return NetPktRssIpV4Udp; + } + if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { + return NetPktRssIpV4; + } + } else if (isip6) { + uint32_t mask = VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | + VIRTIO_NET_RSS_HASH_TYPE_TCPv6; + + if (istcp && (types & mask)) { + return (types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) ? + NetPktRssIpV6TcpEx : NetPktRssIpV6Tcp; + } + mask = VIRTIO_NET_RSS_HASH_TYPE_UDP_EX | VIRTIO_NET_RSS_HASH_TYPE_UDPv6; + if (isudp && (types & mask)) { + return (types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) ? + NetPktRssIpV6UdpEx : NetPktRssIpV6Udp; + } + mask = VIRTIO_NET_RSS_HASH_TYPE_IP_EX | VIRTIO_NET_RSS_HASH_TYPE_IPv6; + if (types & mask) { + return (types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) ? + NetPktRssIpV6Ex : NetPktRssIpV6; + } + } + return 0xff; +} + +static void virtio_set_packet_hash(const uint8_t *buf, uint8_t report, + uint32_t hash) +{ + struct virtio_net_hdr_v1_hash *hdr = (void *)buf; + hdr->hash_value = hash; + hdr->hash_report = report; +} + +static int virtio_net_process_rss(NetClientState *nc, const uint8_t *buf, + size_t size) +{ + VirtIONet *n = qemu_get_nic_opaque(nc); + unsigned int index = nc->queue_index, new_index = index; + struct NetRxPkt *pkt = n->rx_pkt; + uint8_t net_hash_type; + uint32_t hash; + bool isip4, isip6, isudp, istcp; + static const uint8_t reports[NetPktRssIpV6UdpEx + 1] = { + VIRTIO_NET_HASH_REPORT_IPv4, + VIRTIO_NET_HASH_REPORT_TCPv4, + VIRTIO_NET_HASH_REPORT_TCPv6, + VIRTIO_NET_HASH_REPORT_IPv6, + VIRTIO_NET_HASH_REPORT_IPv6_EX, + VIRTIO_NET_HASH_REPORT_TCPv6_EX, + VIRTIO_NET_HASH_REPORT_UDPv4, + VIRTIO_NET_HASH_REPORT_UDPv6, + VIRTIO_NET_HASH_REPORT_UDPv6_EX + }; + + net_rx_pkt_set_protocols(pkt, buf + n->host_hdr_len, + size - n->host_hdr_len); + net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp); + if (isip4 && (net_rx_pkt_get_ip4_info(pkt)->fragment)) { + istcp = isudp = false; + } + if (isip6 && (net_rx_pkt_get_ip6_info(pkt)->fragment)) { + istcp = isudp = false; + } + net_hash_type = virtio_net_get_hash_type(isip4, isip6, isudp, istcp, + n->rss_data.hash_types); + if (net_hash_type > NetPktRssIpV6UdpEx) { + if (n->rss_data.populate_hash) { + virtio_set_packet_hash(buf, VIRTIO_NET_HASH_REPORT_NONE, 0); + } + return n->rss_data.redirect ? n->rss_data.default_queue : -1; + } + + hash = net_rx_pkt_calc_rss_hash(pkt, net_hash_type, n->rss_data.key); + + if (n->rss_data.populate_hash) { + virtio_set_packet_hash(buf, reports[net_hash_type], hash); + } + + if (n->rss_data.redirect) { + new_index = hash & (n->rss_data.indirections_len - 1); + new_index = n->rss_data.indirections_table[new_index]; + } + + return (index == new_index) ? -1 : new_index; +} + static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf, - size_t size) + size_t size, bool no_rss) { VirtIONet *n = qemu_get_nic_opaque(nc); VirtIONetQueue *q = virtio_net_get_subqueue(nc); @@ -1402,6 +1651,14 @@ static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf, return -1; } + if (!no_rss && n->rss_data.enabled) { + int index = virtio_net_process_rss(nc, buf, size); + if (index >= 0) { + NetClientState *nc2 = qemu_get_subqueue(n->nic, index); + return virtio_net_receive_rcu(nc2, buf, size, true); + } + } + /* hdr_len refers to the header we supply to the guest */ if (!virtio_net_has_buffers(q, size + n->guest_hdr_len - n->host_hdr_len)) { return 0; @@ -1452,6 +1709,11 @@ static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf, } receive_header(n, sg, elem->in_num, buf, size); + if (n->rss_data.populate_hash) { + offset = sizeof(mhdr); + iov_from_buf(sg, elem->in_num, offset, + buf + offset, n->host_hdr_len - sizeof(mhdr)); + } offset = n->host_hdr_len; total += n->guest_hdr_len; guest_offset = n->guest_hdr_len; @@ -1496,7 +1758,7 @@ static ssize_t virtio_net_do_receive(NetClientState *nc, const uint8_t *buf, { RCU_READ_LOCK_GUARD(); - return virtio_net_receive_rcu(nc, buf, size); + return virtio_net_receive_rcu(nc, buf, size, false); } static void virtio_net_rsc_extract_unit4(VirtioNetRscChain *chain, @@ -1539,15 +1801,15 @@ static size_t virtio_net_rsc_drain_seg(VirtioNetRscChain *chain, VirtioNetRscSeg *seg) { int ret; - struct virtio_net_hdr *h; + struct virtio_net_hdr_v1 *h; - h = (struct virtio_net_hdr *)seg->buf; + h = (struct virtio_net_hdr_v1 *)seg->buf; h->flags = 0; h->gso_type = VIRTIO_NET_HDR_GSO_NONE; if (seg->is_coalesced) { - *virtio_net_rsc_ext_num_packets(h) = seg->packets; - *virtio_net_rsc_ext_num_dupacks(h) = seg->dup_ack; + h->rsc.segments = seg->packets; + h->rsc.dup_acks = seg->dup_ack; h->flags = VIRTIO_NET_HDR_F_RSC_INFO; if (chain->proto == ETH_P_IP) { h->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; @@ -2444,7 +2706,9 @@ static int virtio_net_post_load_device(void *opaque, int version_id) trace_virtio_net_post_load_device(); virtio_net_set_mrg_rx_bufs(n, n->mergeable_rx_bufs, virtio_vdev_has_feature(vdev, - VIRTIO_F_VERSION_1)); + VIRTIO_F_VERSION_1), + virtio_vdev_has_feature(vdev, + VIRTIO_NET_F_HASH_REPORT)); /* MAC_TABLE_ENTRIES may be different from the saved image */ if (n->mac_table.in_use > MAC_TABLE_ENTRIES) { @@ -2493,6 +2757,13 @@ static int virtio_net_post_load_device(void *opaque, int version_id) } } + if (n->rss_data.enabled) { + trace_virtio_net_rss_enable(n->rss_data.hash_types, + n->rss_data.indirections_len, + sizeof(n->rss_data.key)); + } else { + trace_virtio_net_rss_disable(); + } return 0; } @@ -2670,6 +2941,32 @@ static const VMStateDescription vmstate_virtio_net_has_vnet = { }, }; +static bool virtio_net_rss_needed(void *opaque) +{ + return VIRTIO_NET(opaque)->rss_data.enabled; +} + +static const VMStateDescription vmstate_virtio_net_rss = { + .name = "virtio-net-device/rss", + .version_id = 1, + .minimum_version_id = 1, + .needed = virtio_net_rss_needed, + .fields = (VMStateField[]) { + VMSTATE_BOOL(rss_data.enabled, VirtIONet), + VMSTATE_BOOL(rss_data.redirect, VirtIONet), + VMSTATE_BOOL(rss_data.populate_hash, VirtIONet), + VMSTATE_UINT32(rss_data.hash_types, VirtIONet), + VMSTATE_UINT16(rss_data.indirections_len, VirtIONet), + VMSTATE_UINT16(rss_data.default_queue, VirtIONet), + VMSTATE_UINT8_ARRAY(rss_data.key, VirtIONet, + VIRTIO_NET_RSS_MAX_KEY_SIZE), + VMSTATE_VARRAY_UINT16_ALLOC(rss_data.indirections_table, VirtIONet, + rss_data.indirections_len, 0, + vmstate_info_uint16, uint16_t), + VMSTATE_END_OF_LIST() + }, +}; + static const VMStateDescription vmstate_virtio_net_device = { .name = "virtio-net-device", .version_id = VIRTIO_NET_VM_VERSION, @@ -2720,6 +3017,10 @@ static const VMStateDescription vmstate_virtio_net_device = { has_ctrl_guest_offloads), VMSTATE_END_OF_LIST() }, + .subsections = (const VMStateDescription * []) { + &vmstate_virtio_net_rss, + NULL + } }; static NetClientInfo net_virtio_info = { @@ -3063,7 +3364,7 @@ static void virtio_net_device_realize(DeviceState *dev, Error **errp) n->vqs[0].tx_waiting = 0; n->tx_burst = n->net_conf.txburst; - virtio_net_set_mrg_rx_bufs(n, 0, 0); + virtio_net_set_mrg_rx_bufs(n, 0, 0, 0); n->promisc = 1; /* for compatibility */ n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); @@ -3075,6 +3376,8 @@ static void virtio_net_device_realize(DeviceState *dev, Error **errp) QTAILQ_INIT(&n->rsc_chains); n->qdev = dev; + + net_rx_pkt_init(&n->rx_pkt, false); } static void virtio_net_device_unrealize(DeviceState *dev) @@ -3111,6 +3414,8 @@ static void virtio_net_device_unrealize(DeviceState *dev) g_free(n->vqs); qemu_del_nic(n->nic); virtio_net_rsc_cleanup(n); + g_free(n->rss_data.indirections_table); + net_rx_pkt_uninit(n->rx_pkt); virtio_cleanup(vdev); } @@ -3212,6 +3517,10 @@ static Property virtio_net_properties[] = { DEFINE_PROP_BIT64("ctrl_guest_offloads", VirtIONet, host_features, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, true), DEFINE_PROP_BIT64("mq", VirtIONet, host_features, VIRTIO_NET_F_MQ, false), + DEFINE_PROP_BIT64("rss", VirtIONet, host_features, + VIRTIO_NET_F_RSS, false), + DEFINE_PROP_BIT64("hash", VirtIONet, host_features, + VIRTIO_NET_F_HASH_REPORT, false), DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features, VIRTIO_NET_F_RSC_EXT, false), DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout, |