Virtio uses shiftleft/shiftright to tag packets

src/apps/ipv6/nd_light.lua

@@ -123,7 +123,6 @@ function nd_light:new (conf)
    -- Prepare packet for solicitation of next hop
    local nh = { nsent = 0 }
    local dgram = datagram:new()
-   nh.packet = dgram:packet()
    local sol_node_mcast = ipv6:solicited_node_mcast(conf.next_hop)
    local ipv6 = ipv6:new({ next_header = 58, -- ICMP6
          hop_limit = 255,
@@ -148,6 +147,7 @@ function nd_light:new (conf)
    dgram:push(ethernet:new({ src = conf.local_mac,
                              dst = ethernet:ipv6_mcast(sol_node_mcast),
                              type = 0x86dd }))
+   nh.packet = dgram:packet()
    dgram:free()
 
    -- Timer for retransmits of neighbor solicitations
@@ -176,7 +176,6 @@ function nd_light:new (conf)
    -- Prepare packet for solicited neighbor advertisement
    local sna = {}
    dgram = datagram:new()
-   sna.packet = dgram:packet()
    -- Leave dst address unspecified.  It will be set to the source of
    -- the incoming solicitation
    ipv6 = ipv6:new({ next_header = 58, -- ICMP6
@@ -197,6 +196,8 @@ function nd_light:new (conf)
    -- Leave dst address unspecified.
    dgram:push(ethernet:new({ src = conf.local_mac,
                              type = 0x86dd }))
+   sna.packet = dgram:packet()
+
    -- Parse the headers we want to modify later on from our template
    -- packet.
    dgram = dgram:new(sna.packet, ethernet)

src/apps/keyed_ipv6_tunnel/tunnel.lua

@@ -194,7 +194,7 @@ function SimpleKeyedTunnel:push()
 
    while not link.empty(l_in) do
       local p = link.receive(l_in)
-      packet.prepend(p, self.header, HEADER_SIZE)
+      p = packet.prepend(p, self.header, HEADER_SIZE)
       local plength = ffi.cast(plength_ctype, p.data + LENGTH_OFFSET)
       plength[0] = lib.htons(SESSION_COOKIE_SIZE + p.length - HEADER_SIZE)
       link.transmit(l_out, p)
@@ -249,7 +249,7 @@ function SimpleKeyedTunnel:push()
          -- discard packet
          packet.free(p)
       else
-         packet.shiftleft(p, HEADER_SIZE)
+         p = packet.shiftleft(p, HEADER_SIZE)
          link.transmit(l_out, p)
       end
    end

src/apps/lwaftr/fragmentv6.lua

@@ -73,7 +73,7 @@ local function _reassemble_validated(fragments, fragment_offsets, fragment_lengt
 
    -- Copy the original headers; this automatically does the right thing in the face of vlans.
    local fixed_headers_size = l2_size + constants.ipv6_fixed_header_size
-   ffi.copy(repkt.data, first_fragment, l2_size + constants.ipv6_fixed_header_size)
+   ffi.copy(repkt.data, first_fragment.data, l2_size + constants.ipv6_fixed_header_size)
    -- Update the next header; it's not a fragment anymore
    repkt.data[l2_size + constants.o_ipv6_next_header] = ipv6_next_header
 

src/apps/lwaftr/generator.lua

@@ -279,7 +279,7 @@ function ipv6_encapsulate(ipv4_pkt, params)
 
    local payload_length = p.length - ethernet_header_size
    local dscp_and_ecn = p.data[ethernet_header_size + IPV4_DSCP_AND_ECN_OFFSET]
-   packet.shiftright(p, ipv6_header_size)
+   p = packet.shiftright(p, ipv6_header_size)
 
    -- IPv6 packet is tagged
    if params.vlan_tag then

src/apps/lwaftr/icmp.lua

@@ -85,8 +85,9 @@ function new_icmpv4_packet(from_eth, to_eth, from_ip, to_ip, initial_pkt, l2_siz
                                  protocol = constants.proto_icmp,
                                  src = from_ip, dst = to_ip})
    dgram:push(ipv4_header)
+   new_pkt = dgram:packet()
    ipv4_header:free()
-   packet.shiftright(new_pkt, l2_size)
+   new_pkt = packet.shiftright(new_pkt, l2_size)
    write_eth_header(new_pkt.data, from_eth, to_eth, constants.n_ethertype_ipv4, config.vlan_tag)
 
    -- Generate RFC 1812 ICMPv4 packets, which carry as much payload as they can,
@@ -126,7 +127,7 @@ function new_icmpv6_packet(from_eth, to_eth, from_ip, to_ip, initial_pkt, l2_siz
                                  next_header = constants.proto_icmpv6,
                                  src = from_ip, dst = to_ip})
    dgram:push(ipv6_header)
-   packet.shiftright(new_pkt, l2_size)
+   new_pkt = packet.shiftright(dgram:packet(), l2_size)
    write_eth_header(new_pkt.data, from_eth, to_eth, constants.n_ethertype_ipv6, config.vlan_tag)
 
    local max_size = constants.max_icmpv6_packet_size

src/apps/lwaftr/lwaftr.lua

@@ -380,7 +380,10 @@ local function encapsulate_and_transmit(lwstate, pkt, ipv6_dst, ipv6_src)
    local payload_length = get_ethernet_payload_length(pkt)
    local l3_header = get_ethernet_payload(pkt)
    local dscp_and_ecn = get_ipv4_dscp_and_ecn(l3_header)
-   packet.shiftright(pkt, ipv6_fixed_header_size)
+   -- Note that this may invalidate any pointer into pkt.data.  Be warned!
+   pkt = packet.shiftright(pkt, ipv6_fixed_header_size)
+   -- Fetch possibly-moved L3 header location.
+   l3_header = get_ethernet_payload(pkt)
    write_eth_header(pkt.data, ether_src, ether_dst, n_ethertype_ipv6)
    write_ipv6_header(l3_header, ipv6_src, ipv6_dst,
                      dscp_and_ecn, next_hdr_type, payload_length)
@@ -563,7 +566,8 @@ local function flush_decapsulation(lwstate)
           and ipv6_equals(get_ipv6_src_address(ipv6_header), b4_addr)
           and ipv6_equals(get_ipv6_dst_address(ipv6_header), br_addr)) then
          -- Source softwire is valid; decapsulate and forward.
-         packet.shiftleft(pkt, ipv6_fixed_header_size)
+         -- Note that this may invalidate any pointer into pkt.data.  Be warned!
+         pkt = packet.shiftleft(pkt, ipv6_fixed_header_size)
          write_eth_header(pkt.data, lwstate.aftr_mac_inet_side, lwstate.inet_mac,
                           n_ethertype_ipv4)
          transmit_ipv4(lwstate, pkt)

src/apps/lwaftr/ndp.lua

@@ -214,17 +214,18 @@ function form_ns(local_eth, local_ipv6, dst_ipv6)
    local i = ipv6:new({ hop_limit = hop_limit, 
                         next_header = proto_icmpv6,
                         src = local_ipv6, dst = dst_ipv6 })
-   i:payload_length(ns_pkt.length)
+   i:payload_length(dgram:packet().length)
 
-   local ph = i:pseudo_header(ns_pkt.length, proto_icmpv6)
+   local ph = i:pseudo_header(dgram:packet().length, proto_icmpv6)
    ph_len = ipv6_pseudoheader_size
    local base_checksum = checksum.ipsum(ffi.cast("uint8_t*", ph), ph_len, 0)
-   local csum = checksum.ipsum(ns_pkt.data, ns_pkt.length, bit.bnot(base_checksum))
-   wr16(ns_pkt.data + 2, C.htons(csum))
+   local csum = checksum.ipsum(dgram:packet().data, dgram:packet().length, bit.bnot(base_checksum))
+   wr16(dgram:packet().data + 2, C.htons(csum))
 
    dgram:push(i)
    dgram:push(ethernet:new({ src = local_eth, dst = ethernet_broadcast,
                              type = ethertype_ipv6 }))
+   ns_pkt = dgram:packet()
    dgram:free()
    return ns_pkt
 end
@@ -257,17 +258,19 @@ local function form_sna(local_eth, local_ipv6, is_router, soliciting_pkt)
    local i = ipv6:new({ hop_limit = hop_limit,
                         next_header = proto_icmpv6,
                         src = local_ipv6, dst = dst_ipv6 })
-   i:payload_length(na_pkt.length)
+   i:payload_length(dgram:packet().length)
 
-   local ph = i:pseudo_header(na_pkt.length, proto_icmpv6)
+   local ph = i:pseudo_header(dgram:packet().length, proto_icmpv6)
    ph_len = ipv6_pseudoheader_size
    local base_checksum = checksum.ipsum(ffi.cast("uint8_t*", ph), ph_len, 0)
-   local csum = checksum.ipsum(na_pkt.data, na_pkt.length, bit.bnot(base_checksum))
-   wr16(na_pkt.data + 2, C.htons(csum))
+   local csum = checksum.ipsum(dgram:packet().data, dgram:packet().length,
+                               bit.bnot(base_checksum))
+   wr16(dgram:packet().data + 2, C.htons(csum))
 
    dgram:push(i)
    dgram:push(ethernet:new({ src = local_eth, dst = dst_eth,
                              type = ethertype_ipv6 }))
+   na_pkt = dgram:packet()
    dgram:free()
    return na_pkt
 end

src/core/packet.h

@@ -5,7 +5,7 @@ enum { PACKET_PAYLOAD_SIZE = 10*1024 };
 
 // Packet of network data, with associated metadata.
 struct packet {
-    unsigned char data[PACKET_PAYLOAD_SIZE];
     uint16_t length;           // data payload length
+    unsigned char data[PACKET_PAYLOAD_SIZE];
 };
 

src/core/packet.lua

@@ -5,6 +5,7 @@ module(...,package.seeall)
 local debug = _G.developer_debug
 
 local ffi = require("ffi")
+local bit = require("bit")
 local C = ffi.C
 
 local lib      = require("core.lib")
@@ -16,9 +17,17 @@ require("core.packet_h")
 local packet_t = ffi.typeof("struct packet")
 local packet_ptr_t = ffi.typeof("struct packet *")
 local packet_size = ffi.sizeof(packet_t)
-local header_size = 8
 max_payload = tonumber(C.PACKET_PAYLOAD_SIZE)
 
+-- For operations that add or remove headers from the beginning of a
+-- packet, instead of copying around the payload we just move the
+-- packet structure as a whole around.
+local packet_alignment = 512
+local default_headroom = 256
+-- The Intel82599 driver requires even-byte alignment, so let's keep
+-- things aligned at least this much.
+local minimum_alignment = 2
+
 -- Freelist containing empty packets ready for use.
 
 ffi.cdef[[
@@ -66,17 +75,16 @@ end
 
 -- Create a new empty packet.
 function new_packet ()
-   local p = ffi.cast(packet_ptr_t, memory.dma_alloc(packet_size))
+   local base = memory.dma_alloc(packet_size + packet_alignment,
+                                 packet_alignment)
+   local p = ffi.cast(packet_ptr_t, base + default_headroom)
    p.length = 0
    return p
 end
 
 -- Create an exact copy of a packet.
 function clone (p)
-   local p2 = allocate()
-   ffi.copy(p2, p, p.length)
-   p2.length = p.length
-   return p2
+   return from_pointer(p.data, p.length)
 end
 
 -- Append data to the end of a packet.
@@ -89,25 +97,56 @@ end
 
 -- Prepend data to the start of a packet.
 function prepend (p, ptr, len)
-   assert(p.length + len <= max_payload, "packet payload overflow")
-   C.memmove(p.data + len, p.data, p.length) -- Move the existing payload
+   p = shiftright(p, len)
    ffi.copy(p.data, ptr, len)                -- Fill the gap
-   p.length = p.length + len
    return p
 end
 
 -- Move packet data to the left. This shortens the packet by dropping
 -- the header bytes at the front.
 function shiftleft (p, bytes)
-   C.memmove(p.data, p.data+bytes, p.length-bytes)
-   p.length = p.length - bytes
+   assert(bytes >= 0 and bytes <= p.length)
+   local ptr = ffi.cast("char*", p)
+   local len = p.length
+   local headroom = bit.band(ffi.cast("uint64_t", ptr), packet_alignment - 1)
+   -- We only have a certain amount of headroom, otherwise the end of
+   -- p.data will point out of our allocation.  If we're withing the
+   -- alignment wiggle room, just move the packet around.  Otherwise
+   -- copy the payload, but also reset the headroom at the same time.
+   if bytes + headroom < packet_alignment
+      and bit.band(bytes, minimum_alignment - 1) == 0 then
+      p = ffi.cast(packet_ptr_t, ptr + bytes)
+      p.length = len - bytes
+      return p
+   else
+      local delta_headroom = default_headroom - headroom
+      C.memmove(p.data + delta_headroom, p.data + bytes, len - bytes)
+      p = ffi.cast(packet_ptr_t, ptr + delta_headroom)
+      p.length = len - bytes
+      return p
+   end
 end
 
 -- Move packet data to the right. This leaves length bytes of data
 -- at the beginning of the packet.
 function shiftright (p, bytes)
-   C.memmove(p.data + bytes, p.data, p.length)
-   p.length = p.length + bytes
+   local ptr = ffi.cast("char*", p)
+   local len = p.length
+   local headroom = bit.band(ffi.cast("uint64_t", ptr), packet_alignment - 1)
+   if bytes <= headroom and bit.band(bytes, minimum_alignment - 1) == 0 then
+      -- Take from the headroom.
+      p = ffi.cast(packet_ptr_t, ptr - bytes)
+      p.length = len + bytes
+      return p
+   else
+      -- No headroom for the shift; re-set the headroom to the default.
+      assert(bytes <= max_payload - len)
+      local delta_headroom = default_headroom - headroom
+      C.memmove(p.data + bytes + delta_headroom, p.data, len)
+      p = ffi.cast(packet_ptr_t, ptr + delta_headroom)
+      p.length = len + bytes
+      return p
+   end
 end
 
 -- Conveniently create a packet by copying some existing data.
@@ -116,6 +155,9 @@ function from_string (d)         return from_pointer(d, #d) end
 
 -- Free a packet that is no longer in use.
 local function free_internal (p)
+   local ptr = ffi.cast("char*", p)
+   local headroom = bit.band(ffi.cast("uint64_t", ptr), packet_alignment - 1)
+   p = ffi.cast(packet_ptr_t, ptr - headroom + default_headroom)
    p.length = 0
    freelist_add(packets_fl, p)
 end   

src/lib/ipsec/esp.lua

@@ -200,7 +200,7 @@ ABCDEFGHIJKLMNOPQRSTUVWXYZ
    local p2 = packet.clone(p_enc)
    assert(dec:decapsulate(p2), "decapsulation failed")
    print("decrypted", lib.hexdump(ffi.string(p2.data, p2.length)))
-   assert(p2.length == p.length and C.memcmp(p, p2, p.length) == 0,
+   assert(p2.length == p.length and C.memcmp(p.data, p2.data, p.length) == 0,
           "integrity check failed")
    -- Check invalid packets.
    local p_invalid = packet.from_string("invalid")
@@ -221,7 +221,7 @@ ABCDEFGHIJKLMNOPQRSTUVWXYZ
    print("decrypted", lib.hexdump(ffi.string(e_min.data, e_min.length)))
    assert(e_min.length == PAYLOAD_OFFSET)
    assert(p_min.length == e_min.length
-          and C.memcmp(p_min, e_min, p_min.length) == 0,
+          and C.memcmp(p_min.data, e_min.data, p_min.length) == 0,
           "integrity check failed")
    -- Check transmitted Sequence Number wrap around
    C.memset(dec.window, 0, dec.window_size / 8); -- clear window

src/lib/protocol/datagram.lua

@@ -147,7 +147,7 @@ function datagram:push_raw (data, length)
       -- The memmove() would invalidate the data pointer of headers
       -- that have already been parsed.
       assert(self._parse.index == 0, "parse stack not empty")
-      packet.prepend(self._packet[0], data, length)
+      self._packet[0] = packet.prepend(self._packet[0], data, length)
       self._parse.offset = self._parse.offset + length
    end
 end
@@ -277,7 +277,7 @@ function datagram:pop_raw (length, ulp)
       -- The memmove() would invalidate the data pointer of headers
       -- that have already been parsed.
       assert(self._parse.index == 0, "parse stack not empty")
-      packet.shiftleft(self._packet[0], length)
+      self._packet[0] = packet.shiftleft(self._packet[0], length)
    end
    if ulp then self._parse.ulp = ulp end
 end
@@ -347,6 +347,7 @@ function selftest ()
    dgram:push(l2tp)
    dgram:push(ip)
    dgram:push(ether)
+   p = dgram:packet()
    local _, p_size = dgram:payload(data, data_size)
    assert(p_size == data_size)
    local _, d_size = dgram:data()
@@ -379,7 +380,7 @@ function selftest ()
    dgram:commit()
    _, d_size = dgram:data()
    assert(d_size == ether2:sizeof() + ip2:sizeof() + l2tp:sizeof() + data_size)
-   dgram:new(p, ethernet, { delayed_commit = true })
+   dgram:new(dgram:packet(), ethernet, { delayed_commit = true })
    assert(ether2:eq(dgram:parse()))
    assert(ip2:eq(dgram:parse()))
    assert(l2tp:eq(dgram:parse()))

src/lib/virtio/net_driver.lua

@@ -21,7 +21,6 @@ local bit       = require('bit')
 local virtq     = require('lib.virtio.virtq_driver')
 local VirtioPci = require('lib.virtio.virtio_pci').VirtioPci
 local checksum  = require('lib.checksum')
-require('lib.virtio.virtio_h')
 
 local band, bor, rshift, lshift = bit.band, bit.bor, bit.rshift, bit.lshift
 local prepare_packet4, prepare_packet6 = checksum.prepare_packet4, checksum.prepare_packet6
@@ -34,10 +33,10 @@ local ETHERTYPE_OFF = 12
 local ETHERLEN = 14 -- DST MAC | SRC MAC | ethertype
 local VIRTIO_NET_HDR_F_NEEDS_CSUM = 1
 
-local min_features =   C.VIRTIO_RING_F_INDIRECT_DESC + C.VIRTIO_NET_F_CSUM
-local want_features =  C.VIRTIO_F_ANY_LAYOUT +
-                       C.VIRTIO_RING_F_INDIRECT_DESC +
-                       C.VIRTIO_NET_F_MAC
+local min_features = C.VIRTIO_NET_F_CSUM +
+   C.VIRTIO_F_ANY_LAYOUT +
+   C.VIRTIO_NET_F_CTRL_VQ
+local want_features =  min_features
 
 local RXQ = 0
 local TXQ = 1
@@ -54,7 +53,6 @@ function VirtioNetDriver:new(args)
 
    if args.use_checksum then
       self.transmit = self._transmit_checksum
-      self.want_features = self.want_features + C.VIRTIO_NET_F_CSUM
    else
       self.transmit = self._transmit
    end