diff --git a/src/apps/ipv6/nd_light.lua b/src/apps/ipv6/nd_light.lua index 3147e5a333..6ce9985246 100644 --- a/src/apps/ipv6/nd_light.lua +++ b/src/apps/ipv6/nd_light.lua @@ -123,7 +123,6 @@ function nd_light:new (conf) -- Prepare packet for solicitation of next hop local nh = { nsent = 0 } local dgram = datagram:new() - nh.packet = dgram:packet() local sol_node_mcast = ipv6:solicited_node_mcast(conf.next_hop) local ipv6 = ipv6:new({ next_header = 58, -- ICMP6 hop_limit = 255, @@ -148,6 +147,7 @@ function nd_light:new (conf) dgram:push(ethernet:new({ src = conf.local_mac, dst = ethernet:ipv6_mcast(sol_node_mcast), type = 0x86dd })) + nh.packet = dgram:packet() dgram:free() -- Timer for retransmits of neighbor solicitations @@ -176,7 +176,6 @@ function nd_light:new (conf) -- Prepare packet for solicited neighbor advertisement local sna = {} dgram = datagram:new() - sna.packet = dgram:packet() -- Leave dst address unspecified. It will be set to the source of -- the incoming solicitation ipv6 = ipv6:new({ next_header = 58, -- ICMP6 @@ -197,6 +196,8 @@ function nd_light:new (conf) -- Leave dst address unspecified. dgram:push(ethernet:new({ src = conf.local_mac, type = 0x86dd })) + sna.packet = dgram:packet() + -- Parse the headers we want to modify later on from our template -- packet. dgram = dgram:new(sna.packet, ethernet) diff --git a/src/apps/keyed_ipv6_tunnel/tunnel.lua b/src/apps/keyed_ipv6_tunnel/tunnel.lua index ad78f36cfd..ab23384fa3 100644 --- a/src/apps/keyed_ipv6_tunnel/tunnel.lua +++ b/src/apps/keyed_ipv6_tunnel/tunnel.lua @@ -194,7 +194,7 @@ function SimpleKeyedTunnel:push() while not link.empty(l_in) do local p = link.receive(l_in) - packet.prepend(p, self.header, HEADER_SIZE) + p = packet.prepend(p, self.header, HEADER_SIZE) local plength = ffi.cast(plength_ctype, p.data + LENGTH_OFFSET) plength[0] = lib.htons(SESSION_COOKIE_SIZE + p.length - HEADER_SIZE) link.transmit(l_out, p) @@ -249,7 +249,7 @@ function SimpleKeyedTunnel:push() -- discard packet packet.free(p) else - packet.shiftleft(p, HEADER_SIZE) + p = packet.shiftleft(p, HEADER_SIZE) link.transmit(l_out, p) end end diff --git a/src/apps/lwaftr/fragmentv6.lua b/src/apps/lwaftr/fragmentv6.lua index 0aba639f8e..56ee61681c 100644 --- a/src/apps/lwaftr/fragmentv6.lua +++ b/src/apps/lwaftr/fragmentv6.lua @@ -73,7 +73,7 @@ local function _reassemble_validated(fragments, fragment_offsets, fragment_lengt -- Copy the original headers; this automatically does the right thing in the face of vlans. local fixed_headers_size = l2_size + constants.ipv6_fixed_header_size - ffi.copy(repkt.data, first_fragment, l2_size + constants.ipv6_fixed_header_size) + ffi.copy(repkt.data, first_fragment.data, l2_size + constants.ipv6_fixed_header_size) -- Update the next header; it's not a fragment anymore repkt.data[l2_size + constants.o_ipv6_next_header] = ipv6_next_header diff --git a/src/apps/lwaftr/generator.lua b/src/apps/lwaftr/generator.lua index 258692490c..8253c8c830 100644 --- a/src/apps/lwaftr/generator.lua +++ b/src/apps/lwaftr/generator.lua @@ -279,7 +279,7 @@ function ipv6_encapsulate(ipv4_pkt, params) local payload_length = p.length - ethernet_header_size local dscp_and_ecn = p.data[ethernet_header_size + IPV4_DSCP_AND_ECN_OFFSET] - packet.shiftright(p, ipv6_header_size) + p = packet.shiftright(p, ipv6_header_size) -- IPv6 packet is tagged if params.vlan_tag then diff --git a/src/apps/lwaftr/icmp.lua b/src/apps/lwaftr/icmp.lua index 7def8ef9cf..01b224d82f 100644 --- a/src/apps/lwaftr/icmp.lua +++ b/src/apps/lwaftr/icmp.lua @@ -85,8 +85,9 @@ function new_icmpv4_packet(from_eth, to_eth, from_ip, to_ip, initial_pkt, l2_siz protocol = constants.proto_icmp, src = from_ip, dst = to_ip}) dgram:push(ipv4_header) + new_pkt = dgram:packet() ipv4_header:free() - packet.shiftright(new_pkt, l2_size) + new_pkt = packet.shiftright(new_pkt, l2_size) write_eth_header(new_pkt.data, from_eth, to_eth, constants.n_ethertype_ipv4, config.vlan_tag) -- Generate RFC 1812 ICMPv4 packets, which carry as much payload as they can, @@ -126,7 +127,7 @@ function new_icmpv6_packet(from_eth, to_eth, from_ip, to_ip, initial_pkt, l2_siz next_header = constants.proto_icmpv6, src = from_ip, dst = to_ip}) dgram:push(ipv6_header) - packet.shiftright(new_pkt, l2_size) + new_pkt = packet.shiftright(dgram:packet(), l2_size) write_eth_header(new_pkt.data, from_eth, to_eth, constants.n_ethertype_ipv6, config.vlan_tag) local max_size = constants.max_icmpv6_packet_size diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 7a9563849c..4ebe3a2574 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -380,7 +380,10 @@ local function encapsulate_and_transmit(lwstate, pkt, ipv6_dst, ipv6_src) local payload_length = get_ethernet_payload_length(pkt) local l3_header = get_ethernet_payload(pkt) local dscp_and_ecn = get_ipv4_dscp_and_ecn(l3_header) - packet.shiftright(pkt, ipv6_fixed_header_size) + -- Note that this may invalidate any pointer into pkt.data. Be warned! + pkt = packet.shiftright(pkt, ipv6_fixed_header_size) + -- Fetch possibly-moved L3 header location. + l3_header = get_ethernet_payload(pkt) write_eth_header(pkt.data, ether_src, ether_dst, n_ethertype_ipv6) write_ipv6_header(l3_header, ipv6_src, ipv6_dst, dscp_and_ecn, next_hdr_type, payload_length) @@ -563,7 +566,8 @@ local function flush_decapsulation(lwstate) and ipv6_equals(get_ipv6_src_address(ipv6_header), b4_addr) and ipv6_equals(get_ipv6_dst_address(ipv6_header), br_addr)) then -- Source softwire is valid; decapsulate and forward. - packet.shiftleft(pkt, ipv6_fixed_header_size) + -- Note that this may invalidate any pointer into pkt.data. Be warned! + pkt = packet.shiftleft(pkt, ipv6_fixed_header_size) write_eth_header(pkt.data, lwstate.aftr_mac_inet_side, lwstate.inet_mac, n_ethertype_ipv4) transmit_ipv4(lwstate, pkt) diff --git a/src/apps/lwaftr/ndp.lua b/src/apps/lwaftr/ndp.lua index c2c5f29535..1c21ad3293 100644 --- a/src/apps/lwaftr/ndp.lua +++ b/src/apps/lwaftr/ndp.lua @@ -214,17 +214,18 @@ function form_ns(local_eth, local_ipv6, dst_ipv6) local i = ipv6:new({ hop_limit = hop_limit, next_header = proto_icmpv6, src = local_ipv6, dst = dst_ipv6 }) - i:payload_length(ns_pkt.length) + i:payload_length(dgram:packet().length) - local ph = i:pseudo_header(ns_pkt.length, proto_icmpv6) + local ph = i:pseudo_header(dgram:packet().length, proto_icmpv6) ph_len = ipv6_pseudoheader_size local base_checksum = checksum.ipsum(ffi.cast("uint8_t*", ph), ph_len, 0) - local csum = checksum.ipsum(ns_pkt.data, ns_pkt.length, bit.bnot(base_checksum)) - wr16(ns_pkt.data + 2, C.htons(csum)) + local csum = checksum.ipsum(dgram:packet().data, dgram:packet().length, bit.bnot(base_checksum)) + wr16(dgram:packet().data + 2, C.htons(csum)) dgram:push(i) dgram:push(ethernet:new({ src = local_eth, dst = ethernet_broadcast, type = ethertype_ipv6 })) + ns_pkt = dgram:packet() dgram:free() return ns_pkt end @@ -257,17 +258,19 @@ local function form_sna(local_eth, local_ipv6, is_router, soliciting_pkt) local i = ipv6:new({ hop_limit = hop_limit, next_header = proto_icmpv6, src = local_ipv6, dst = dst_ipv6 }) - i:payload_length(na_pkt.length) + i:payload_length(dgram:packet().length) - local ph = i:pseudo_header(na_pkt.length, proto_icmpv6) + local ph = i:pseudo_header(dgram:packet().length, proto_icmpv6) ph_len = ipv6_pseudoheader_size local base_checksum = checksum.ipsum(ffi.cast("uint8_t*", ph), ph_len, 0) - local csum = checksum.ipsum(na_pkt.data, na_pkt.length, bit.bnot(base_checksum)) - wr16(na_pkt.data + 2, C.htons(csum)) + local csum = checksum.ipsum(dgram:packet().data, dgram:packet().length, + bit.bnot(base_checksum)) + wr16(dgram:packet().data + 2, C.htons(csum)) dgram:push(i) dgram:push(ethernet:new({ src = local_eth, dst = dst_eth, type = ethertype_ipv6 })) + na_pkt = dgram:packet() dgram:free() return na_pkt end diff --git a/src/core/packet.h b/src/core/packet.h index de57e390c0..a5dcac9a5f 100644 --- a/src/core/packet.h +++ b/src/core/packet.h @@ -5,7 +5,7 @@ enum { PACKET_PAYLOAD_SIZE = 10*1024 }; // Packet of network data, with associated metadata. struct packet { - unsigned char data[PACKET_PAYLOAD_SIZE]; uint16_t length; // data payload length + unsigned char data[PACKET_PAYLOAD_SIZE]; }; diff --git a/src/core/packet.lua b/src/core/packet.lua index 1f8fc579a8..d32616f128 100644 --- a/src/core/packet.lua +++ b/src/core/packet.lua @@ -5,6 +5,7 @@ module(...,package.seeall) local debug = _G.developer_debug local ffi = require("ffi") +local bit = require("bit") local C = ffi.C local lib = require("core.lib") @@ -16,9 +17,17 @@ require("core.packet_h") local packet_t = ffi.typeof("struct packet") local packet_ptr_t = ffi.typeof("struct packet *") local packet_size = ffi.sizeof(packet_t) -local header_size = 8 max_payload = tonumber(C.PACKET_PAYLOAD_SIZE) +-- For operations that add or remove headers from the beginning of a +-- packet, instead of copying around the payload we just move the +-- packet structure as a whole around. +local packet_alignment = 512 +local default_headroom = 256 +-- The Intel82599 driver requires even-byte alignment, so let's keep +-- things aligned at least this much. +local minimum_alignment = 2 + -- Freelist containing empty packets ready for use. ffi.cdef[[ @@ -66,17 +75,16 @@ end -- Create a new empty packet. function new_packet () - local p = ffi.cast(packet_ptr_t, memory.dma_alloc(packet_size)) + local base = memory.dma_alloc(packet_size + packet_alignment, + packet_alignment) + local p = ffi.cast(packet_ptr_t, base + default_headroom) p.length = 0 return p end -- Create an exact copy of a packet. function clone (p) - local p2 = allocate() - ffi.copy(p2, p, p.length) - p2.length = p.length - return p2 + return from_pointer(p.data, p.length) end -- Append data to the end of a packet. @@ -89,25 +97,56 @@ end -- Prepend data to the start of a packet. function prepend (p, ptr, len) - assert(p.length + len <= max_payload, "packet payload overflow") - C.memmove(p.data + len, p.data, p.length) -- Move the existing payload + p = shiftright(p, len) ffi.copy(p.data, ptr, len) -- Fill the gap - p.length = p.length + len return p end -- Move packet data to the left. This shortens the packet by dropping -- the header bytes at the front. function shiftleft (p, bytes) - C.memmove(p.data, p.data+bytes, p.length-bytes) - p.length = p.length - bytes + assert(bytes >= 0 and bytes <= p.length) + local ptr = ffi.cast("char*", p) + local len = p.length + local headroom = bit.band(ffi.cast("uint64_t", ptr), packet_alignment - 1) + -- We only have a certain amount of headroom, otherwise the end of + -- p.data will point out of our allocation. If we're withing the + -- alignment wiggle room, just move the packet around. Otherwise + -- copy the payload, but also reset the headroom at the same time. + if bytes + headroom < packet_alignment + and bit.band(bytes, minimum_alignment - 1) == 0 then + p = ffi.cast(packet_ptr_t, ptr + bytes) + p.length = len - bytes + return p + else + local delta_headroom = default_headroom - headroom + C.memmove(p.data + delta_headroom, p.data + bytes, len - bytes) + p = ffi.cast(packet_ptr_t, ptr + delta_headroom) + p.length = len - bytes + return p + end end -- Move packet data to the right. This leaves length bytes of data -- at the beginning of the packet. function shiftright (p, bytes) - C.memmove(p.data + bytes, p.data, p.length) - p.length = p.length + bytes + local ptr = ffi.cast("char*", p) + local len = p.length + local headroom = bit.band(ffi.cast("uint64_t", ptr), packet_alignment - 1) + if bytes <= headroom and bit.band(bytes, minimum_alignment - 1) == 0 then + -- Take from the headroom. + p = ffi.cast(packet_ptr_t, ptr - bytes) + p.length = len + bytes + return p + else + -- No headroom for the shift; re-set the headroom to the default. + assert(bytes <= max_payload - len) + local delta_headroom = default_headroom - headroom + C.memmove(p.data + bytes + delta_headroom, p.data, len) + p = ffi.cast(packet_ptr_t, ptr + delta_headroom) + p.length = len + bytes + return p + end end -- Conveniently create a packet by copying some existing data. @@ -116,6 +155,9 @@ function from_string (d) return from_pointer(d, #d) end -- Free a packet that is no longer in use. local function free_internal (p) + local ptr = ffi.cast("char*", p) + local headroom = bit.band(ffi.cast("uint64_t", ptr), packet_alignment - 1) + p = ffi.cast(packet_ptr_t, ptr - headroom + default_headroom) p.length = 0 freelist_add(packets_fl, p) end diff --git a/src/lib/ipsec/esp.lua b/src/lib/ipsec/esp.lua index d011fe56a5..a9ca071f18 100644 --- a/src/lib/ipsec/esp.lua +++ b/src/lib/ipsec/esp.lua @@ -200,7 +200,7 @@ ABCDEFGHIJKLMNOPQRSTUVWXYZ local p2 = packet.clone(p_enc) assert(dec:decapsulate(p2), "decapsulation failed") print("decrypted", lib.hexdump(ffi.string(p2.data, p2.length))) - assert(p2.length == p.length and C.memcmp(p, p2, p.length) == 0, + assert(p2.length == p.length and C.memcmp(p.data, p2.data, p.length) == 0, "integrity check failed") -- Check invalid packets. local p_invalid = packet.from_string("invalid") @@ -221,7 +221,7 @@ ABCDEFGHIJKLMNOPQRSTUVWXYZ print("decrypted", lib.hexdump(ffi.string(e_min.data, e_min.length))) assert(e_min.length == PAYLOAD_OFFSET) assert(p_min.length == e_min.length - and C.memcmp(p_min, e_min, p_min.length) == 0, + and C.memcmp(p_min.data, e_min.data, p_min.length) == 0, "integrity check failed") -- Check transmitted Sequence Number wrap around C.memset(dec.window, 0, dec.window_size / 8); -- clear window diff --git a/src/lib/protocol/datagram.lua b/src/lib/protocol/datagram.lua index f2b180e64c..a8860b27aa 100644 --- a/src/lib/protocol/datagram.lua +++ b/src/lib/protocol/datagram.lua @@ -147,7 +147,7 @@ function datagram:push_raw (data, length) -- The memmove() would invalidate the data pointer of headers -- that have already been parsed. assert(self._parse.index == 0, "parse stack not empty") - packet.prepend(self._packet[0], data, length) + self._packet[0] = packet.prepend(self._packet[0], data, length) self._parse.offset = self._parse.offset + length end end @@ -277,7 +277,7 @@ function datagram:pop_raw (length, ulp) -- The memmove() would invalidate the data pointer of headers -- that have already been parsed. assert(self._parse.index == 0, "parse stack not empty") - packet.shiftleft(self._packet[0], length) + self._packet[0] = packet.shiftleft(self._packet[0], length) end if ulp then self._parse.ulp = ulp end end @@ -347,6 +347,7 @@ function selftest () dgram:push(l2tp) dgram:push(ip) dgram:push(ether) + p = dgram:packet() local _, p_size = dgram:payload(data, data_size) assert(p_size == data_size) local _, d_size = dgram:data() @@ -379,7 +380,7 @@ function selftest () dgram:commit() _, d_size = dgram:data() assert(d_size == ether2:sizeof() + ip2:sizeof() + l2tp:sizeof() + data_size) - dgram:new(p, ethernet, { delayed_commit = true }) + dgram:new(dgram:packet(), ethernet, { delayed_commit = true }) assert(ether2:eq(dgram:parse())) assert(ip2:eq(dgram:parse())) assert(l2tp:eq(dgram:parse())) diff --git a/src/lib/virtio/net_driver.lua b/src/lib/virtio/net_driver.lua index 0ab83ea2b1..3110f33202 100644 --- a/src/lib/virtio/net_driver.lua +++ b/src/lib/virtio/net_driver.lua @@ -21,7 +21,6 @@ local bit = require('bit') local virtq = require('lib.virtio.virtq_driver') local VirtioPci = require('lib.virtio.virtio_pci').VirtioPci local checksum = require('lib.checksum') -require('lib.virtio.virtio_h') local band, bor, rshift, lshift = bit.band, bit.bor, bit.rshift, bit.lshift local prepare_packet4, prepare_packet6 = checksum.prepare_packet4, checksum.prepare_packet6 @@ -34,10 +33,10 @@ local ETHERTYPE_OFF = 12 local ETHERLEN = 14 -- DST MAC | SRC MAC | ethertype local VIRTIO_NET_HDR_F_NEEDS_CSUM = 1 -local min_features = C.VIRTIO_RING_F_INDIRECT_DESC + C.VIRTIO_NET_F_CSUM -local want_features = C.VIRTIO_F_ANY_LAYOUT + - C.VIRTIO_RING_F_INDIRECT_DESC + - C.VIRTIO_NET_F_MAC +local min_features = C.VIRTIO_NET_F_CSUM + + C.VIRTIO_F_ANY_LAYOUT + + C.VIRTIO_NET_F_CTRL_VQ +local want_features = min_features local RXQ = 0 local TXQ = 1 @@ -54,7 +53,6 @@ function VirtioNetDriver:new(args) if args.use_checksum then self.transmit = self._transmit_checksum - self.want_features = self.want_features + C.VIRTIO_NET_F_CSUM else self.transmit = self._transmit end diff --git a/src/lib/virtio/virtq_driver.lua b/src/lib/virtio/virtq_driver.lua index d5cc27b105..517a3763b7 100644 --- a/src/lib/virtio/virtq_driver.lua +++ b/src/lib/virtio/virtq_driver.lua @@ -12,53 +12,23 @@ local debug = _G.developer_debug local ffi = require("ffi") local C = ffi.C local memory = require('core.memory') +local packet = require('core.packet') local band = require('bit').band +require("lib.virtio.virtio.h") +require("lib.virtio.virtio_vring.h") local physical = memory.virtual_to_physical local VirtioVirtq = {} VirtioVirtq.__index = VirtioVirtq --- The Host uses this in used->flags to advise the Guest: don't kick me when you add a buffer. -local VRING_USED_F_NO_NOTIFY = 1 --- The Guest uses this in avail->flags to advise the Host: don't interrupt me when you consume a buffer -local VRING_AVAIL_F_NO_INTERRUPT = 1 - --- This marks a buffer as continuing via the next field. -local VRING_DESC_F_NEXT = 1 --- This marks a buffer as write-only (otherwise read-only). -local VRING_DESC_F_WRITE = 2 --- This means the buffer contains a list of buffer descriptors. -local VRING_DESC_F_INDIRECT = 4 - -ffi.cdef([[ -struct pk_header { - uint8_t flags; - uint8_t gso_type; - uint16_t hdr_len; - uint16_t gso_size; - uint16_t csum_start; - uint16_t csum_offset; -} __attribute__((packed)); -]]) -local pk_header_t = ffi.typeof("struct pk_header") -local pk_header_size = ffi.sizeof(pk_header_t) +local VRING_F_NO_INTERRUPT = C.VRING_F_NO_INTERRUPT +local VRING_F_NO_NOTIFY = C.VRING_F_NO_NOTIFY -ffi.cdef([[ - struct vring_desc { - /* Address (guest-physical). */ - uint64_t addr; - /* Length. */ - uint32_t len; - /* The flags as indicated above. */ - uint16_t flags; - /* Next field if flags & NEXT */ - uint16_t next; -} __attribute__((packed)); -]]) +local pk_header_t = ffi.typeof("struct virtio_net_hdr") +local pk_header_size = ffi.sizeof(pk_header_t) local vring_desc_t = ffi.typeof("struct vring_desc") - local ringtypes = {} local function vring_type(n) if ringtypes[n] then return ringtypes[n] end @@ -88,10 +58,8 @@ local function vring_type(n) $ *vring; uint64_t vring_physaddr; struct packet *packets[$]; - struct pk_header *headers[$]; - struct vring_desc *desc_tables[$]; } - ]], rng, n, n, n) + ]], rng, n) ffi.metatype(t, VirtioVirtq) ringtypes[n] = t return t @@ -104,42 +72,16 @@ local function allocate_virtq(n) local ptr, phys = memory.dma_alloc(ffi.sizeof(vr.vring[0])) vr.vring = ffi.cast(ring_t, ptr) vr.vring_physaddr = phys - - for i = 0, n-1 do - local desc = vr.vring.desc[i] - local len = 2 * ffi.sizeof(vring_desc_t) - ptr, phys = memory.dma_alloc(len) - vr.desc_tables[i] = ffi.cast("struct vring_desc *", ptr) - - desc.addr = phys - desc.len = len - desc.flags = VRING_DESC_F_INDIRECT - desc.next = i + 1 - end - - for i = 0, n-1 do - local desc_table = vr.desc_tables[i] - - -- Packet header descriptor - local desc = desc_table[0] - ptr, phys = memory.dma_alloc(pk_header_size) - vr.headers[i] = ffi.cast("struct pk_header *", ptr) - desc.addr = phys - desc.len = pk_header_size - desc.flags = VRING_DESC_F_NEXT - desc.next = 1 - - -- Packet data descriptor - desc = desc_table[1] - desc.addr = 0 - desc.len = 0 - desc.flags = 0 - desc.next = -1 + -- Initialize free list. + vr.free_head = -1 + vr.num_free = 0 + for i = n-1, 0, -1 do + vr.vring.desc[i].next = vr.free_head + vr.free_head = i + vr.num_free = vr.num_free + 1 end - vr.num_free = n - -- Disable the interrupts forever, we don't need them - vr.vring.avail.flags = VRING_AVAIL_F_NO_INTERRUPT + vr.vring.avail.flags = VRING_F_NO_INTERRUPT return vr end @@ -148,24 +90,22 @@ function VirtioVirtq:can_add() end function VirtioVirtq:add(p, len, flags, csum_start, csum_offset) - local idx = self.free_head local desc = self.vring.desc[idx] - local desc_table = self.desc_tables[idx] self.free_head = desc.next self.num_free = self.num_free -1 desc.next = -1 - -- Header - local header = self.headers[idx] - header[0].flags = flags - header[0].csum_start = csum_start - header[0].csum_offset = csum_offset - - -- Packet - desc = desc_table[1] + p = packet.shiftright(p, pk_header_size) + local header = ffi.cast("struct virtio_net_hdr *", p.data) + header.flags = flags + header.gso_type = 0 + header.hdr_len = 0 + header.gso_size = 0 + header.csum_start = csum_start + header.csum_offset = csum_offset desc.addr = physical(p.data) - desc.len = len + desc.len = len + pk_header_size desc.flags = 0 desc.next = -1 @@ -175,28 +115,7 @@ function VirtioVirtq:add(p, len, flags, csum_start, csum_offset) end function VirtioVirtq:add_empty_header(p, len) - local idx = self.free_head - local desc = self.vring.desc[idx] - local desc_table = self.desc_tables[idx] - self.free_head = desc.next - self.num_free = self.num_free -1 - desc.next = -1 - - -- Header - local header = self.headers[idx] - header[0].flags = 0 - - -- Packet - desc = desc_table[1] - desc.addr = physical(p.data) - - desc.len = len - desc.flags = 0 - desc.next = -1 - - self.vring.avail.ring[band(self.last_avail_idx, self.num-1)] = idx - self.last_avail_idx = self.last_avail_idx + 1 - self.packets[idx] = p + self:add(p, len, 0, 0, 0) end function VirtioVirtq:update_avail_idx() @@ -216,16 +135,19 @@ function VirtioVirtq:can_get() end function VirtioVirtq:get() - local last_used_idx = band(self.last_used_idx, self.num-1) local used = self.vring.used.ring[last_used_idx] local idx = used.id local desc = self.vring.desc[idx] + -- FIXME: we should allow the NEXT flag or something, though with worse perf + if debug then assert(desc.flags == 0) end local p = self.packets[idx] + self.packets[idx] = nil if debug then assert(p ~= nil) end - p.length = used.len - pk_header_size - if debug then assert(physical(p.data) == self.desc_tables[idx][1].addr) end + if debug then assert(physical(p.data) == desc.addr) end + p.length = used.len + p = packet.shiftleft(p, pk_header_size) self.last_used_idx = self.last_used_idx + 1 desc.next = self.free_head @@ -237,7 +159,7 @@ end function VirtioVirtq:should_notify() -- Notify only if the used ring lacks the "no notify" flag - return band(self.vring.used.flags, VRING_USED_F_NO_NOTIFY) == 0 + return band(self.vring.used.flags, VRING_F_NO_NOTIFY) == 0 end return {