Deploying to gh-pages from @ eunomia-bpf/bpf-developer-tutorial@bbd7b1638d 🚀

42-xdp-loadbalancer/.gitignore

@@ -0,0 +1,5 @@
+.output
+uprobe
+merge-btf
+*.btf
+xdp_lb

42-xdp-loadbalancer/Makefile

@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+OUTPUT := .output
+CLANG ?= clang
+LIBBPF_SRC := $(abspath ../third_party/libbpf/src)
+BPFTOOL_SRC := $(abspath ../third_party/bpftool/src)
+LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
+BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
+BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
+LIBBLAZESYM_SRC := $(abspath ../third_party/blazesym/)
+LIBBLAZESYM_OBJ := $(abspath $(OUTPUT)/libblazesym.a)
+LIBBLAZESYM_HEADER := $(abspath $(OUTPUT)/blazesym.h)
+ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
+			 | sed 's/arm.*/arm/' \
+			 | sed 's/aarch64/arm64/' \
+			 | sed 's/ppc64le/powerpc/' \
+			 | sed 's/mips.*/mips/' \
+			 | sed 's/riscv64/riscv/' \
+			 | sed 's/loongarch64/loongarch/')
+VMLINUX := ../third_party/vmlinux/$(ARCH)/vmlinux.h
+# Use our own libbpf API headers and Linux UAPI headers distributed with
+# libbpf to avoid dependency on system-wide headers, which could be missing or
+# outdated
+INCLUDES := -I$(OUTPUT) -I../third_party/libbpf/include/uapi -I$(dir $(VMLINUX))
+CFLAGS := -g -Wall
+ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
+
+APPS = xdp_lb
+
+CARGO ?= $(shell which cargo)
+ifeq ($(strip $(CARGO)),)
+BZS_APPS :=
+else
+BZS_APPS := 
+APPS += $(BZS_APPS)
+# Required by libblazesym
+ALL_LDFLAGS += -lrt -ldl -lpthread -lm
+endif
+
+# Get Clang's default includes on this system. We'll explicitly add these dirs
+# to the includes list when compiling with `-target bpf` because otherwise some
+# architecture-specific dirs will be "missing" on some architectures/distros -
+# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h,
+# sys/cdefs.h etc. might be missing.
+#
+# Use '-idirafter': Don't interfere with include mechanics except where the
+# build would have failed anyways.
+CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - </dev/null 2>&1 \
+	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
+
+ifeq ($(V),1)
+	Q =
+	msg =
+else
+	Q = @
+	msg = @printf '  %-8s %s%s\n'					\
+		      "$(1)"						\
+		      "$(patsubst $(abspath $(OUTPUT))/%,%,$(2))"	\
+		      "$(if $(3), $(3))";
+	MAKEFLAGS += --no-print-directory
+endif
+
+define allow-override
+  $(if $(or $(findstring environment,$(origin $(1))),\
+            $(findstring command line,$(origin $(1)))),,\
+    $(eval $(1) = $(2)))
+endef
+
+$(call allow-override,CC,$(CROSS_COMPILE)cc)
+$(call allow-override,LD,$(CROSS_COMPILE)ld)
+
+.PHONY: all
+all: $(APPS)
+
+.PHONY: clean
+clean:
+	$(call msg,CLEAN)
+	$(Q)rm -rf $(OUTPUT) $(APPS)
+
+$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
+	$(call msg,MKDIR,$@)
+	$(Q)mkdir -p $@
+
+# Build libbpf
+$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
+	$(call msg,LIB,$@)
+	$(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1		      \
+		    OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@)		      \
+		    INCLUDEDIR= LIBDIR= UAPIDIR=			      \
+		    install
+
+# Build bpftool
+$(BPFTOOL): | $(BPFTOOL_OUTPUT)
+	$(call msg,BPFTOOL,$@)
+	$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap
+
+
+$(LIBBLAZESYM_SRC)/target/release/libblazesym.a::
+	$(Q)cd $(LIBBLAZESYM_SRC) && $(CARGO) build --features=cheader,dont-generate-test-files --release
+
+$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
+	$(call msg,LIB, $@)
+	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym.a $@
+
+$(LIBBLAZESYM_HEADER): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
+	$(call msg,LIB,$@)
+	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/blazesym.h $@
+
+# Build BPF code
+$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL)
+	$(call msg,BPF,$@)
+	$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH)		      \
+		     $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES)		      \
+		     -c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+	$(Q)$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+
+# Generate BPF skeletons
+$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL)
+	$(call msg,GEN-SKEL,$@)
+	$(Q)$(BPFTOOL) gen skeleton $< > $@
+
+# Build user-space code
+$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
+
+$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
+	$(call msg,CC,$@)
+	$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
+
+$(patsubst %,$(OUTPUT)/%.o,$(BZS_APPS)): $(LIBBLAZESYM_HEADER)
+
+$(BZS_APPS): $(LIBBLAZESYM_OBJ)
+
+# Build application binary
+$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) | $(OUTPUT)
+	$(call msg,BINARY,$@)
+	$(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lelf -lz -o $@
+
+# delete failed targets
+.DELETE_ON_ERROR:
+
+# keep intermediate (.skel.h, .bpf.o, etc) targets
+.SECONDARY:

42-xdp-loadbalancer/no-docker/xdp_pass.c

@@ -0,0 +1,14 @@
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+
+SEC("xdp")
+int xdp_pass(struct xdp_md* ctx) {
+    void* data = (void*)(long)ctx->data;
+    void* data_end = (void*)(long)ctx->data_end;
+    int pkt_sz = data_end - data;
+
+    bpf_printk("packet size is %d", pkt_sz);
+    return XDP_PASS;
+}
+
+char __license[] SEC("license") = "GPL";

42-xdp-loadbalancer/setup.sh

@@ -0,0 +1,159 @@
+#!/bin/bash
+
+set -xe
+
+part_mac="DE:AD:BE:EF:00:"
+
+create_bridge () {
+  if ! ip link show $1 &> /dev/null; then
+    ip link add name $1 type bridge
+    ip link set dev $1 up
+  else
+    echo "Bridge $1 already exists."
+  fi
+}
+
+create_pair () {
+  if ! ip link show $1 &> /dev/null; then
+    ip link add name $1 type veth peer name $2
+    ip link set $1 address "$part_mac""$5"
+    ip addr add $3 brd + dev $1
+    ip link set $2 master $4
+    ip link set dev $1 up
+    ip link set dev $2 up
+  else
+    echo "Veth pair $1 <--> $2 already exists."
+  fi
+}
+
+create_pair_ns () {
+  if ! ip link show $2 &> /dev/null; then
+    ip link add name $1 type veth peer name $2
+    ip link set $2 master $4
+    ip link set dev $2 up
+
+    ip netns add $5
+    ip link set $1 netns $5
+    ip netns exec $5 ip addr add $3 brd + dev $1
+    ip netns exec $5 ip link set $1 address "$part_mac""$6"
+    ip netns exec $5 ip link set dev $1 up
+    ip netns exec $5 ip link set lo up  # Bring up loopback interface
+  else
+    echo "Veth pair $1 <--> $2 already exists in namespace $5."
+  fi
+}
+
+# Create bridge br0
+create_bridge br0
+
+# Create veth pairs and assign IPs
+create_pair veth0 veth1 "10.0.0.1/24" br0 01
+
+# Create veth pairs in namespaces h2, h3, and lb
+create_pair_ns veth2 veth3 "10.0.0.2/24" br0 h2 02
+create_pair_ns veth4 veth5 "10.0.0.3/24" br0 h3 03
+
+# Create the lb namespace
+create_pair_ns veth6 veth7 "10.0.0.10/24" br0 lb 10
+
+# Enable IP forwarding on the host
+sudo sysctl -w net.ipv4.ip_forward=1
+
+# Set the FORWARD chain policy to ACCEPT in iptables to ensure packets are forwarded
+sudo iptables -P FORWARD ACCEPT
+
+# maybe you can do similar things
+# sudo ip netns exec h2 bpftool load xdp_pass.o veth2
+# sudo ip netns exec h3 bpftool load  xdp_pass.o veth4
+
+# Helper function for error exit on ping failure
+function ping_or_fail() {
+  if ! sudo ip netns exec $1 ping -c 3 $2; then
+    echo "Ping from $1 to $2 failed!"
+    exit 1
+  fi
+}
+
+# Ping test with failure checks
+function check_connectivity() {
+  echo "Testing connectivity between namespaces and Load Balancer..."
+
+  # Ping from h2 to h3 and h3 to h2
+  ping_or_fail h2 10.0.0.3
+  ping_or_fail h3 10.0.0.2
+
+  # Ping from h2 to Load Balancer and h3 to Load Balancer
+  ping_or_fail h2 10.0.0.10
+  ping_or_fail h3 10.0.0.10
+
+  # Ping from Load Balancer to h2 and h3
+  ping_or_fail lb 10.0.0.2
+  ping_or_fail lb 10.0.0.3
+
+  # Ping from Local Machine to Load Balancer
+  ping -c 3 10.0.0.10 || { echo "Ping from Local Machine to Load Balancer failed!"; exit 1; }
+
+  echo "All ping tests passed!"
+}
+
+# Debugging helper functions
+
+# Check if all interfaces are up and running
+check_interfaces () {
+  for ns in h2 h3 lb; do
+    echo "Checking interfaces in namespace $ns..."
+    sudo ip netns exec $ns ip addr show
+    sudo ip netns exec $ns ip link show
+  done
+
+  echo "Checking bridge br0..."
+  ip addr show br0
+  ip link show br0
+}
+
+# Check IP forwarding settings
+check_ip_forwarding () {
+  echo "Checking IP forwarding status on the host..."
+  sudo sysctl net.ipv4.ip_forward
+
+  echo "Checking IP forwarding status in namespace $ns..."
+  sudo ip netns exec $ns sysctl net.ipv4.ip_forward
+}
+
+# Check ARP table
+check_arp_table () {
+  echo "Checking ARP table on the host..."
+  arp -n
+
+  for ns in h2 h3 lb; do
+    echo "Checking ARP table in namespace $ns..."
+    sudo ip netns exec $ns ip neigh show
+  done
+}
+
+# Check routing tables
+check_routing_table () {
+  echo "Checking routing table on the host..."
+  ip route show
+
+  for ns in h2 h3 lb; do
+    echo "Checking routing table in namespace $ns..."
+    sudo ip netns exec $ns ip route show
+  done
+}
+
+# Check if firewall rules are blocking traffic
+check_firewall_rules () {
+  echo "Checking firewall rules on the host..."
+  sudo iptables -L
+}
+
+# Run checks to verify the network
+check_interfaces
+check_ip_forwarding
+check_arp_table
+check_routing_table
+check_firewall_rules
+check_connectivity
+
+echo "Setup and checks completed!"

42-xdp-loadbalancer/teardown.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -xe
+
+rm_bridge () {
+  if ip link show $1 &> /dev/null; then
+    ip link set dev $1 down
+    ip link delete $1 type bridge
+  fi
+}
+
+rm_pair () {
+  if ip link show $1 &> /dev/null; then
+    ip link delete $1 type veth
+  fi
+}
+
+rm_ns () {
+  if ip netns list | grep -w "$1" &> /dev/null; then
+    ip netns delete $1
+  fi
+}
+
+# Remove bridge br0
+rm_bridge br0
+
+# Remove veth pairs
+rm_pair veth0
+rm_pair veth2
+rm_pair veth4
+rm_pair veth6
+
+# Remove namespaces
+rm_ns h2
+rm_ns h3
+rm_ns lb

42-xdp-loadbalancer/xdp_lb.bpf.c

@@ -0,0 +1,117 @@
+// xdp_lb.bpf.c
+#include <bpf/bpf_endian.h>
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <linux/if_ether.h>
+#include <linux/ip.h>
+#include <linux/in.h>
+#include <linux/tcp.h>
+#include "xx_hash.h"
+
+struct backend_config {
+    __u32 ip;
+    unsigned char mac[ETH_ALEN];
+};
+
+// Backend IP and MAC address map
+struct {
+    __uint(type, BPF_MAP_TYPE_ARRAY);
+    __uint(max_entries, 2);  // Two backends
+    __type(key, __u32);
+    __type(value, struct backend_config);
+} backends SEC(".maps");
+
+int client_ip = bpf_htonl(0xa000001);  
+unsigned char client_mac[ETH_ALEN] = {0xDE, 0xAD, 0xBE, 0xEF, 0x0, 0x1};
+int load_balancer_ip = bpf_htonl(0xa00000a);
+unsigned char load_balancer_mac[ETH_ALEN] = {0xDE, 0xAD, 0xBE, 0xEF, 0x0, 0x10};
+
+static __always_inline __u16
+csum_fold_helper(__u64 csum)
+{
+    int i;
+    for (i = 0; i < 4; i++)
+    {
+        if (csum >> 16)
+            csum = (csum & 0xffff) + (csum >> 16);
+    }
+    return ~csum;
+}
+
+static __always_inline __u16
+iph_csum(struct iphdr *iph)
+{
+    iph->check = 0;
+    unsigned long long csum = bpf_csum_diff(0, 0, (unsigned int *)iph, sizeof(struct iphdr), 0);
+    return csum_fold_helper(csum);
+}
+
+SEC("xdp")
+int xdp_load_balancer(struct xdp_md *ctx) {
+    void *data_end = (void *)(long)ctx->data_end;
+    void *data = (void *)(long)ctx->data;
+
+    bpf_printk("xdp_load_balancer received packet");
+
+    // Ethernet header
+    struct ethhdr *eth = data;
+    if ((void *)(eth + 1) > data_end)
+        return XDP_PASS;
+
+    // Check if the packet is IP (IPv4)
+    if (eth->h_proto != __constant_htons(ETH_P_IP))
+        return XDP_PASS;
+
+    // IP header
+    struct iphdr *iph = (struct iphdr *)(eth + 1);
+    if ((void *)(iph + 1) > data_end)
+        return XDP_PASS;
+
+    // Check if the protocol is TCP or UDP
+    if (iph->protocol != IPPROTO_TCP)
+        return XDP_PASS;
+    
+    bpf_printk("Received Source IP: 0x%x", bpf_ntohl(iph->saddr));
+    bpf_printk("Received Destination IP: 0x%x", bpf_ntohl(iph->daddr));
+    bpf_printk("Received Source MAC: %x:%x:%x:%x:%x:%x", eth->h_source[0], eth->h_source[1], eth->h_source[2], eth->h_source[3], eth->h_source[4], eth->h_source[5]);
+    bpf_printk("Received Destination MAC: %x:%x:%x:%x:%x:%x", eth->h_dest[0], eth->h_dest[1], eth->h_dest[2], eth->h_dest[3], eth->h_dest[4], eth->h_dest[5]);
+
+    if (iph->saddr == client_ip)
+    {
+        bpf_printk("Packet from client");
+
+        __u32 key = xxhash32((const char*)iph, sizeof(struct iphdr), 0) % 2;
+
+        struct backend_config *backend = bpf_map_lookup_elem(&backends, &key);
+        if (!backend)
+            return XDP_PASS;
+        
+        iph->daddr = backend->ip;
+        __builtin_memcpy(eth->h_dest, backend->mac, ETH_ALEN);
+    }
+    else
+    {
+        bpf_printk("Packet from backend");
+        iph->daddr = client_ip;
+        __builtin_memcpy(eth->h_dest, client_mac, ETH_ALEN);
+    }
+
+    // Update IP source address to the load balancer's IP
+    iph->saddr = load_balancer_ip;
+    // Update Ethernet source MAC address to the current lb's MAC
+    __builtin_memcpy(eth->h_source, load_balancer_mac, ETH_ALEN);
+
+    // Recalculate IP checksum
+	iph->check = iph_csum(iph);
+
+    bpf_printk("Redirecting packet to new IP 0x%x from IP 0x%x", 
+                bpf_ntohl(iph->daddr), 
+                bpf_ntohl(iph->saddr)
+            );
+    bpf_printk("New Dest MAC: %x:%x:%x:%x:%x:%x", eth->h_dest[0], eth->h_dest[1], eth->h_dest[2], eth->h_dest[3], eth->h_dest[4], eth->h_dest[5]);
+    bpf_printk("New Source MAC: %x:%x:%x:%x:%x:%x\n", eth->h_source[0], eth->h_source[1], eth->h_source[2], eth->h_source[3], eth->h_source[4], eth->h_source[5]);
+    // Return XDP_TX to transmit the modified packet back to the network
+    return XDP_TX;
+}
+
+char _license[] SEC("license") = "GPL";

42-xdp-loadbalancer/xdp_lb.c

@@ -0,0 +1,96 @@
+// xdp_lb.c
+#include <arpa/inet.h>
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <net/if.h>
+#include "xdp_lb.skel.h"  // The generated skeleton
+
+struct backend_config {
+    __u32 ip;
+    unsigned char mac[6];
+};
+
+static int parse_mac(const char *str, unsigned char *mac) {
+    if (sscanf(str, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
+               &mac[0], &mac[1], &mac[2], &mac[3], &mac[4], &mac[5]) != 6) {
+        fprintf(stderr, "Invalid MAC address format\n");
+        return -1;
+    }
+    return 0;
+}
+
+int main(int argc, char **argv) {
+    if (argc != 6) {
+        fprintf(stderr, "Usage: %s <ifname> <backend1_ip> <backend1_mac> <backend2_ip> <backend2_mac>\n", argv[0]);
+        return 1;
+    }
+
+    const char *ifname = argv[1];
+    struct backend_config backend[2];
+
+    // Parse backend 1
+    if (inet_pton(AF_INET, argv[2], &backend[0].ip) != 1) {
+        fprintf(stderr, "Invalid backend 1 IP address\n");
+        return 1;
+    }
+    if (parse_mac(argv[3], backend[0].mac) < 0) {
+        return 1;
+    }
+
+    // Parse backend 2
+    if (inet_pton(AF_INET, argv[4], &backend[1].ip) != 1) {
+        fprintf(stderr, "Invalid backend 2 IP address\n");
+        return 1;
+    }
+    if (parse_mac(argv[5], backend[1].mac) < 0) {
+        return 1;
+    }
+
+    // Load and attach the BPF program
+    struct xdp_lb_bpf *skel = xdp_lb_bpf__open_and_load();
+    if (!skel) {
+        fprintf(stderr, "Failed to open and load BPF skeleton\n");
+        return 1;
+    }
+
+    int ifindex = if_nametoindex(ifname);
+    if (ifindex < 0) {
+        perror("if_nametoindex");
+        xdp_lb_bpf__destroy(skel);
+        return 1;
+    }
+
+    if (bpf_program__attach_xdp(skel->progs.xdp_load_balancer, ifindex) < 0) {
+        fprintf(stderr, "Failed to attach XDP program\n");
+        xdp_lb_bpf__destroy(skel);
+        return 1;
+    }
+
+    // Update backend configurations
+    for (int i = 0; i < 2; i++) {
+        if (bpf_map_update_elem(bpf_map__fd(skel->maps.backends), &i, &backend[i], 0) < 0) {
+            perror("bpf_map_update_elem");
+            xdp_lb_bpf__destroy(skel);
+            return 1;
+        }
+    }
+
+    printf("XDP load balancer configured with backends:\n");
+    printf("Backend 1 - IP: %s, MAC: %s\n", argv[2], argv[3]);
+    printf("Backend 2 - IP: %s, MAC: %s\n", argv[4], argv[5]);
+
+    printf("Press Ctrl+C to exit...\n");
+    while (1) {
+        sleep(1);  // Keep the program running
+    }
+
+    // Cleanup and detach
+    bpf_xdp_detach(ifindex, 0, NULL);
+    xdp_lb_bpf__detach(skel);
+    xdp_lb_bpf__destroy(skel);
+    return 0;
+}

42-xdp-loadbalancer/xx_hash.h

@@ -0,0 +1,57 @@
+#ifndef XXHASH_BPF_H
+#define XXHASH_BPF_H
+
+#define PRIME1 0x9E3779B1U
+#define PRIME2 0x85EBCA77U
+#define PRIME3 0xC2B2AE3DU
+#define PRIME4 0x27D4EB2FU
+#define PRIME5 0x165667B1U
+
+static __always_inline unsigned int rotl (unsigned int x, int r) {
+    return ((x << r) | (x >> (32 - r)));
+}
+// Normal stripe processing routine.
+static __always_inline unsigned int round_xxhash(unsigned int acc, const unsigned int input) {
+    return rotl(acc + (input * PRIME2), 13) * PRIME1;
+}
+
+static __always_inline unsigned int avalanche_step (const unsigned int h, const int rshift, const unsigned int prime) {
+    return (h ^ (h >> rshift)) * prime;
+}
+// Mixes all bits to finalize the hash.
+static __always_inline unsigned int avalanche (const unsigned int h) {
+    return avalanche_step(avalanche_step(avalanche_step(h, 15, PRIME2), 13, PRIME3), 16, 1);
+}
+
+static __always_inline unsigned int endian32 (const char *v) {
+    return (unsigned int)((unsigned char)(v[0]))|((unsigned int)((unsigned char)(v[1])) << 8)
+            |((unsigned int)((unsigned char)(v[2])) << 16)|((unsigned int)((unsigned char)(v[3])) << 24);
+}
+
+static __always_inline unsigned int fetch32 (const char *p, const unsigned int v) {
+    return round_xxhash(v, endian32(p));
+}
+
+// Processes the last 0-15 bytes of p.
+static __always_inline unsigned int finalize (const unsigned int h, const char *p, unsigned int len) {
+    return
+        (len >= 4) ? finalize(rotl(h + (endian32(p) * PRIME3), 17) * PRIME4, p + 4, len - 4) :
+        (len > 0)  ? finalize(rotl(h + ((unsigned char)(*p) * PRIME5), 11) * PRIME1, p + 1, len - 1) :
+        avalanche(h);
+}
+
+static __always_inline unsigned int h16bytes_4 (const char *p, unsigned int len, const unsigned int v1, const unsigned int v2, const unsigned int v3, const unsigned int v4) {
+    return
+        (len >= 16) ? h16bytes_4(p + 16, len - 16, fetch32(p, v1), fetch32(p+4, v2), fetch32(p+8, v3), fetch32(p+12, v4)) :
+        rotl(v1, 1) + rotl(v2, 7) + rotl(v3, 12) + rotl(v4, 18);
+}
+
+static __always_inline unsigned int h16bytes_3 (const char *p, unsigned int len, const unsigned int seed) {
+    return h16bytes_4(p, len, seed + PRIME1 + PRIME2, seed + PRIME2, seed, seed - PRIME1);
+}
+
+static __always_inline unsigned int xxhash32 (const char *input, unsigned int len, unsigned int seed) {
+    return finalize((len >= 16 ? h16bytes_3(input, len, seed) : seed + PRIME5) + len, (input) + (len & ~0xF), len & 0xF);
+}
+
+#endif