version 0.20

If cgroup is empty, then the process is not exsit any more.
So can be ignored.

.gitignore

@@ -1,4 +1,5 @@
-build
+build*
+.cache
 .directory
 .clangd
 v2ray_config/proxy

.vscode/c_cpp_properties.json

@@ -0,0 +1,15 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [],
+            "defines": ["USE_VMLINUX"],
+            "compilerPath": "/usr/bin/gcc",
+            "cStandard": "c18",
+            "cppStandard": "gnu++17",
+            "intelliSenseMode": "gcc-x64",
+            "configurationProvider": "ms-vscode.cmake-tools"
+        }
+    ],
+    "version": 4
+}

.vscode/settings.json

@@ -0,0 +1,9 @@
+{
+    "cmake.configureOnOpen": true,
+    "cmake.configureArgs": [
+        "-Dbuild_static=OFF",
+        "-Dbuild_execsnoop_dl=ON",
+        "-Dbuild_tools=ON",
+        "-Dbuild_test=ON"
+    ]
+}

CMakeLists.txt

@@ -2,21 +2,23 @@ cmake_minimum_required(VERSION 3.14)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-project(cgproxy VERSION 0.17)
+project(cgproxy VERSION 0.20)
 
 include(GNUInstallDirs)
 
-add_compile_options(-Wall -Wextra -Wpedantic -Wno-unused-result -Wno-unused-parameter)
+add_compile_options(-Wall -Wextra -Wpedantic -Wno-unused-result -Wno-unused-parameter -Wno-overlength-strings)
 
 # for clangd
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # option(with_execsnoop "enable program level proxy control feature, need bcc installed" ON)
+option(build_execsnoop_dl "build libexecsnoop.so which will be dynamic loaded, otherwise built directly into cgproxy" ON)
+option(build_static "build with static link prefered" OFF)
 option(build_tools OFF)
-option(build_test  OFF)
+option(build_test "for develop" OFF)
 
 add_subdirectory(src)
-add_subdirectory(execsnoop-kernel)
+add_subdirectory(execsnoop-libbpf)
 add_subdirectory(pack)
 if (build_tools)
     add_subdirectory(tools)
@@ -25,13 +27,17 @@ if (build_test)
     add_subdirectory(test)
 endif()
 
+configure_file(cgnoproxy.cmake cgnoproxy)
+configure_file(cgproxyd.cmake cgproxyd)
+configure_file(cgproxy.service.cmake cgproxy.service)
+
 # instal scripts and other things
-install(PROGRAMS cgproxyd TYPE BIN)
-install(PROGRAMS cgnoproxy TYPE BIN)
-install(PROGRAMS cgroup-tproxy.sh DESTINATION ${CMAKE_INSTALL_DATADIR}/cgproxy/scripts)
-install(FILES cgproxy.service DESTINATION ${CMAKE_INSTALL_LIBDIR}/systemd/system)
+install(PROGRAMS ${CMAKE_BINARY_DIR}/cgproxyd DESTINATION ${CMAKE_INSTALL_FULL_BINDIR})
+install(PROGRAMS ${CMAKE_BINARY_DIR}/cgnoproxy DESTINATION ${CMAKE_INSTALL_FULL_BINDIR})
+install(PROGRAMS cgroup-tproxy.sh DESTINATION ${CMAKE_INSTALL_FULL_DATADIR}/cgproxy/scripts)
+install(FILES ${CMAKE_BINARY_DIR}/cgproxy.service DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/systemd/system)
 install(FILES config.json DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/cgproxy)
-install(FILES readme.md DESTINATION ${CMAKE_INSTALL_DOCDIR})
+install(FILES readme.md DESTINATION ${CMAKE_INSTALL_FULL_DOCDIR})
 
 # man pages
 set(man_gz ${PROJECT_SOURCE_DIR}/man/cgproxyd.1.gz ${PROJECT_SOURCE_DIR}/man/cgproxy.1.gz ${PROJECT_SOURCE_DIR}/man/cgnoproxy.1.gz)
@@ -40,4 +46,4 @@ add_custom_command(OUTPUT ${man_gz}
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/man
 )
 add_custom_target(man ALL DEPENDS ${man_gz})
-install(FILES ${man_gz} DESTINATION ${CMAKE_INSTALL_MANDIR}/man1/)
+install(FILES ${man_gz} DESTINATION ${CMAKE_INSTALL_FULL_MANDIR}/man1/)

cgnoproxy

@@ -1,2 +0,0 @@
-#!/bin/sh
-exec /usr/bin/cgproxy --noproxy $@

cgnoproxy.cmake

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec @CMAKE_INSTALL_FULL_BINDIR@/cgproxy --noproxy $@

cgproxy.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=cgproxy service
-After=network.target
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/cgproxyd --execsnoop
-
-[Install]
-WantedBy=multi-user.target

cgproxy.service.cmake

@@ -0,0 +1,10 @@
+[Unit]
+Description=cgproxy service
+After=network.target network-online.target
+
+[Service]
+Type=simple
+ExecStart=@CMAKE_INSTALL_FULL_BINDIR@/cgproxyd --execsnoop
+
+[Install]
+WantedBy=multi-user.target

cgproxyd

@@ -1,2 +0,0 @@
-#!/bin/sh
-exec /usr/bin/cgproxy --daemon $@

cgproxyd.cmake

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec @CMAKE_INSTALL_FULL_BINDIR@/cgproxy --daemon $@

cgroup-tproxy.sh

@@ -1,33 +1,35 @@
 #!/bin/bash
+### This script will proxy/noproxy anything running in specific cgroup
+### need cgroup2 support, and iptables cgroup2 path match support
+### 
+### script usage:
+###     cgroup-tproxy.sh [--help|--config|stop]
+### options:
+###     --config=FILE   load config from file
+###            --help   show help info
+###              stop   clean then stop. Variables may change when stopping, which should be avoid
+###                     so always stop first in the last context before start new context
+###
+### available variables with default value:
+###     cgroup_noproxy="/noproxy.slice" 
+###     cgroup_proxy="/proxy.slice" 
+###     port=12345
+###     enable_dns=true
+###     enable_udp=true
+###     enable_tcp=true
+###     enable_ipv4=true
+###     enable_ipv6=true
+###     enable_gateway=false
+###     table=10007
+###     fwmark=0x9973
+###     cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET | head -n 1)
+###
+### semicolon to seperate multi cgroup:
+###     cgroup_noproxy="/noproxy1.slice:/noproxy2.slice"
+###     cgroup_proxy="/proxy1.slice:/proxy2.slice"
+
 print_help(){
-cat << 'DOC'
-#############################################################################
-# 
-# 1. This script need cgroup v2 
-# 
-# 2. Listening port is expected to accept iptables TPROXY, while REDIRECT 
-#    will not work in this script, because REDIRECT only support tcp/ipv4
-# 
-# 3. TPROXY need root or special capability whatever process is listening on port
-#    v2ray as example: 
-#       sudo setcap "cap_net_bind_service=+ep cap_net_admin=+ep" /usr/lib/v2ray/v2ray
-# 
-# 4. this script will proxy anything running in specific cgroup
-# 
-# script usage:
-#     cgroup-tproxy.sh [--help|--config|stop]
-#     --config=FILE
-#           load config from file
-#     --help
-#           show help info
-#     stop
-#           clean then stop
-# 
-# proxy usage:
-#     cgproxy <program>
-# 
-#############################################################################
-DOC
+    sed -rn 's/^### ?//;T;p' "$0"
 }
 
 ## check root
@@ -47,67 +49,75 @@ else
     IFS=':' read -r -a cgroup_noproxy   <<< "$cgroup_noproxy"
 fi
 
-# allow as gateway for local network
-[ -z ${enable_gateway+x} ] && enable_gateway=false
-
-## some variables
+## tproxy listening port
 [ -z ${port+x} ] && port=12345
 
-## some options
+## controll options
 [ -z ${enable_dns+x} ]  && enable_dns=true
-[ -z ${enable_tcp+x} ]  && enable_tcp=true
 [ -z ${enable_udp+x} ]  && enable_udp=true
+[ -z ${enable_tcp+x} ]  && enable_tcp=true
 [ -z ${enable_ipv4+x} ] && enable_ipv4=true
 [ -z ${enable_ipv6+x} ] && enable_ipv6=true
-
-##
-get_available_route_table(){
-    table=10007 
-    while true; do 
-        ip route show table $table &> /dev/null && ((table++)) || { echo $table && break; }
-    done
-}
+[ -z ${enable_gateway+x} ] && enable_gateway=false
 
 ## mark/route things
-[ -z ${table+x} ]       && table=10007 # just a prime number
-[ -z ${fwmark+x} ]      && fwmark=0x9973
-[ -z ${mark_newin+x} ]  && mark_newin=0x9967
+[ -z ${table+x} ]           && table=10007
+[ -z ${fwmark+x} ]          && fwmark=0x9973
+[ -z ${table_reroute+x} ]   && table_reroute=$table
+[ -z ${table_tproxy+x} ]    && table_tproxy=$table
+[ -z ${fwmark_reroute+x} ]  && fwmark_reroute=$fwmark
+[ -z ${fwmark_tproxy+x} ]   && fwmark_tproxy=$fwmark
 
-
-# echo "table: $table fwmark: $fwmark, mark_newin: $mark_newin"
-
-## cgroup things
-cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET)
-cgroup_type="cgroup2"
-cgroup_procs_file="cgroup.procs"
+## cgroup mount point things
+[ -z ${cgroup_mount_point+x} ] && cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET | head -n 1)
 
 
 stop(){
-    iptables -t mangle -L TPROXY_PRE &> /dev/null || return
+    iptables -w 60 -t mangle -L TPROXY_ENT &> /dev/null || return
     echo "iptables: cleaning tproxy iptables"
-    iptables -t mangle -D PREROUTING -j TPROXY_PRE
-    iptables -t mangle -D OUTPUT -j TPROXY_OUT
-    iptables -t mangle -F TPROXY_PRE
-    iptables -t mangle -F TPROXY_OUT
-    iptables -t mangle -F TPROXY_ENT
-    iptables -t mangle -X TPROXY_PRE
-    iptables -t mangle -X TPROXY_OUT
-    iptables -t mangle -X TPROXY_ENT
-    ip6tables -t mangle -D PREROUTING -j TPROXY_PRE
-    ip6tables -t mangle -D OUTPUT -j TPROXY_OUT
-    ip6tables -t mangle -F TPROXY_PRE
-    ip6tables -t mangle -F TPROXY_OUT
-    ip6tables -t mangle -F TPROXY_ENT
-    ip6tables -t mangle -X TPROXY_PRE
-    ip6tables -t mangle -X TPROXY_OUT
-    ip6tables -t mangle -X TPROXY_ENT
-    ip rule delete fwmark $fwmark lookup $table
-    ip route flush table $table
-    ip -6 rule delete fwmark $fwmark lookup $table
-    ip -6 route flush table $table
+
+    iptables -w 60 -t mangle -D PREROUTING -j TPROXY_PRE
+    iptables -w 60 -t mangle -D OUTPUT -j TPROXY_OUT
+
+    iptables -w 60 -t mangle -F TPROXY_PRE
+    iptables -w 60 -t mangle -F TPROXY_ENT
+    iptables -w 60 -t mangle -F TPROXY_OUT
+    iptables -w 60 -t mangle -F TPROXY_MARK
+
+    iptables -w 60 -t mangle -X TPROXY_PRE
+    iptables -w 60 -t mangle -X TPROXY_ENT
+    iptables -w 60 -t mangle -X TPROXY_OUT
+    iptables -w 60 -t mangle -X TPROXY_MARK
+
+    ip rule delete fwmark $fwmark_tproxy lookup $table_tproxy
+    ip rule delete fwmark $fwmark_reroute lookup $table_reroute &> /dev/null
+    ip route flush table $table_tproxy
+    ip route flush table $table_reroute &> /dev/null
+
+    ip6tables -w 60 -t mangle -D PREROUTING -j TPROXY_PRE
+    ip6tables -w 60 -t mangle -D OUTPUT -j TPROXY_OUT
+
+    ip6tables -w 60 -t mangle -F TPROXY_PRE
+    ip6tables -w 60 -t mangle -F TPROXY_OUT
+    ip6tables -w 60 -t mangle -F TPROXY_ENT
+    ip6tables -w 60 -t mangle -F TPROXY_MARK
+
+    ip6tables -w 60 -t mangle -X TPROXY_PRE
+    ip6tables -w 60 -t mangle -X TPROXY_OUT
+    ip6tables -w 60 -t mangle -X TPROXY_ENT
+    ip6tables -w 60 -t mangle -X TPROXY_MARK
+
+    ip -6 rule delete fwmark $fwmark_tproxy lookup $table_tproxy
+    ip -6 rule delete fwmark $fwmark_reroute lookup $table_reroute &> /dev/null
+    ip -6 route flush table $table_tproxy
+    ip -6 route flush table $table_reroute &> /dev/null
+
     ## may not exist, just ignore, and tracking their existence is not reliable
-    iptables -t nat -D POSTROUTING -m owner ! --socket-exists -j MASQUERADE &> /dev/null
-    ip6tables -t nat -D POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE &> /dev/null
+    iptables -w 60 -t nat -D POSTROUTING -m owner ! --socket-exists -j MASQUERADE &> /dev/null
+    ip6tables -w 60 -t nat -D POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE &> /dev/null
+
+    ## unmount cgroup2
+    [ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" = "cgroup2" ] && umount $cgroup_mount_point
 }
 
 ## parse parameter
@@ -126,10 +136,20 @@ case $i in
         print_help
         exit 0
         ;;
+    *)
+        print_help
+        exit 0
+        ;;
 esac
 done
 
-## TODO cgroup need to exists before using in iptables since 5.6.5, maybe it's bug
+
+## check cgroup_mount_point, create and mount if necessary
+[ -z $cgroup_mount_point ] && { >&2 echo "iptables: no cgroup2 mount point available"; exit -1; }
+[ ! -d $cgroup_mount_point ] && mkdir -p $cgroup_mount_point
+[ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" != "cgroup2" ] && mount -t cgroup2 none $cgroup_mount_point
+[ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" != "cgroup2" ] && { >&2 echo "iptables: mount $cgroup_mount_point failed"; exit -1; }
+
 ## only create the first one in arrary
 test -d $cgroup_mount_point$cgroup_proxy    || mkdir $cgroup_mount_point$cgroup_proxy   || exit -1; 
 test -d $cgroup_mount_point$cgroup_noproxy  || mkdir $cgroup_mount_point$cgroup_noproxy || exit -1; 
@@ -149,111 +169,106 @@ done
 unset cgroup_proxy && cgroup_proxy=${_cgroup_proxy[@]}
 
 
+## ipv4 #########################################################################
 echo "iptables: applying tproxy iptables"
-## use TPROXY
-#ipv4#
-ip rule add fwmark $fwmark table $table
-ip route add local default dev lo table $table
-iptables -t mangle -N TPROXY_ENT
-iptables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark
-iptables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark
+## mangle prerouting
+ip rule add fwmark $fwmark_tproxy table $table_tproxy
+ip route add local default dev lo table $table_tproxy
+# core
+iptables -w 60 -t mangle -N TPROXY_ENT
+iptables -w 60 -t mangle -A TPROXY_ENT -m socket -j MARK --set-mark $fwmark_tproxy
+iptables -w 60 -t mangle -A TPROXY_ENT -m socket -j ACCEPT
+iptables -w 60 -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark_tproxy
+iptables -w 60 -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark_tproxy
+# filter
+iptables -w 60 -t mangle -N TPROXY_PRE
+iptables -w 60 -t mangle -A TPROXY_PRE -m addrtype --dst-type LOCAL -j RETURN
+iptables -w 60 -t mangle -A TPROXY_PRE -m addrtype ! --dst-type UNICAST -j RETURN
+$enable_gateway  || iptables -w 60 -t mangle -A TPROXY_PRE -m addrtype ! --src-type LOCAL -j RETURN
+$enable_dns && iptables -w 60 -t mangle -A TPROXY_PRE -p udp --dport 53 -j TPROXY_ENT
+$enable_udp && iptables -w 60 -t mangle -A TPROXY_PRE -p udp -j TPROXY_ENT
+$enable_tcp && iptables -w 60 -t mangle -A TPROXY_PRE -p tcp -j TPROXY_ENT
+# hook
+iptables -w 60 -t mangle -A PREROUTING -j TPROXY_PRE
 
-iptables -t mangle -N TPROXY_PRE
-iptables -t mangle -A TPROXY_PRE -m socket --transparent -j MARK --set-mark $fwmark
-iptables -t mangle -A TPROXY_PRE -m socket --transparent -j RETURN
-iptables -t mangle -A TPROXY_PRE -p icmp -j RETURN
-iptables -t mangle -A TPROXY_PRE -p udp --dport 53 -j TPROXY_ENT
-iptables -t mangle -A TPROXY_PRE -p tcp --dport 53 -j TPROXY_ENT
-iptables -t mangle -A TPROXY_PRE -m addrtype --dst-type LOCAL -j RETURN
-iptables -t mangle -A TPROXY_PRE -m addrtype ! --dst-type UNICAST -j RETURN
-iptables -t mangle -A TPROXY_PRE -j TPROXY_ENT
-iptables -t mangle -A PREROUTING -j TPROXY_PRE
-
-iptables -t mangle -N TPROXY_OUT
-iptables -t mangle -A TPROXY_OUT -p icmp -j RETURN
-iptables -t mangle -A TPROXY_OUT -m connmark --mark  $mark_newin -j RETURN
-iptables -t mangle -A TPROXY_OUT -m addrtype --dst-type LOCAL -j RETURN
-iptables -t mangle -A TPROXY_OUT -m addrtype ! --dst-type UNICAST -j RETURN
+## mangle output
+if [ $fwmark_reroute != $fwmark_tproxy ]; then
+ip rule add fwmark $fwmark_reroute table $table_reroute
+ip route add local default dev lo table $table_reroute
+fi
+# filter
+iptables -w 60 -t mangle -N TPROXY_MARK
+iptables -w 60 -t mangle -A TPROXY_MARK -m addrtype ! --dst-type UNICAST -j RETURN
+$enable_dns && iptables -w 60 -t mangle -A TPROXY_MARK -p udp --dport 53 -j MARK --set-mark $fwmark_reroute
+$enable_udp && iptables -w 60 -t mangle -A TPROXY_MARK -p udp -j MARK --set-mark $fwmark_reroute
+$enable_tcp && iptables -w 60 -t mangle -A TPROXY_MARK -p tcp -j MARK --set-mark $fwmark_reroute
+# cgroup
+iptables -w 60 -t mangle -N TPROXY_OUT
+iptables -w 60 -t mangle -A TPROXY_OUT -m conntrack --ctdir REPLY -j RETURN
 for cg in ${cgroup_noproxy[@]}; do
-iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
+iptables -w 60 -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
 done
 for cg in ${cgroup_proxy[@]}; do
-iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark
+iptables -w 60 -t mangle -A TPROXY_OUT -m cgroup --path $cg -j TPROXY_MARK
 done
-iptables -t mangle -A OUTPUT -j TPROXY_OUT
+# hook
+$enable_ipv4 && iptables -w 60 -t mangle -A OUTPUT -j TPROXY_OUT
 
-#ipv6#
-ip -6 rule add fwmark $fwmark table $table
-ip -6 route add local default dev lo table $table
-ip6tables -t mangle -N TPROXY_ENT
-ip6tables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark
-ip6tables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark
+## ipv6 #########################################################################
+## mangle prerouting
+ip -6 rule add fwmark $fwmark_tproxy table $table_tproxy
+ip -6 route add local default dev lo table $table_tproxy
+# core
+ip6tables -w 60 -t mangle -N TPROXY_ENT
+ip6tables -w 60 -t mangle -A TPROXY_ENT -m socket -j MARK --set-mark $fwmark_tproxy
+ip6tables -w 60 -t mangle -A TPROXY_ENT -m socket -j ACCEPT
+ip6tables -w 60 -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark_tproxy
+ip6tables -w 60 -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark_tproxy
+# filter
+ip6tables -w 60 -t mangle -N TPROXY_PRE
+ip6tables -w 60 -t mangle -A TPROXY_PRE -m addrtype --dst-type LOCAL -j RETURN
+ip6tables -w 60 -t mangle -A TPROXY_PRE -m addrtype ! --dst-type UNICAST -j RETURN
+$enable_gateway  || ip6tables -w 60 -t mangle -A TPROXY_PRE -m addrtype ! --src-type LOCAL -j RETURN
+$enable_dns && ip6tables -w 60 -t mangle -A TPROXY_PRE -p udp --dport 53 -j TPROXY_ENT
+$enable_udp && ip6tables -w 60 -t mangle -A TPROXY_PRE -p udp -j TPROXY_ENT
+$enable_tcp && ip6tables -w 60 -t mangle -A TPROXY_PRE -p tcp -j TPROXY_ENT
+# hook
+ip6tables -w 60 -t mangle -A PREROUTING -j TPROXY_PRE
 
-ip6tables -t mangle -N TPROXY_PRE
-ip6tables -t mangle -A TPROXY_PRE -m socket --transparent -j MARK --set-mark $fwmark
-ip6tables -t mangle -A TPROXY_PRE -m socket --transparent -j RETURN
-ip6tables -t mangle -A TPROXY_PRE -p icmpv6 -j RETURN
-ip6tables -t mangle -A TPROXY_PRE -p udp --dport 53 -j TPROXY_ENT
-ip6tables -t mangle -A TPROXY_PRE -p tcp --dport 53 -j TPROXY_ENT
-ip6tables -t mangle -A TPROXY_PRE -m addrtype --dst-type LOCAL -j RETURN
-ip6tables -t mangle -A TPROXY_PRE -m addrtype ! --dst-type UNICAST -j RETURN
-ip6tables -t mangle -A TPROXY_PRE -j TPROXY_ENT
-ip6tables -t mangle -A PREROUTING -j TPROXY_PRE
-
-ip6tables -t mangle -N TPROXY_OUT
-ip6tables -t mangle -A TPROXY_OUT -p icmpv6 -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m connmark --mark  $mark_newin -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m addrtype --dst-type LOCAL -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m addrtype ! --dst-type UNICAST -j RETURN
+## mangle output
+if [ $fwmark_reroute != $fwmark_tproxy ]; then
+ip -6 rule add fwmark $fwmark_reroute table $table_reroute
+ip -6 route add local default dev lo table $table_reroute
+fi
+# filter
+ip6tables -w 60 -t mangle -N TPROXY_MARK
+ip6tables -w 60 -t mangle -A TPROXY_MARK -m addrtype ! --dst-type UNICAST -j RETURN
+$enable_dns && ip6tables -w 60 -t mangle -A TPROXY_MARK -p udp --dport 53 -j MARK --set-mark $fwmark_reroute
+$enable_udp && ip6tables -w 60 -t mangle -A TPROXY_MARK -p udp -j MARK --set-mark $fwmark_reroute
+$enable_tcp && ip6tables -w 60 -t mangle -A TPROXY_MARK -p tcp -j MARK --set-mark $fwmark_reroute
+# cgroup
+ip6tables -w 60 -t mangle -N TPROXY_OUT
+ip6tables -w 60 -t mangle -A TPROXY_OUT -m conntrack --ctdir REPLY -j RETURN
 for cg in ${cgroup_noproxy[@]}; do
-ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
+ip6tables -w 60 -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
 done
 for cg in ${cgroup_proxy[@]}; do
-ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark
+ip6tables -w 60 -t mangle -A TPROXY_OUT -m cgroup --path $cg -j TPROXY_MARK
 done
-ip6tables -t mangle -A OUTPUT -j TPROXY_OUT
-
-## allow to disable, order is important
-$enable_dns     || iptables  -t mangle -I TPROXY_OUT -p udp --dport 53 -j RETURN
-$enable_dns     || ip6tables -t mangle -I TPROXY_OUT -p udp --dport 53 -j RETURN
-$enable_udp     || iptables  -t mangle -I TPROXY_OUT -p udp -j RETURN
-$enable_udp     || ip6tables -t mangle -I TPROXY_OUT -p udp -j RETURN
-$enable_tcp     || iptables  -t mangle -I TPROXY_OUT -p tcp -j RETURN
-$enable_tcp     || ip6tables -t mangle -I TPROXY_OUT -p tcp -j RETURN
-$enable_ipv4    || iptables  -t mangle -I TPROXY_OUT -j RETURN
-$enable_ipv6    || ip6tables -t mangle -I TPROXY_OUT -j RETURN
+# hook
+$enable_ipv6 && ip6tables -w 60 -t mangle -A OUTPUT -j TPROXY_OUT
 
+## forward #######################################################################
 if $enable_gateway; then
-$enable_dns     || iptables  -t mangle -I TPROXY_PRE -p udp --dport 53 -j RETURN
-$enable_dns     || ip6tables -t mangle -I TPROXY_PRE -p udp --dport 53 -j RETURN
-$enable_udp     || iptables  -t mangle -I TPROXY_PRE -p udp -j RETURN
-$enable_udp     || ip6tables -t mangle -I TPROXY_PRE -p udp -j RETURN
-$enable_tcp     || iptables  -t mangle -I TPROXY_PRE -p tcp -j RETURN
-$enable_tcp     || ip6tables -t mangle -I TPROXY_PRE -p tcp -j RETURN
-$enable_ipv4    || iptables  -t mangle -I TPROXY_PRE -j RETURN
-$enable_ipv6    || ip6tables -t mangle -I TPROXY_PRE -j RETURN
+    iptables  -t nat -A POSTROUTING -m owner ! --socket-exists -j MASQUERADE
+    ip6tables -w 60 -t nat -A POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE # only masquerade ipv6 private address
+    sysctl -w net.ipv4.ip_forward=1
+    sysctl -w net.ipv6.conf.all.forwarding=1
+    echo "iptables: gateway enabled"
 fi
 
-## do not handle local device connection through tproxy if gateway is not enabled
-$enable_gateway || iptables  -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -j RETURN
-$enable_gateway || ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -j RETURN
-
-## make sure following rules are the first in chain TPROXY_PRE to mark new incoming connection or gateway proxy connection
-## so must put at last to insert first
-iptables  -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $mark_newin
-ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $mark_newin
-
-# message for user
+## message for user
 cat << DOC
 iptables: noproxy cgroup: ${cgroup_noproxy[@]}
 iptables: proxied cgroup: ${cgroup_proxy[@]}
 DOC
-
-
-if $enable_gateway; then
-    iptables  -t nat -A POSTROUTING -m owner ! --socket-exists -j MASQUERADE
-    ip6tables -t nat -A POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE # only masquerade ipv6 private address
-    sysctl -w net.ipv4.ip_forward=1
-    sysctl -w net.ipv6.conf.all.forwarding=1
-    echo "ipatbles: gateway enabled"
-fi

config.json

@@ -1,5 +1,5 @@
 {
-    "comment":"For usgae, see https://github.com/springzfx/cgproxy",
+    "comment":"For usage, see https://github.com/springzfx/cgproxy",
 
     "port": 12345,
     "program_noproxy": ["v2ray", "qv2ray"],
@@ -13,6 +13,5 @@
     "enable_ipv4": true,
     "enable_ipv6": true,
     "table": 10007,
-    "fwmark": 39283,
-    "mark_newin": 39271
+    "fwmark": 39283
 }

execsnoop-bcc/CMakeLists.txt

@@ -3,4 +3,4 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 add_library(execsnoop MODULE execsnoop.cpp ../src/common.cpp)
 target_link_libraries(execsnoop bcc)
-install(TARGETS execsnoop DESTINATION /usr/lib/cgproxy/ PERMISSIONS ${basic_permission})
+install(TARGETS execsnoop DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cgproxy/ PERMISSIONS ${basic_permission})

execsnoop-kernel/CMakeLists.txt

@@ -1,9 +1,15 @@
 # find libbpf
-find_library(LIBBPF bpf)
-if (LIBBPF-NOTFOUND)
-    message(FATAL_ERROR "libbpf not found")
+if (build_static)
+    find_library(LIBBPF libbpf.a REQUIRED)
+else()
+    find_library(LIBBPF bpf REQUIRED)
 endif()
 
-add_library(execsnoop MODULE execsnoop_share.cpp)
-target_link_libraries(execsnoop PRIVATE bpf)
-install(TARGETS execsnoop DESTINATION ${CMAKE_INSTALL_LIBDIR}/cgproxy/)
+if (build_execsnoop_dl)
+    add_library(execsnoop MODULE execsnoop_share.cpp)
+    install(TARGETS execsnoop DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cgproxy/)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF} -lelf -lz)
+else()
+    add_library(execsnoop STATIC execsnoop_share.cpp)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF} -l:libelf.a -l:libz.a)
+endif()

execsnoop-kernel/aarch64/execsnoop_kern_skel.h

@@ -0,0 +1,216 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+
+/* THIS FILE IS AUTOGENERATED! */
+#ifndef __EXECSNOOP_KERN_SKEL_H__
+#define __EXECSNOOP_KERN_SKEL_H__
+
+#include <stdlib.h>
+#include <bpf/libbpf.h>
+
+struct execsnoop_kern {
+	struct bpf_object_skeleton *skeleton;
+	struct bpf_object *obj;
+	struct {
+		struct bpf_map *perf_events;
+		struct bpf_map *records;
+	} maps;
+	struct {
+		struct bpf_program *syscall_enter_execve;
+		struct bpf_program *syscall_exit_execve;
+	} progs;
+	struct {
+		struct bpf_link *syscall_enter_execve;
+		struct bpf_link *syscall_exit_execve;
+	} links;
+};
+
+static void
+execsnoop_kern__destroy(struct execsnoop_kern *obj)
+{
+	if (!obj)
+		return;
+	if (obj->skeleton)
+		bpf_object__destroy_skeleton(obj->skeleton);
+	free(obj);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj);
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_opts(const struct bpf_object_open_opts *opts)
+{
+	struct execsnoop_kern *obj;
+
+	obj = (typeof(obj))calloc(1, sizeof(*obj));
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__create_skeleton(obj))
+		goto err;
+	if (bpf_object__open_skeleton(obj->skeleton, opts))
+		goto err;
+
+	return obj;
+err:
+	execsnoop_kern__destroy(obj);
+	return NULL;
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open(void)
+{
+	return execsnoop_kern__open_opts(NULL);
+}
+
+static inline int
+execsnoop_kern__load(struct execsnoop_kern *obj)
+{
+	return bpf_object__load_skeleton(obj->skeleton);
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_and_load(void)
+{
+	struct execsnoop_kern *obj;
+
+	obj = execsnoop_kern__open();
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__load(obj)) {
+		execsnoop_kern__destroy(obj);
+		return NULL;
+	}
+	return obj;
+}
+
+static inline int
+execsnoop_kern__attach(struct execsnoop_kern *obj)
+{
+	return bpf_object__attach_skeleton(obj->skeleton);
+}
+
+static inline void
+execsnoop_kern__detach(struct execsnoop_kern *obj)
+{
+	return bpf_object__detach_skeleton(obj->skeleton);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj)
+{
+	struct bpf_object_skeleton *s;
+
+	s = (typeof(s))calloc(1, sizeof(*s));
+	if (!s)
+		return -1;
+	obj->skeleton = s;
+
+	s->sz = sizeof(*s);
+	s->name = "execsnoop_kern";
+	s->obj = &obj->obj;
+
+	/* maps */
+	s->map_cnt = 2;
+	s->map_skel_sz = sizeof(*s->maps);
+	s->maps = (typeof(s->maps))calloc(s->map_cnt, s->map_skel_sz);
+	if (!s->maps)
+		goto err;
+
+	s->maps[0].name = "perf_events";
+	s->maps[0].map = &obj->maps.perf_events;
+
+	s->maps[1].name = "records";
+	s->maps[1].map = &obj->maps.records;
+
+	/* programs */
+	s->prog_cnt = 2;
+	s->prog_skel_sz = sizeof(*s->progs);
+	s->progs = (typeof(s->progs))calloc(s->prog_cnt, s->prog_skel_sz);
+	if (!s->progs)
+		goto err;
+
+	s->progs[0].name = "syscall_enter_execve";
+	s->progs[0].prog = &obj->progs.syscall_enter_execve;
+	s->progs[0].link = &obj->links.syscall_enter_execve;
+
+	s->progs[1].name = "syscall_exit_execve";
+	s->progs[1].prog = &obj->progs.syscall_exit_execve;
+	s->progs[1].link = &obj->links.syscall_exit_execve;
+
+	s->data_sz = 2024;
+	s->data = (void *)"\
+\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\x28\x05\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0b\0\
+\x01\0\x85\0\0\0\x0e\0\0\0\xbf\x06\0\0\0\0\0\0\x63\x6a\xfc\xff\0\0\0\0\x85\0\0\
+\0\x0f\0\0\0\xbf\x07\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\
+\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\
+\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
+\xd0\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb7\x04\0\0\x01\0\0\0\x85\
+\0\0\0\x02\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x19\0\0\0\0\0\
+\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\0\x15\x08\x12\0\0\0\0\0\x77\x06\0\
+\0\x20\0\0\0\x61\xa1\xfc\xff\0\0\0\0\x63\x78\x1c\0\0\0\0\0\x63\x68\x14\0\0\0\0\
+\0\x63\x18\x10\0\0\0\0\0\x85\0\0\0\x23\0\0\0\x07\0\0\0\x78\x04\0\0\xbf\xa1\0\0\
+\0\0\0\0\x07\x01\0\0\xf0\xff\xff\xff\xb7\x02\0\0\x08\0\0\0\xbf\x03\0\0\0\0\0\0\
+\x85\0\0\0\x04\0\0\0\x07\x08\0\0\x18\0\0\0\x79\xa3\xf0\xff\0\0\0\0\x07\x03\0\0\
+\x6c\x04\0\0\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\
+\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\xbf\x16\0\0\0\0\0\0\x85\0\0\0\x0e\0\0\0\x63\
+\x0a\xfc\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x15\x07\x13\
+\0\0\0\0\0\x79\x61\x10\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\x6d\x12\x0b\0\0\0\0\0\xbf\
+\x71\0\0\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x85\0\0\0\x10\0\0\0\xbf\x61\0\0\0\0\0\0\
+\x18\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\x03\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\
+\0\xbf\x74\0\0\0\0\0\0\xb7\x05\0\0\x20\0\0\0\x85\0\0\0\x19\0\0\0\xbf\xa2\0\0\0\
+\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\
+\0\x03\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x01\0\0\0\x04\0\0\0\x20\0\0\0\
+\0\x28\0\0\0\0\0\0\x04\0\0\0\x04\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\
+\0\x06\x07\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xac\0\0\0\x04\
+\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdb\0\0\0\0\0\x03\0\x70\x01\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\xd4\0\0\0\0\0\x05\0\xc0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xcd\
+\0\0\0\0\0\x05\0\xe8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa3\0\0\0\x11\0\x08\0\0\0\0\
+\0\0\0\0\0\x04\0\0\0\0\0\0\0\x20\0\0\0\x11\0\x09\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\
+\0\0\0\x07\0\0\0\x11\0\x07\0\x14\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x18\0\0\0\x11\
+\0\x07\0\0\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x8e\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\
+\0\x80\x01\0\0\0\0\0\0\x51\0\0\0\x12\0\x05\0\0\0\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\
+\x70\0\0\0\0\0\0\0\x01\0\0\0\x08\0\0\0\xb8\0\0\0\0\0\0\0\x01\0\0\0\x08\0\0\0\
+\x28\0\0\0\0\0\0\0\x01\0\0\0\x08\0\0\0\x88\0\0\0\0\0\0\0\x01\0\0\0\x07\0\0\0\
+\xd0\0\0\0\0\0\0\0\x01\0\0\0\x08\0\0\0\0\x2e\x74\x65\x78\x74\0\x70\x65\x72\x66\
+\x5f\x65\x76\x65\x6e\x74\x73\0\x6d\x61\x70\x73\0\x72\x65\x63\x6f\x72\x64\x73\0\
+\x5f\x76\x65\x72\x73\x69\x6f\x6e\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\x6f\
+\x69\x6e\x74\x2f\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\x78\
+\x69\x74\x5f\x65\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\x65\x78\
+\x69\x74\x5f\x65\x78\x65\x63\x76\x65\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\
+\x6f\x69\x6e\x74\x2f\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\
+\x6e\x74\x65\x72\x5f\x65\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\
+\x65\x6e\x74\x65\x72\x5f\x65\x78\x65\x63\x76\x65\0\x5f\x6c\x69\x63\x65\x6e\x73\
+\x65\0\x65\x78\x65\x63\x73\x6e\x6f\x6f\x70\x5f\x6b\x65\x72\x6e\x2e\x63\0\x2e\
+\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x4c\x42\x42\x31\x5f\
+\x34\0\x4c\x42\x42\x31\x5f\x33\0\x4c\x42\x42\x30\x5f\x33\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbd\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\x40\x04\0\0\0\0\0\0\xe2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\x69\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\
+\0\x80\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x65\0\
+\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf0\x03\0\0\0\0\0\0\x20\0\0\0\0\
+\0\0\0\x0a\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x2d\0\0\0\x01\0\
+\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc0\x01\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x29\0\0\0\x09\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\x10\x04\0\0\0\0\0\0\x30\0\0\0\0\0\0\0\x0a\0\0\0\x05\0\0\
+\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x13\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\xb8\x02\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\xa4\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xe0\x02\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\x21\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe4\x02\0\0\0\
+\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc5\
+\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe8\x02\0\0\0\0\0\0\x08\x01\0\
+\0\0\0\0\0\x01\0\0\0\x05\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
+
+	return 0;
+err:
+	bpf_object__destroy_skeleton(s);
+	return -1;
+}
+
+#endif /* __EXECSNOOP_KERN_SKEL_H__ */

execsnoop-kernel/arm64.md

@@ -0,0 +1,82 @@
+##  Cross Compile
+
+```bash
+docker pull debian:buster
+
+# in container
+dpkg --add-architecture arm64
+apt update
+apt install gcc-8-aarch64-linux-gnu # cross compile toolchain
+apt install libelf-dev:arm64 # target depency library
+...
+```
+
+## Emulation
+
+### Register qemu-user-static
+
+- before register binfmt
+
+```bash
+docker run --rm -t arm64/ubuntu uname -m
+```
+
+- register binfmt
+
+```bash
+# through docker
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+# or through systemd
+pacman -S qemu-user-static binfmt-qemu-static
+systemctl restart systemd-binfmt.service
+# test
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
+```
+
+- after register binfmt
+
+```bash
+docker run --rm -t arm64/ubuntu uname -m
+```
+
+### M1: Docker
+
+```bash
+# start container background
+docker run -dit --name arm64 -v /home/fancy/workspace-xps:/data arm64v8/ubuntu
+
+# enter container
+docker exec -it arm64 bash
+```
+
+### M2: Chroot
+
+download image [ubuntu-base-20.04-base-arm64.tar.gz](http://cdimage.ubuntu.com/ubuntu-base/releases/20.04/release/ubuntu-base-20.04-base-arm64.tar.gz), extract and chroot to it.
+
+```bash
+sudo arch-chroot ubuntu-base-20.04-base-arm64
+```
+
+### Refer
+
+- https://www.stereolabs.com/docs/docker/building-arm-container-on-x86/
+
+- https://github.com/junaruga/fedora-workshop-multiarch/blob/master/slides/Lets-add-Fedora-multiarch-to-CI.pdf
+
+- https://wiki.debian.org/QemuUserEmulation
+
+### Compile
+
+ready some depencies.
+
+```bash
+# maybe repository: https://mirrors.tuna.tsinghua.edu.cn/help/ubuntu/
+# install in container for kernel bpf build
+apt install dialog apt-utils
+apt install build-essential gcc clang llvm
+apt install bison flex bc rsync libssl-dev binutils-dev libreadline-dev libelf-dev
+apt install make cmake
+# for cgproxy
+apt install nlohmann-json3-dev rpm
+```
+

execsnoop-kernel/execsnoop_kern.c

@@ -1,7 +1,11 @@
+#ifdef USE_VMLINUX
+#include "vmlinux.h"
+#else
 #include "linux/sched.h"
-#include <linux/version.h>
 #include <linux/ptrace.h>
 #include <uapi/linux/bpf.h>
+#endif
+#include <linux/version.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
 
@@ -34,7 +38,7 @@ struct bpf_map_def SEC("maps") records = {
     .type = BPF_MAP_TYPE_HASH,
     .key_size = sizeof(pid_t),
     .value_size = sizeof(struct event),
-    .max_entries = 1024,
+    .max_entries = 10240,
 };
 struct bpf_map_def SEC("maps") perf_events = {
     .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,

execsnoop-kernel/execsnoop_share.cpp

@@ -1,9 +1,15 @@
+#include "execsnoop_share.h"
 
+#include <errno.h>
 #include <signal.h>
 #include <bpf/libbpf.h>
 #include <sys/resource.h>
-#include "execsnoop_kern_skel.h"
-#include "execsnoop_share.h"
+
+#if defined(__x86_64__)
+	#include "x86_64/execsnoop_kern_skel.h"
+#elif defined(__aarch64__)
+	#include "aarch64/execsnoop_kern_skel.h"
+#endif
 
 namespace CGPROXY::EXECSNOOP {
 
@@ -38,6 +44,7 @@ int execsnoop() {
 	struct perf_buffer_opts pb_opts = {};
 	struct perf_buffer *pb;
 	int err;
+	bool notified=false;
 	
 	err = bump_memlock_rlimit();
 	if (err) {
@@ -57,6 +64,7 @@ int execsnoop() {
 		return err;
 	}
 
+main_loop:
 	pb_opts.sample_cb = handle_event;
 	pb_opts.lost_cb = handle_lost_events;
 	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
@@ -67,9 +75,14 @@ int execsnoop() {
 	}
 
 	// notify
-  	status.set_value();
+  	if (!notified) {status.set_value(); notified=true;}
 
 	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	perf_buffer__free(pb);
+	/* handle Interrupted system call when sleep */
+	if (err == -EINTR) goto main_loop;
+
+	perror("perf_buffer__poll");
 	kill(0, SIGINT);
 	return err;
 }

execsnoop-kernel/execsnoop_share.h

@@ -10,9 +10,11 @@ namespace CGPROXY::EXECSNOOP {
 
 extern "C" void startThread(function<int(int)> c, promise<void> _status);
 
+#ifdef BUIlD_EXECSNOOP_DL
 // only for dlsym()
 using startThread_t=decltype(startThread);
 startThread_t *_startThread;
+#endif
 
 } // namespace CGPROXY::EXECSNOOP
 #endif

execsnoop-kernel/execsnoop_user_1.c

@@ -1,9 +1,12 @@
-
 #include <signal.h>
 #include <bpf/libbpf.h>
 #include <sys/resource.h>
-#include "execsnoop_kern_skel.h"
-// #include "bpf_load.h"
+
+#if defined(__x86_64__)
+	#include "x86_64/execsnoop_kern_skel.h"
+#elif defined(__aarch64__)
+	#include "aarch64/execsnoop_kern_skel.h"
+#endif
 
 #define TASK_COMM_LEN 16
 struct event {

execsnoop-kernel/readme.md

@@ -2,20 +2,28 @@
 ## Prons and cons
 
 - use stable execve tracepoint, so build once and should work everywhere
-- need to build in kernel tree
+- build in kernel tree or build with VMLINUX
 
-## Build in kernel tree
 
+
+## Build `execsnoop_kern.o`
+
+### M1: Build in kernel tree
+
+- download kernel source code
 - ready and config kernel tree
 
 ```bash
 # kernel config
-gunzip -c /proc/config.gz > .config
-make oldconfig && make prepare
+#gunzip -c /proc/config.gz > .config
+#make oldconfig && make prepare
+make defconfig && make prepare
 # install headers to ./usr/include
 make headers_install -j8
-# build bpf
+# build samples/bpf
 make M=samples/bpf -j8
+# build bpftool
+make tools/bpf -j8
 ```
 
 - put or link `execsnoop_kern.c` and `execsnoop_user.c` to *samples/bpf/*
@@ -28,20 +36,26 @@ execsnoop-objs := bpf_load.o execsnoop_user.o $(TRACE_HELPERS)
 always-y += execsnoop_kern.o
 ```
 
-## Run
+- compile again
+
+```
+make M=samples/bpf -j8
+```
+
+- run test
 
 ```bash
 cd samples/bpf
 sudo bash -c "ulimit -l unlimited && ./execsnoop"
 ```
 
-## Detail build command
+
+**Detail build command**
 
 using `make V=1 M=samples/bpf | tee -a log.txt` to get and filter following command
 
 - build `execsnoop_kern.o`
 
-  note `-g` is needed if with BPF CO-RE
 
 ```bash
 clang -nostdinc \
@@ -65,55 +79,120 @@ clang -nostdinc \
 	-Wno-unknown-warning-option  \
 	-fno-stack-protector \
 	-O2 -emit-llvm -c samples/bpf/execsnoop_kern.c \
-	-o - | llc -march=bpf  -filetype=obj -o samples/bpf/execsnoop_kern.o
-```
-
-- build `execsnoop_user.o`
-
-```bash
-  gcc -Wall -O2 -Wmissing-prototypes -Wstrict-prototypes \
-  	-I./usr/include \
-  	-I./tools/testing/selftests/bpf/ \
-  	-I./tools/lib/ \
-  	-I./tools/include \
-  	-I./tools/perf \
-  	-DHAVE_ATTR_TEST=0  \
-  	-c -o samples/bpf/execsnoop_user.o samples/bpf/execsnoop_user.c
-```
-
-- build `execsnoop`
-
-```bash
-  gcc -Wall -O2 -Wmissing-prototypes -Wstrict-prototypes \
-	  -I./usr/include \
-	  -I./tools/testing/selftests/bpf/ \
-	  -I./tools/lib/ \
-	  -I./tools/include \
-	  -I./tools/perf \
-	  -DHAVE_ATTR_TEST=0 \
-	  -o samples/bpf/execsnoop \
-	  samples/bpf/bpf_load.o samples/bpf/execsnoop_user.o \
-	  tools/testing/selftests/bpf/trace_helpers.o tools/lib/bpf/libbpf.a \
-	  -lelf -lz
+	-o - | llc -march=bpf -filetype=obj -o samples/bpf/execsnoop_kern.o
 ```
 
 
 
-## With bpftool
+### M2: Build with VMLINUX
 
-- gen
+- get `vmlinux.h`
+
+```
+bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
+```
+
+- compile
+
+  note `-g` is needed if with BPF CO-RE
+
+```
+clang -O2 -target bpf -DUSE_VMLINUX -c execsnoop_kern.c -o execsnoop_kern.o
+```
+
+## Generate `execsnoop_kern_skel.h`
+
+- generate `execsnoop_kern_skel.h`
 
 ```
 bpftool gen skeleton execsnoop_kern.o > execsnoop_kern_skel.h
 ```
 
-- build
+- build execsnoop
 
 ```
 gcc -Wall -O2 execsnoop_user_1.c -o execsnoop -lbpf
 ```
 
-## Some resources
+
+
+## Multiarch build
+
+- Cross compile, fast, but library link can be mess
+  - aarch64-linux-gnu-gcc
+- Emulation, the easist, but with perfomance cost
+  - qemu-user-static + binfmt-qemu-static + docker/chroot
+- see `arm64.md` to see how to setup
+
+
+
+```bash
+# if cross compile
+export ARCH=arm64
+export CROSS_COMPILE=aarch64-linux-gnu-
+export SYSROOT=/home/fancy/workspace-xps/linux/ArchLinuxARM-aarch64-latest
+export C_INCLUDE_PATH=$SYSROOT/usr/include
+```
+
+-  edit `tools/lib/bpf/makefile` #192 to:
+
+```makefile
+$(OUTPUT)libbpf.so.$(LIBBPF_VERSION): $(BPF_IN_SHARED)
+     $(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS)
+```
+
+- make
+
+```bash
+# clean
+make mrproper
+make clean
+make -C tools clean
+make -C samples/bpf clean
+# make
+make defconfig && make prepare
+make headers_install -j8
+# build samples/bpf
+make M=samples/bpf -j8
+# build bpftool
+make tools/bpf -j8
+```
+
+- detail build `execsnoop_kern.o`
+
+
+```bash
+clang  -nostdinc \
+	-isystem /usr/lib/gcc/aarch64-linux-gnu/9/include \
+	-I./arch/arm64/include -I./arch/arm64/include/generated  \
+	-I./include -I./arch/arm64/include/uapi \
+	-I./arch/arm64/include/generated/uapi \
+	-I./include/uapi \
+	-I./include/generated/uapi \
+	-include ./include/linux/kconfig.h \
+    -I./samples/bpf \
+    -I./tools/testing/selftests/bpf/ \
+    -I./tools/lib/ \
+    -include asm_goto_workaround.h \
+    -D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \
+    -D__TARGET_ARCH_arm64 -Wno-compare-distinct-pointer-types \
+    -Wno-gnu-variable-sized-type-not-at-end \
+    -Wno-address-of-packed-member -Wno-tautological-compare \
+    -Wno-unknown-warning-option  \
+    -fno-stack-protector \
+    -O2 -emit-llvm -c samples/bpf/execsnoop_kern.c \
+    -o -| llc -march=bpf  -filetype=obj -o samples/bpf/execsnoop_kern.o
+```
+
+- generate
+
+```
+bpftool gen skeleton execsnoop_kern.o > aarch64/execsnoop_kern_skel.h
+```
+
+
+
+## Refer
 
 - [A thorough introduction to eBPF](https://lwn.net/Articles/740157/)
-- [Write eBPF program in pure C](http://terenceli.github.io/%E6%8A%80%E6%9C%AF/2020/01/18/ebpf-in-c)
+

execsnoop-kernel/x86_64/execsnoop_kern_skel.h

@@ -150,10 +150,10 @@ execsnoop_kern__create_skeleton(struct execsnoop_kern *obj)
 \xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\
 \0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\0\x15\x08\x12\0\0\0\0\0\x77\x06\0\
 \0\x20\0\0\0\x61\xa1\xfc\xff\0\0\0\0\x63\x78\x1c\0\0\0\0\0\x63\x68\x14\0\0\0\0\
-\0\x63\x18\x10\0\0\0\0\0\x85\0\0\0\x23\0\0\0\x07\0\0\0\x18\x05\0\0\xbf\xa1\0\0\
+\0\x63\x18\x10\0\0\0\0\0\x85\0\0\0\x23\0\0\0\x07\0\0\0\xa0\x04\0\0\xbf\xa1\0\0\
 \0\0\0\0\x07\x01\0\0\xf0\xff\xff\xff\xb7\x02\0\0\x08\0\0\0\xbf\x03\0\0\0\0\0\0\
 \x85\0\0\0\x04\0\0\0\x07\x08\0\0\x18\0\0\0\x79\xa3\xf0\xff\0\0\0\0\x07\x03\0\0\
-\x0c\x05\0\0\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\
+\x94\x04\0\0\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\
 \0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\xbf\x16\0\0\0\0\0\0\x85\0\0\0\x0e\0\0\0\x63\
 \x0a\xfc\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\
 \0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x15\x07\x13\
@@ -163,7 +163,7 @@ execsnoop_kern__create_skeleton(struct execsnoop_kern *obj)
 \0\xbf\x74\0\0\0\0\0\0\xb7\x05\0\0\x20\0\0\0\x85\0\0\0\x19\0\0\0\xbf\xa2\0\0\0\
 \0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\
 \0\x03\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x01\0\0\0\x04\0\0\0\x20\0\0\0\
-\0\x04\0\0\0\0\0\0\x04\0\0\0\x04\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\
+\0\x28\0\0\0\0\0\0\x04\0\0\0\x04\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\
 \0\x06\x07\x05\0\x10\0\0\0\0\0\0\0\x01\x7a\x52\0\x08\x7c\x0b\x01\x0c\0\0\0\x18\
 \0\0\0\x18\0\0\0\0\0\0\0\0\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\x34\0\0\
 \0\0\0\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\

execsnoop-libbpf/CMakeLists.txt

@@ -0,0 +1,21 @@
+find_library(LIBBPF bpf REQUIRED)
+
+# generate execsnoop.skel.h
+if (EXISTS /sys/kernel/btf/vmlinux)
+add_custom_command(OUTPUT execsnoop.skel.h
+    COMMAND bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
+    COMMAND clang -O2 -g -target bpf -c execsnoop.bpf.c -o execsnoop.bpf.o
+    COMMAND bpftool gen skeleton execsnoop.bpf.o > execsnoop.skel.h
+    DEPENDS execsnoop.bpf.c
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+endif()
+
+if (build_execsnoop_dl)
+    add_library(execsnoop MODULE execsnoop_share.cpp execsnoop.skel.h)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF})
+    install(TARGETS execsnoop DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cgproxy/)
+else()
+    add_library(execsnoop execsnoop_share.cpp execsnoop.skel.h)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF})
+endif()

execsnoop-libbpf/Makefile

@@ -1,87 +0,0 @@
-# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
-OUTPUT := $(abspath build)
-CLANG ?= clang
-CFLAGS ?= -g -O2 -Wall
-LLVM_STRIP ?= llvm-strip
-BPFTOOL ?= bpftool
-LIBBPF_DIR := $(abspath libbpf/src)
-LIBBPF_OBJ := $(OUTPUT)/usr/lib64/libbpf.a
-LIBBPF_H := $(OUTPUT)/usr/include
-INCLUDES := -I$(OUTPUT) -I$(LIBBPF_H)
-ARCH := $(shell uname -m | sed 's/x86_64/x86/')
-
-APPS = execsnoop
-LIBSO = libexecsnoop.so
-
-.PHONY: all
-all: $(APPS) $(LIBSO)
-
-ifeq ($(V),1)
-Q =
-msg =
-else
-Q = @
-msg = @printf '  %-8s %s%s\n' "$(1)" "$(notdir $(2))" "$(if $(3), $(3))";
-MAKEFLAGS += --no-print-directory
-endif
-
-COMMON_OBJ = \
-	$(OUTPUT)/trace_helpers.o \
-	$(OUTPUT)/syscall_helpers.o \
-	$(OUTPUT)/errno_helpers.o \
-
-
-.PHONY: install
-install: $(LIBSO)
-	install -D $(LIBSO) -t $(DESTDIR)/usr/lib/cgproxy/
-
-.PHONY: clean
-clean:
-	$(call msg,CLEAN)
-	$(Q)rm -rf $(OUTPUT) $(APPS) $(LIBSO)
-
-$(OUTPUT) $(OUTPUT)/libbpf:
-	$(call msg,MKDIR,$@)
-	$(Q)mkdir -p $@
-
-$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) $(COMMON_OBJ) | $(OUTPUT)
-	$(call msg,BINARY,$@)
-	$(Q)$(CC) $(CFLAGS) $^ -lelf -lz -o $@
-
-$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
-
-$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
-	$(call msg,CC,$@)
-	$(Q)$(CC) $(CFLAGS) $(INCLUDES) -fPIC -c $(filter %.c,$^) -o $@
-
-$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT)
-	$(call msg,GEN-SKEL,$@)
-	$(Q)$(BPFTOOL) gen skeleton $< > $@
-
-$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) vmlinux.h | $(OUTPUT)
-	$(call msg,BPF,$@)
-	$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH)		      \
-		     $(INCLUDES) -c $(filter %.c,$^) -o $@ &&		      \
-	$(LLVM_STRIP) -g $@
-
-vmlinux.h:
-	$(Q)$(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
-
-$(LIBBPF_OBJ):
-	$(Q)cd $(LIBBPF_DIR) && make DESTDIR=${OUTPUT} prefix=$(OUTPUT) CPPFLAGS="-fPIC" install install_headers
-
-# patch: | $(LIBBPF_DIR)/Makefile
-# 	@echo "patching libbpf-fPIC.patch"
-# 	$(Q)cd libbpf/src && patch -p1 --forward -i ../../libbpf-fPIC.patch || true
-
-# build libexecsnoop.so
-$(LIBSO): execsnoop_share.cpp $(OUTPUT)/execsnoop.skel.h $(LIBBPF_OBJ) $(COMMON_OBJ) | $(OUTPUT) 
-	$(call msg,LIB,$@)
-	$(Q)$(CXX) -std=c++17 $(CFLAGS) $(INCLUDES) -fPIC $< $(COMMON_OBJ) -shared -lelf -lz -Wl,--no-whole-archive,--no-undefined $(LIBBPF_OBJ) -o $@
-
-
-# delete failed targets
-.DELETE_ON_ERROR:
-# keep intermediate (.skel.h, .bpf.o, etc) targets
-.SECONDARY:
-

execsnoop-libbpf/errno_helpers.c

@@ -1,232 +0,0 @@
-// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
-// Copyright (c) 2020 Anton Protopopov
-#include <stdlib.h>
-#include <limits.h>
-#include <string.h>
-#include <errno.h>
-#include <stdio.h>
-
-#define warn(...) fprintf(stderr, __VA_ARGS__)
-
-#ifdef __x86_64__
-static int errno_by_name_x86_64(const char *errno_name)
-{
-
-#define strcase(X, N) if (!strcmp(errno_name, (X))) return N
-
-	strcase("EPERM", 1);
-	strcase("ENOENT", 2);
-	strcase("ESRCH", 3);
-	strcase("EINTR", 4);
-	strcase("EIO", 5);
-	strcase("ENXIO", 6);
-	strcase("E2BIG", 7);
-	strcase("ENOEXEC", 8);
-	strcase("EBADF", 9);
-	strcase("ECHILD", 10);
-	strcase("EAGAIN", 11);
-	strcase("EWOULDBLOCK", 11);
-	strcase("ENOMEM", 12);
-	strcase("EACCES", 13);
-	strcase("EFAULT", 14);
-	strcase("ENOTBLK", 15);
-	strcase("EBUSY", 16);
-	strcase("EEXIST", 17);
-	strcase("EXDEV", 18);
-	strcase("ENODEV", 19);
-	strcase("ENOTDIR", 20);
-	strcase("EISDIR", 21);
-	strcase("EINVAL", 22);
-	strcase("ENFILE", 23);
-	strcase("EMFILE", 24);
-	strcase("ENOTTY", 25);
-	strcase("ETXTBSY", 26);
-	strcase("EFBIG", 27);
-	strcase("ENOSPC", 28);
-	strcase("ESPIPE", 29);
-	strcase("EROFS", 30);
-	strcase("EMLINK", 31);
-	strcase("EPIPE", 32);
-	strcase("EDOM", 33);
-	strcase("ERANGE", 34);
-	strcase("EDEADLK", 35);
-	strcase("EDEADLOCK", 35);
-	strcase("ENAMETOOLONG", 36);
-	strcase("ENOLCK", 37);
-	strcase("ENOSYS", 38);
-	strcase("ENOTEMPTY", 39);
-	strcase("ELOOP", 40);
-	strcase("ENOMSG", 42);
-	strcase("EIDRM", 43);
-	strcase("ECHRNG", 44);
-	strcase("EL2NSYNC", 45);
-	strcase("EL3HLT", 46);
-	strcase("EL3RST", 47);
-	strcase("ELNRNG", 48);
-	strcase("EUNATCH", 49);
-	strcase("ENOCSI", 50);
-	strcase("EL2HLT", 51);
-	strcase("EBADE", 52);
-	strcase("EBADR", 53);
-	strcase("EXFULL", 54);
-	strcase("ENOANO", 55);
-	strcase("EBADRQC", 56);
-	strcase("EBADSLT", 57);
-	strcase("EBFONT", 59);
-	strcase("ENOSTR", 60);
-	strcase("ENODATA", 61);
-	strcase("ETIME", 62);
-	strcase("ENOSR", 63);
-	strcase("ENONET", 64);
-	strcase("ENOPKG", 65);
-	strcase("EREMOTE", 66);
-	strcase("ENOLINK", 67);
-	strcase("EADV", 68);
-	strcase("ESRMNT", 69);
-	strcase("ECOMM", 70);
-	strcase("EPROTO", 71);
-	strcase("EMULTIHOP", 72);
-	strcase("EDOTDOT", 73);
-	strcase("EBADMSG", 74);
-	strcase("EOVERFLOW", 75);
-	strcase("ENOTUNIQ", 76);
-	strcase("EBADFD", 77);
-	strcase("EREMCHG", 78);
-	strcase("ELIBACC", 79);
-	strcase("ELIBBAD", 80);
-	strcase("ELIBSCN", 81);
-	strcase("ELIBMAX", 82);
-	strcase("ELIBEXEC", 83);
-	strcase("EILSEQ", 84);
-	strcase("ERESTART", 85);
-	strcase("ESTRPIPE", 86);
-	strcase("EUSERS", 87);
-	strcase("ENOTSOCK", 88);
-	strcase("EDESTADDRREQ", 89);
-	strcase("EMSGSIZE", 90);
-	strcase("EPROTOTYPE", 91);
-	strcase("ENOPROTOOPT", 92);
-	strcase("EPROTONOSUPPORT", 93);
-	strcase("ESOCKTNOSUPPORT", 94);
-	strcase("ENOTSUP", 95);
-	strcase("EOPNOTSUPP", 95);
-	strcase("EPFNOSUPPORT", 96);
-	strcase("EAFNOSUPPORT", 97);
-	strcase("EADDRINUSE", 98);
-	strcase("EADDRNOTAVAIL", 99);
-	strcase("ENETDOWN", 100);
-	strcase("ENETUNREACH", 101);
-	strcase("ENETRESET", 102);
-	strcase("ECONNABORTED", 103);
-	strcase("ECONNRESET", 104);
-	strcase("ENOBUFS", 105);
-	strcase("EISCONN", 106);
-	strcase("ENOTCONN", 107);
-	strcase("ESHUTDOWN", 108);
-	strcase("ETOOMANYREFS", 109);
-	strcase("ETIMEDOUT", 110);
-	strcase("ECONNREFUSED", 111);
-	strcase("EHOSTDOWN", 112);
-	strcase("EHOSTUNREACH", 113);
-	strcase("EALREADY", 114);
-	strcase("EINPROGRESS", 115);
-	strcase("ESTALE", 116);
-	strcase("EUCLEAN", 117);
-	strcase("ENOTNAM", 118);
-	strcase("ENAVAIL", 119);
-	strcase("EISNAM", 120);
-	strcase("EREMOTEIO", 121);
-	strcase("EDQUOT", 122);
-	strcase("ENOMEDIUM", 123);
-	strcase("EMEDIUMTYPE", 124);
-	strcase("ECANCELED", 125);
-	strcase("ENOKEY", 126);
-	strcase("EKEYEXPIRED", 127);
-	strcase("EKEYREVOKED", 128);
-	strcase("EKEYREJECTED", 129);
-	strcase("EOWNERDEAD", 130);
-	strcase("ENOTRECOVERABLE", 131);
-	strcase("ERFKILL", 132);
-	strcase("EHWPOISON", 133);
-
-#undef strcase
-
-	return -1;
-
-}
-#endif
-
-/* Try to find the errno number using the errno(1) program */
-static int errno_by_name_dynamic(const char *errno_name)
-{
-	int len = strlen(errno_name);
-	int err, number = -1;
-	char buf[128];
-	char cmd[64];
-	char *end;
-	long val;
-	FILE *f;
-
-	/* sanity check to not call popen with random input */
-	for (int i = 0; i < len; i++) {
-		if (errno_name[i] < 'A' || errno_name[i] > 'Z') {
-			warn("errno_name contains invalid char 0x%02x: %s\n",
-					errno_name[i], errno_name);
-			return -1;
-		}
-	}
-
-	snprintf(cmd, sizeof(cmd), "errno %s", errno_name);
-	f = popen(cmd, "r");
-	if (!f) {
-		warn("popen: %s: %s\n", cmd, strerror(errno));
-		return -1;
-	}
-
-	if (!fgets(buf, sizeof(buf), f)) {
-		goto close;
-	} else if (ferror(f)) {
-		warn("fgets: %s\n", strerror(errno));
-		goto close;
-	}
-
-	// expecting "<name> <number> <description>"
-	if (strncmp(errno_name, buf, len) || strlen(buf) < len+2) {
-		warn("expected '%s': %s\n", errno_name, buf);
-		goto close;
-	}
-	errno = 0;
-	val = strtol(buf+len+2, &end, 10);
-	if (errno || end == (buf+len+2) || number < 0 || number > INT_MAX) {
-		warn("can't parse the second column, expected int: %s\n", buf);
-		goto close;
-	}
-	number = val;
-
-close:
-	err = pclose(f);
-	if (err < 0)
-		warn("pclose: %s\n", strerror(errno));
-#ifndef __x86_64__
-	/* Ignore the error for x86_64 where we have a table compiled in */
-	else if (err && WEXITSTATUS(err) == 127) {
-		warn("errno(1) required for errno name/number mapping\n");
-	} else if (err) {
-		warn("errno(1) exit status (see wait(2)): 0x%x\n", err);
-	}
-#endif
-	return number;
-}
-
-int errno_by_name(const char *errno_name)
-{
-#ifdef __x86_64__
-	int err;
-
-	err = errno_by_name_x86_64(errno_name);
-	if (err >= 0)
-		return err;
-#endif
-
-	return errno_by_name_dynamic(errno_name);
-}

execsnoop-libbpf/errno_helpers.h

@@ -1,7 +0,0 @@
-/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
-#ifndef __ERRNO_HELPERS_H
-#define __ERRNO_HELPERS_H
-
-int errno_by_name(const char *errno_name);
-
-#endif /* __ERRNO_HELPERS_H */

execsnoop-libbpf/execsnoop.bpf.c

@@ -1,128 +1,92 @@
 #include "vmlinux.h"
+#include <linux/version.h>
 #include <bpf/bpf_helpers.h>
-#include <bpf/bpf_core_read.h>
-#include "execsnoop.h"
+#include <bpf/bpf_tracing.h>
 
-const volatile bool ignore_failed = true;
-const volatile uid_t targ_uid = INVALID_UID;
-const volatile int max_args = DEFAULT_MAXARGS;
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
 
-static const struct event empty_event = {};
+/* /sys/kernel/debug/tracing/events/syscalls/sys_enter_execve/format */
+struct syscalls_enter_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	const char filename_ptr;
+    const char *const * argv;
+    const char *const * envp;
+};
+
+/* /sys/kernel/debug/tracing/events/syscalls/sys_exit_execve/format */
+struct syscalls_exit_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	long ret;
+};
 
 struct {
-	__uint(type, BPF_MAP_TYPE_HASH);
-	__uint(max_entries, 10240);
-	__type(key, pid_t);
-	__type(value, struct event);
-} execs SEC(".maps");
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 10240);
+    __type(key, pid_t);
+    __type(value, struct event);
+} records SEC(".maps");
 
 struct {
-	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
-	__uint(key_size, sizeof(u32));
-	__uint(value_size, sizeof(u32));
-} events SEC(".maps");
-
-static __always_inline bool valid_uid(uid_t uid) {
-	return uid != INVALID_UID;
-}
+    __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
+    __uint(max_entries, 128);
+    __type(key, u32);
+    __type(value, u32);
+} perf_events SEC(".maps");
 
 SEC("tracepoint/syscalls/sys_enter_execve")
-int tracepoint__syscalls__sys_enter_execve(struct trace_event_raw_sys_enter* ctx)
-{
-	u64 id;
-	pid_t pid, tgid;
-	unsigned int ret;
+int syscall_enter_execve(struct syscalls_enter_execve_args *ctx){
+    pid_t pid, tgid; uid_t uid;
 	struct event *event;
-	struct task_struct *task;
-	const char **args = (const char **)(ctx->args[1]);
-	const char *argp;
-	uid_t uid = (u32)bpf_get_current_uid_gid();
+	struct task_struct *task, *task_p;
 
-	if (valid_uid(targ_uid) && targ_uid != uid)
-		return 0;
-
-	id = bpf_get_current_pid_tgid();
+    u64 id = bpf_get_current_pid_tgid();
 	pid = (pid_t)id;
 	tgid = id >> 32;
-	if (bpf_map_update_elem(&execs, &pid, &empty_event, BPF_NOEXIST))
-		return 0;
+	uid = (u32)bpf_get_current_uid_gid();
 
-	event = bpf_map_lookup_elem(&execs, &pid);
-	if (!event)
-		return 0;
+	struct event empty_event={};
+	if (bpf_map_update_elem(&records, &pid, &empty_event, BPF_NOEXIST)!=0) return 0;
+	event = bpf_map_lookup_elem(&records, &pid);
+	if (!event) return 0;
 
 	event->pid = pid;
 	event->tgid = tgid;
 	event->uid = uid;
+
+	/* ppid is not reliable here, normal in arch, but become 0 in ubuntu 20.04 */
 	task = (struct task_struct*)bpf_get_current_task();
-	event->ppid = (pid_t)BPF_CORE_READ(task, real_parent, tgid);
-	event->args_count = 0;
-	event->args_size = 0;
+	bpf_probe_read(&task_p, sizeof(struct task_struct*),&task->real_parent);
+	bpf_probe_read(&event->ppid,sizeof(pid_t),&task_p->tgid);
 
-	ret = bpf_probe_read_str(event->args, ARGSIZE, (const char*)ctx->args[0]);
-	if (ret <= ARGSIZE) {
-		event->args_size += ret;
-	} else {
-		/* write an empty string */
-		event->args[0] = '\0';
-		event->args_size++;
-	}
-
-	event->args_count++;
-	#pragma unroll
-	for (int i = 1; i < TOTAL_MAX_ARGS && i < max_args; i++) {
-		bpf_probe_read(&argp, sizeof(argp), &args[i]);
-		if (!argp)
-			return 0;
-
-		if (event->args_size > LAST_ARG)
-			return 0;
-
-		ret = bpf_probe_read_str(&event->args[event->args_size], ARGSIZE, argp);
-		if (ret > ARGSIZE)
-			return 0;
-
-		event->args_count++;
-		event->args_size += ret;
-	}
-	/* try to read one more argument to check if there is one */
-	bpf_probe_read(&argp, sizeof(argp), &args[max_args]);
-	if (!argp)
-		return 0;
-
-	/* pointer to max_args+1 isn't null, asume we have more arguments */
-	event->args_count++;
-	return 0;
+    return 0;
 }
 
 SEC("tracepoint/syscalls/sys_exit_execve")
-int tracepoint__syscalls__sys_exit_execve(struct trace_event_raw_sys_exit* ctx)
-{
-	u64 id;
+int syscall_exit_execve(struct syscalls_exit_execve_args *ctx){
 	pid_t pid;
-	int ret;
 	struct event *event;
-	u32 uid = (u32)bpf_get_current_uid_gid();
 
-	if (valid_uid(targ_uid) && targ_uid != uid)
-		return 0;
-	id = bpf_get_current_pid_tgid();
-	pid = (pid_t)id;
-	event = bpf_map_lookup_elem(&execs, &pid);
-	if (!event)
-		return 0;
-	ret = ctx->ret;
-	if (ignore_failed && ret < 0)
-		goto cleanup;
+	pid = (pid_t)bpf_get_current_pid_tgid();
+	event = bpf_map_lookup_elem(&records,&pid);
+	if (!event) return 0;
+	if (ctx->ret < 0) goto cleanup;
 
-	event->retval = ret;
-	bpf_get_current_comm(&event->comm, sizeof(event->comm));
-	size_t len = EVENT_SIZE(event);
-	if (len <= sizeof(*event))
-		bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, event, len);
+	/* get comm */
+	bpf_get_current_comm(&event->comm,sizeof(event->comm));
+	bpf_perf_event_output(ctx, &perf_events, BPF_F_CURRENT_CPU, event, sizeof(struct event));
 cleanup:
-	bpf_map_delete_elem(&execs, &pid);
+	bpf_map_delete_elem(&records, &pid);
 	return 0;
 }
 
-char LICENSE[] SEC("license") = "GPL";
+char _license[] SEC("license") = "GPL";
+u32 _version SEC("version") = LINUX_VERSION_CODE;

execsnoop-libbpf/execsnoop.c

@@ -1,329 +1,71 @@
-// Based on execsnoop(8) from BCC by Brendan Gregg and others.
-//
-#include <argp.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <time.h>
-#include <unistd.h>
+#include <signal.h>
 #include <bpf/libbpf.h>
-#include <bpf/bpf.h>
-#include "execsnoop.h"
+#include <sys/resource.h>
 #include "execsnoop.skel.h"
-#include "trace_helpers.h"
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
 
 #define PERF_BUFFER_PAGES   64
-#define NSEC_PRECISION (NSEC_PER_SEC / 1000)
-#define MAX_ARGS_KEY 259
 
-static struct env {
-	bool time;
-	bool timestamp;
-	bool fails;
-	uid_t uid;
-	bool quote;
-	const char *name;
-	const char *line;
-	bool print_uid;
-	bool verbose;
-	int max_args;
-} env = {
-	.max_args = DEFAULT_MAXARGS,
-	.uid = INVALID_UID
-};
-
-static struct timespec start_time;
-
-const char *argp_program_version = "execsnoop 0.1";
-const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
-const char argp_program_doc[] =
-"Trace open family syscalls\n"
-"\n"
-"USAGE: execsnoop [-h] [-T] [-t] [-x] [-u UID] [-q] [-n NAME] [-l LINE] [-U]\n"
-"                 [--max-args MAX_ARGS]\n"
-"\n"
-"EXAMPLES:\n"
-"   ./execsnoop           # trace all exec() syscalls\n"
-"   ./execsnoop -x        # include failed exec()s\n"
-"   ./execsnoop -T        # include time (HH:MM:SS)\n"
-"   ./execsnoop -U        # include UID\n"
-"   ./execsnoop -u 1000   # only trace UID 1000\n"
-"   ./execsnoop -t        # include timestamps\n"
-"   ./execsnoop -q        # add \"quotemarks\" around arguments\n"
-"   ./execsnoop -n main   # only print command lines containing \"main\"\n"
-"   ./execsnoop -l tpkg   # only print command where arguments contains \"tpkg\"";
-
-static const struct argp_option opts[] = {
-	{ "time", 'T', NULL, 0, "include time column on output (HH:MM:SS)"},
-	{ "timestamp", 't', NULL, 0, "include timestamp on output"},
-	{ "fails", 'x', NULL, 0, "include failed exec()s"},
-	{ "uid", 'u', "UID", 0, "trace this UID only"},
-	{ "quote", 'q', NULL, 0, "Add quotemarks (\") around arguments"},
-	{ "name", 'n', "NAME", 0, "only print commands matching this name, any arg"},
-	{ "line", 'l', "LINE", 0, "only print commands where arg contains this line"},
-	{ "print-uid", 'U', NULL, 0, "print UID column"},
-	{ "max-args", MAX_ARGS_KEY, "MAX_ARGS", 0,
-		"maximum number of arguments parsed and displayed, defaults to 20"},
-	{ "verbose", 'v', NULL, 0, "Verbose debug output" },
-	{},
-};
-
-static error_t parse_arg(int key, char *arg, struct argp_state *state)
-{
-	long int uid, max_args;
-
-	switch (key) {
-	case 'h':
-		argp_usage(state);
-		break;
-	case 'T':
-		env.time = true;
-		break;
-	case 't':
-		env.timestamp = true;
-		break;
-	case 'x':
-		env.fails = true;
-		break;
-	case 'u':
-		errno = 0;
-		uid = strtol(arg, NULL, 10);
-		if (errno || uid < 0 || uid >= INVALID_UID) {
-			fprintf(stderr, "Invalid UID %s\n", arg);
-			argp_usage(state);
-		}
-		env.uid = uid;
-		break;
-	case 'q':
-		env.quote = true;
-		break;
-	case 'n':
-		env.name = arg;
-		break;
-	case 'l':
-		env.line = arg;
-		break;
-	case 'U':
-		env.print_uid = true;
-		break;
-	case 'v':
-		env.verbose = true;
-		break;
-	case MAX_ARGS_KEY:
-		errno = 0;
-		max_args = strtol(arg, NULL, 10);
-		if (errno || max_args < 1 || max_args > TOTAL_MAX_ARGS) {
-			fprintf(stderr, "Invalid MAX_ARGS %s, should be in [1, %d] range\n",
-					arg, TOTAL_MAX_ARGS);
-
-			argp_usage(state);
-		}
-		env.max_args = max_args;
-		break;
-	default:
-		return ARGP_ERR_UNKNOWN;
-	}
-	return 0;
+static void handle_event(void *ctx, int cpu, void *data, __u32 size) {
+  	auto e = static_cast<event*>(data);
+    printf("comm: %s, pid: %d, tgid: %d, ppid: %d, uid: %d\n",e->comm,e->pid,e->tgid,e->ppid,e->uid);
 }
 
-int libbpf_print_fn(enum libbpf_print_level level,
-		    const char *format, va_list args)
-{
-	if (level == LIBBPF_DEBUG && !env.verbose)
-		return 0;
-	return vfprintf(stderr, format, args);
-}
-
-static void time_since_start()
-{
-	long nsec, sec;
-	static struct timespec cur_time;
-	double time_diff;
-
-	clock_gettime(CLOCK_MONOTONIC, &cur_time);
-	nsec = cur_time.tv_nsec - start_time.tv_nsec;
-	sec = cur_time.tv_sec - start_time.tv_sec;
-	if (nsec < 0) {
-		nsec += NSEC_PER_SEC;
-		sec--;
-	}
-	time_diff = sec + (double)nsec / NSEC_PER_SEC;
-	printf("%-8.3f", time_diff);
-}
-
-static void inline quoted_symbol(char c) {
-	switch(c) {
-		case '"':
-			putchar('\\');
-			putchar('"');
-			break;
-		case '\t':
-			putchar('\\');
-			putchar('t');
-			break;
-		case '\n':
-			putchar('\\');
-			putchar('n');
-			break;
-		default:
-			putchar(c);
-			break;
-	}
-}
-
-static void print_args(const struct event *e, bool quote)
-{
-	int args_counter = 0;
-
-	if (env.quote)
-		putchar('"');
-
-	for (int i = 0; i < e->args_size && args_counter < e->args_count; i++) {
-		char c = e->args[i];
-		if (env.quote) {
-			if (c == '\0') {
-				args_counter++;
-				putchar('"');
-				putchar(' ');
-				if (args_counter < e->args_count) {
-					putchar('"');
-				}
-			} else {
-				quoted_symbol(c);
-			}
-		} else {
-			if (c == '\0') {
-				args_counter++;
-				putchar(' ');
-			} else {
-				putchar(c);
-			}
-		}
-	}
-	if (e->args_count == env.max_args + 1) {
-		fputs(" ...", stdout);
-	}
-}
-
-void handle_event(void *ctx, int cpu, void *data, __u32 data_sz)
-{
-	const struct event *e = data;
-	time_t t;
-	struct tm *tm;
-	char ts[32];
-
-	/* TODO: use pcre lib */
-	if (env.name && strstr(e->comm, env.name) == NULL)
-		return;
-
-	/* TODO: use pcre lib */
-	if (env.line && strstr(e->comm, env.line) == NULL)
-		return;
-
-	time(&t);
-	tm = localtime(&t);
-	strftime(ts, sizeof(ts), "%H:%M:%S", tm);
-
-	if (env.time) {
-		printf("%-8s ", ts);
-	}
-	if (env.timestamp) {
-		time_since_start();
-	}
-
-	if (env.print_uid)
-		printf("%-6d", e->uid);
-
-	printf("%-16s %-6d %-6d %3d ", e->comm, e->pid, e->ppid, e->retval);
-	print_args(e, env.quote);
-	putchar('\n');
-}
-
-void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt)
-{
+void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) {
 	fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
 }
 
-int main(int argc, char **argv)
-{
-	static const struct argp argp = {
-		.options = opts,
-		.parser = parse_arg,
-		.doc = argp_program_doc,
-	};
-	struct perf_buffer_opts pb_opts;
-	struct perf_buffer *pb = NULL;
-	struct execsnoop_bpf *obj;
+int bump_memlock_rlimit(void) {
+	struct rlimit rlim_new = { RLIM_INFINITY, RLIM_INFINITY };
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
+int main(int argc, char **argv) {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
 	int err;
-
-	err = argp_parse(&argp, argc, argv, 0, NULL, NULL);
-	if (err)
-		return err;
-
-	libbpf_set_print(libbpf_print_fn);
-
+	
 	err = bump_memlock_rlimit();
 	if (err) {
 		fprintf(stderr, "failed to increase rlimit: %d\n", err);
 		return 1;
 	}
-
-	obj = execsnoop_bpf__open();
+	
+	struct execsnoop_bpf *obj=execsnoop_bpf__open_and_load();
 	if (!obj) {
 		fprintf(stderr, "failed to open and/or load BPF object\n");
 		return 1;
 	}
 
-	/* initialize global data (filtering options) */
-	obj->rodata->ignore_failed = !env.fails;
-	obj->rodata->targ_uid = env.uid;
-	obj->rodata->max_args = env.max_args;
-
-	err = execsnoop_bpf__load(obj);
-	if (err) {
-		fprintf(stderr, "failed to load BPF object: %d\n", err);
-		goto cleanup;
-	}
-
-	clock_gettime(CLOCK_MONOTONIC, &start_time);
 	err = execsnoop_bpf__attach(obj);
 	if (err) {
 		fprintf(stderr, "failed to attach BPF programs\n");
-		goto cleanup;
-	}
-	/* print headers */
-	if (env.time) {
-		printf("%-9s", "TIME");
-	}
-	if (env.timestamp) {
-		printf("%-8s ", "TIME(s)");
-	}
-	if (env.print_uid) {
-		printf("%-6s ", "UID");
+		return err;
 	}
 
-	printf("%-16s %-6s %-6s %3s %s\n", "PCOMM", "PID", "PPID", "RET", "ARGS");
-
-	/* setup event callbacks */
+	#if LIBBPF_MAJOR_VERSION == 0
 	pb_opts.sample_cb = handle_event;
 	pb_opts.lost_cb = handle_lost_events;
-	pb = perf_buffer__new(bpf_map__fd(obj->maps.events), PERF_BUFFER_PAGES, &pb_opts);
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	#else
+	pb_opts.sz = sizeof(pb_opts);
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, handle_event, handle_lost_events, nullptr, &pb_opts);
+	#endif
 	err = libbpf_get_error(pb);
 	if (err) {
-		pb = NULL;
-		fprintf(stderr, "failed to open perf buffer: %d\n", err);
-		goto cleanup;
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
 	}
 
-	/* main: poll */
-	while ((err = perf_buffer__poll(pb, 100)) >= 0)
-		;
-	printf("Error polling perf buffer: %d\n", err);
-
-cleanup:
-	perf_buffer__free(pb);
-	execsnoop_bpf__destroy(obj);
-
-	return err != 0;
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return err;
 }

execsnoop-libbpf/execsnoop.h

@@ -1,27 +0,0 @@
-/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
-#ifndef __EXECSNOOP_H
-#define __EXECSNOOP_H
-
-#define ARGSIZE  128
-#define TASK_COMM_LEN 16
-#define TOTAL_MAX_ARGS 60
-#define DEFAULT_MAXARGS 20
-#define FULL_MAX_ARGS_ARR (TOTAL_MAX_ARGS * ARGSIZE)
-#define INVALID_UID ((uid_t)-1)
-#define BASE_EVENT_SIZE (size_t)(&((struct event*)0)->args)
-#define EVENT_SIZE(e) (BASE_EVENT_SIZE + e->args_size)
-#define LAST_ARG (FULL_MAX_ARGS_ARR - ARGSIZE)
-
-struct event {
-	char comm[TASK_COMM_LEN];
-	pid_t pid;
-	pid_t tgid;
-	pid_t ppid;
-	uid_t uid;
-	int retval;
-	int args_count;
-	unsigned int args_size;
-	char args[FULL_MAX_ARGS_ARR];
-};
-
-#endif /* __EXECSNOOP_H */

execsnoop-libbpf/execsnoop_share.cpp

@@ -1,101 +1,96 @@
-// Based on execsnoop(8) from BCC by Brendan Gregg and others.
-//
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-extern "C" {
-#define typeof(x) decltype(x)
-#include "execsnoop.h"
-#include "execsnoop.skel.h"
-#include "trace_helpers.h"
-}
 #include "execsnoop_share.h"
 
-#define PERF_BUFFER_PAGES   64
-#define NSEC_PRECISION (NSEC_PER_SEC / 1000)
-#define MAX_ARGS_KEY 259
+#include <errno.h>
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include <sys/resource.h>
+#include "execsnoop.skel.h"
 
 namespace CGPROXY::EXECSNOOP {
-	
+
+#define PERF_BUFFER_PAGES 64
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
 function<int(int)> callback = NULL;
 promise<void> status;
 
-void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) {
-
+static void handle_event(void *ctx, int cpu, void *data, __u32 size) {
   	auto e = static_cast<event*>(data);
-  	int pid = e->pid;
-  	if (callback) callback(pid);
-
+  	if (callback) callback(e->pid);
 }
 
 void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) {
 	fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
 }
 
+int bump_memlock_rlimit(void) {
+	struct rlimit rlim_new = { RLIM_INFINITY, RLIM_INFINITY };
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
 int execsnoop() {
-	struct perf_buffer_opts pb_opts;
-	struct perf_buffer *pb = NULL;
-	struct execsnoop_bpf *obj;
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
 	int err;
-
-	// libbpf_set_print(libbpf_print_fn);
-
+	bool notified=false;
+	
 	err = bump_memlock_rlimit();
 	if (err) {
 		fprintf(stderr, "failed to increase rlimit: %d\n", err);
 		return 1;
 	}
-
-	obj = execsnoop_bpf__open();
+	
+	struct execsnoop_bpf *obj=execsnoop_bpf__open_and_load();
 	if (!obj) {
 		fprintf(stderr, "failed to open and/or load BPF object\n");
 		return 1;
 	}
 
-	/* initialize global data (filtering options) */
-	obj->rodata->ignore_failed = true;
-
-	err = execsnoop_bpf__load(obj);
-	if (err) {
-		fprintf(stderr, "failed to load BPF object: %d\n", err);
-		goto cleanup;
-	}
-
 	err = execsnoop_bpf__attach(obj);
 	if (err) {
 		fprintf(stderr, "failed to attach BPF programs\n");
-		goto cleanup;
+		return err;
 	}
 
-	/* setup event callbacks */
+main_loop:
+	#if LIBBPF_MAJOR_VERSION == 0
 	pb_opts.sample_cb = handle_event;
 	pb_opts.lost_cb = handle_lost_events;
-	pb = perf_buffer__new(bpf_map__fd(obj->maps.events), PERF_BUFFER_PAGES, &pb_opts);
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	#else
+	pb_opts.sz = sizeof(pb_opts);
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, handle_event, handle_lost_events, nullptr, &pb_opts);
+	#endif
 	err = libbpf_get_error(pb);
 	if (err) {
-		pb = NULL;
-		fprintf(stderr, "failed to open perf buffer: %d\n", err);
-		goto cleanup;
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
 	}
 
-  status.set_value();
+	// notify
+  	if (!notified) {status.set_value(); notified=true;}
 
-	/* main: poll */
-	while ((err = perf_buffer__poll(pb, 100)) >= 0);
-	printf("Error polling perf buffer: %d\n", err);
-
-cleanup:
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
 	perf_buffer__free(pb);
-	execsnoop_bpf__destroy(obj);
+	/* handle Interrupted system call when sleep */
+	if (err == -EINTR) goto main_loop;
 
-	return err != 0;
+	perror("perf_buffer__poll");
+	kill(0, SIGINT);
+	return err;
 }
 
-
 void startThread(function<int(int)> c, promise<void> _status) {
   status = move(_status);
   callback = c;
   execsnoop();
 }
+
 }

execsnoop-libbpf/execsnoop_share.h

@@ -7,10 +7,14 @@
 using namespace std;
 
 namespace CGPROXY::EXECSNOOP {
+
 extern "C" void startThread(function<int(int)> c, promise<void> _status);
-// typedef void startThread_t(function<int(int)>, promise<void>);
+
+#ifdef BUIlD_EXECSNOOP_DL
+// only for dlsym()
 using startThread_t=decltype(startThread);
-startThread_t *_startThread; // only for dlsym()
+startThread_t *_startThread;
+#endif
 
 } // namespace CGPROXY::EXECSNOOP
 #endif

execsnoop-libbpf/libbpf-fPIC.patch

@@ -1,21 +0,0 @@
-diff --git src/Makefile src/Makefile
-index d0308c3..fcc3b6f 100644
---- src/Makefile
-+++ src/Makefile
-@@ -15,6 +15,7 @@ ifneq ($(FEATURE_REALLOCARRAY),)
- 	ALL_CFLAGS += -DCOMPAT_NEED_REALLOCARRAY
- endif
- 
-+STATIC_CFLAGS += -fPIC
- SHARED_CFLAGS += -fPIC -fvisibility=hidden -DSHARED
- 
- CFLAGS ?= -g -O2 -Werror -Wall
-@@ -99,7 +100,7 @@ $(SHARED_OBJDIR):
- 	mkdir -p $(SHARED_OBJDIR)
- 
- $(STATIC_OBJDIR)/%.o: %.c | $(STATIC_OBJDIR)
--	$(CC) $(ALL_CFLAGS) $(CPPFLAGS) -c $< -o $@
-+	$(CC) $(ALL_CFLAGS) $(STATIC_CFLAGS) $(CPPFLAGS) -c $< -o $@
- 
- $(SHARED_OBJDIR)/%.o: %.c | $(SHARED_OBJDIR)
- 	$(CC) $(ALL_CFLAGS) $(SHARED_CFLAGS) $(CPPFLAGS) -c $< -o $@

execsnoop-libbpf/readme.md

@@ -1,21 +1,18 @@
+generate `vmlinux.h`
 
-## Resource
+```shell
+bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
+```
 
-- [libbpf](https://github.com/libbpf/libbpf)
-- [libbpf-tools](https://github.com/iovisor/bcc/tree/master/libbpf-tools) examples use libbpf
-- [bpftool](https://www.archlinux.org/packages/community/x86_64/bpf/) to generate skeleton and dump btf
+compiled into BPF ELF file
 
-## Prons
+```shell
+clang -O2 -g -target bpf -c execsnoop.bpf.c -o execsnoop.bpf.o
+```
 
-- BPF CO-RE (Compile Once – Run Everywhere)
-- small memory usage
+generate BPF skeleton .skel.h
 
-## Cons
+```shell
+bpftool gen skeleton execsnoop.bpf.o > execsnoop.skel.h
+```
 
-- `vmlinux.h`does not contain `#define` etc. And often causes confilct with other headers to cause redifinition error
-- comment in code for different types is gone
-- need kernel built with `CONFIG_DEBUG_INFO_BTF=y`, while ubuntu 20.04 still not
-
-## Build
-
-- see *makefile*

execsnoop-libbpf/syscall_helpers.c

@@ -1,526 +0,0 @@
-// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
-// Copyright (c) 2020 Anton Protopopov
-#include <stdlib.h>
-#include <string.h>
-#include <limits.h>
-#include <stdio.h>
-#include <errno.h>
-
-static const char **syscall_names;
-static size_t syscall_names_size;
-
-#define warn(...) fprintf(stderr, __VA_ARGS__)
-#define MAX(x, y) (((x) > (y)) ? (x) : (y))
-
-static const char *parse_syscall(const char *buf, int *number)
-{
-	char *end;
-	long x;
-
-	errno = 0;
-	x = strtol(buf, &end, 10);
-	if (errno) {
-		warn("strtol(%s): %s\n", buf, strerror(errno));
-		return NULL;
-	} else if (end == buf) {
-		warn("strtol(%s): no digits found\n", buf);
-		return NULL;
-	} else if (x < 0 || x > INT_MAX) {
-		warn("strtol(%s): bad syscall number: %ld\n", buf, x);
-		return NULL;
-	}
-	if (*end != '\t') {
-		warn("bad input: %s (expected <num>\t<name>)\n", buf);
-		return NULL;
-	}
-
-	*number = x;
-	return ++end;
-}
-
-void init_syscall_names(void)
-{
-	size_t old_size, size = 1024;
-	const char *name;
-	char buf[64];
-	int number;
-	int err;
-	FILE *f;
-
-	f = popen("ausyscall --dump 2>/dev/null", "r");
-	if (!f) {
-		warn("popen: ausyscall --dump: %s\n", strerror(errno));
-		return;
-	}
-
-	syscall_names = calloc(size, sizeof(char *));
-	if (!syscall_names) {
-		warn("calloc: %s\n", strerror(errno));
-		goto close;
-	}
-
-	/* skip the header */
-	fgets(buf, sizeof(buf), f);
-
-	while (fgets(buf, sizeof(buf), f)) {
-		if (buf[strlen(buf) - 1] == '\n')
-			buf[strlen(buf) - 1] = '\0';
-
-		name = parse_syscall(buf, &number);
-		if (!name || !name[0])
-			goto close;
-
-		/* In a rare case when syscall number is > than initial 1024 */
-		if (number >= size) {
-			old_size = size;
-			size = 1024 * (1 + number / 1024);
-			syscall_names = realloc(syscall_names,
-						size * sizeof(char *));
-			if (!syscall_names) {
-				warn("realloc: %s\n", strerror(errno));
-				goto close;
-			}
-			memset(syscall_names+old_size, 0,
-			       (size - old_size) * sizeof(char *));
-		}
-
-		if (syscall_names[number]) {
-			warn("duplicate number: %d (stored: %s)",
-				number, syscall_names[number]);
-			goto close;
-		}
-
-		syscall_names[number] = strdup(name);
-		if (!syscall_names[number]) {
-			warn("strdup: %s\n", strerror(errno));
-			goto close;
-		}
-		syscall_names_size = MAX(number+1, syscall_names_size);
-	}
-
-	if (ferror(f))
-		warn("fgets: %s\n", strerror(errno));
-close:
-	err = pclose(f);
-	if (err < 0)
-		warn("pclose: %s\n", strerror(errno));
-#ifndef __x86_64__
-	/* Ignore the error for x86_64 where we have a table compiled in */
-	else if (err && WEXITSTATUS(err) == 127) {
-		warn("ausyscall required for syscalls number/name mapping\n");
-	} else if (err) {
-		warn("ausyscall exit status (see wait(2)): 0x%x\n", err);
-	}
-#endif
-}
-
-void free_syscall_names(void)
-{
-	for (size_t i = 0; i < syscall_names_size; i++)
-		free((void *) syscall_names[i]);
-	free(syscall_names);
-}
-
-/*
- * Syscall table for Linux x86_64.
- *
- * Semi-automatically generated from strace/linux/x86_64/syscallent.h and
- * linux/syscallent-common.h using the following commands:
- *
- *  awk -F\" '/SEN/{printf("%d %s\n", substr($0,2,3), $(NF-1));}' syscallent.h
- *  awk '/SEN/ { printf("%d %s\n", $3, $9); }' syscallent-common.h
- *
- * (The idea is taken from src/python/bcc/syscall.py.)
- */
-#ifdef __x86_64__
-static const char *syscall_names_x86_64[] = {
-	[0] = "read",
-	[1] = "write",
-	[2] = "open",
-	[3] = "close",
-	[4] = "stat",
-	[5] = "fstat",
-	[6] = "lstat",
-	[7] = "poll",
-	[8] = "lseek",
-	[9] = "mmap",
-	[10] = "mprotect",
-	[11] = "munmap",
-	[12] = "brk",
-	[13] = "rt_sigaction",
-	[14] = "rt_sigprocmask",
-	[15] = "rt_sigreturn",
-	[16] = "ioctl",
-	[17] = "pread64",
-	[18] = "pwrite64",
-	[19] = "readv",
-	[20] = "writev",
-	[21] = "access",
-	[22] = "pipe",
-	[23] = "select",
-	[24] = "sched_yield",
-	[25] = "mremap",
-	[26] = "msync",
-	[27] = "mincore",
-	[28] = "madvise",
-	[29] = "shmget",
-	[30] = "shmat",
-	[31] = "shmctl",
-	[32] = "dup",
-	[33] = "dup2",
-	[34] = "pause",
-	[35] = "nanosleep",
-	[36] = "getitimer",
-	[37] = "alarm",
-	[38] = "setitimer",
-	[39] = "getpid",
-	[40] = "sendfile",
-	[41] = "socket",
-	[42] = "connect",
-	[43] = "accept",
-	[44] = "sendto",
-	[45] = "recvfrom",
-	[46] = "sendmsg",
-	[47] = "recvmsg",
-	[48] = "shutdown",
-	[49] = "bind",
-	[50] = "listen",
-	[51] = "getsockname",
-	[52] = "getpeername",
-	[53] = "socketpair",
-	[54] = "setsockopt",
-	[55] = "getsockopt",
-	[56] = "clone",
-	[57] = "fork",
-	[58] = "vfork",
-	[59] = "execve",
-	[60] = "exit",
-	[61] = "wait4",
-	[62] = "kill",
-	[63] = "uname",
-	[64] = "semget",
-	[65] = "semop",
-	[66] = "semctl",
-	[67] = "shmdt",
-	[68] = "msgget",
-	[69] = "msgsnd",
-	[70] = "msgrcv",
-	[71] = "msgctl",
-	[72] = "fcntl",
-	[73] = "flock",
-	[74] = "fsync",
-	[75] = "fdatasync",
-	[76] = "truncate",
-	[77] = "ftruncate",
-	[78] = "getdents",
-	[79] = "getcwd",
-	[80] = "chdir",
-	[81] = "fchdir",
-	[82] = "rename",
-	[83] = "mkdir",
-	[84] = "rmdir",
-	[85] = "creat",
-	[86] = "link",
-	[87] = "unlink",
-	[88] = "symlink",
-	[89] = "readlink",
-	[90] = "chmod",
-	[91] = "fchmod",
-	[92] = "chown",
-	[93] = "fchown",
-	[94] = "lchown",
-	[95] = "umask",
-	[96] = "gettimeofday",
-	[97] = "getrlimit",
-	[98] = "getrusage",
-	[99] = "sysinfo",
-	[100] = "times",
-	[101] = "ptrace",
-	[102] = "getuid",
-	[103] = "syslog",
-	[104] = "getgid",
-	[105] = "setuid",
-	[106] = "setgid",
-	[107] = "geteuid",
-	[108] = "getegid",
-	[109] = "setpgid",
-	[110] = "getppid",
-	[111] = "getpgrp",
-	[112] = "setsid",
-	[113] = "setreuid",
-	[114] = "setregid",
-	[115] = "getgroups",
-	[116] = "setgroups",
-	[117] = "setresuid",
-	[118] = "getresuid",
-	[119] = "setresgid",
-	[120] = "getresgid",
-	[121] = "getpgid",
-	[122] = "setfsuid",
-	[123] = "setfsgid",
-	[124] = "getsid",
-	[125] = "capget",
-	[126] = "capset",
-	[127] = "rt_sigpending",
-	[128] = "rt_sigtimedwait",
-	[129] = "rt_sigqueueinfo",
-	[130] = "rt_sigsuspend",
-	[131] = "sigaltstack",
-	[132] = "utime",
-	[133] = "mknod",
-	[134] = "uselib",
-	[135] = "personality",
-	[136] = "ustat",
-	[137] = "statfs",
-	[138] = "fstatfs",
-	[139] = "sysfs",
-	[140] = "getpriority",
-	[141] = "setpriority",
-	[142] = "sched_setparam",
-	[143] = "sched_getparam",
-	[144] = "sched_setscheduler",
-	[145] = "sched_getscheduler",
-	[146] = "sched_get_priority_max",
-	[147] = "sched_get_priority_min",
-	[148] = "sched_rr_get_interval",
-	[149] = "mlock",
-	[150] = "munlock",
-	[151] = "mlockall",
-	[152] = "munlockall",
-	[153] = "vhangup",
-	[154] = "modify_ldt",
-	[155] = "pivot_root",
-	[156] = "_sysctl",
-	[157] = "prctl",
-	[158] = "arch_prctl",
-	[159] = "adjtimex",
-	[160] = "setrlimit",
-	[161] = "chroot",
-	[162] = "sync",
-	[163] = "acct",
-	[164] = "settimeofday",
-	[165] = "mount",
-	[166] = "umount2",
-	[167] = "swapon",
-	[168] = "swapoff",
-	[169] = "reboot",
-	[170] = "sethostname",
-	[171] = "setdomainname",
-	[172] = "iopl",
-	[173] = "ioperm",
-	[174] = "create_module",
-	[175] = "init_module",
-	[176] = "delete_module",
-	[177] = "get_kernel_syms",
-	[178] = "query_module",
-	[179] = "quotactl",
-	[180] = "nfsservctl",
-	[181] = "getpmsg",
-	[182] = "putpmsg",
-	[183] = "afs_syscall",
-	[184] = "tuxcall",
-	[185] = "security",
-	[186] = "gettid",
-	[187] = "readahead",
-	[188] = "setxattr",
-	[189] = "lsetxattr",
-	[190] = "fsetxattr",
-	[191] = "getxattr",
-	[192] = "lgetxattr",
-	[193] = "fgetxattr",
-	[194] = "listxattr",
-	[195] = "llistxattr",
-	[196] = "flistxattr",
-	[197] = "removexattr",
-	[198] = "lremovexattr",
-	[199] = "fremovexattr",
-	[200] = "tkill",
-	[201] = "time",
-	[202] = "futex",
-	[203] = "sched_setaffinity",
-	[204] = "sched_getaffinity",
-	[205] = "set_thread_area",
-	[206] = "io_setup",
-	[207] = "io_destroy",
-	[208] = "io_getevents",
-	[209] = "io_submit",
-	[210] = "io_cancel",
-	[211] = "get_thread_area",
-	[212] = "lookup_dcookie",
-	[213] = "epoll_create",
-	[214] = "epoll_ctl_old",
-	[215] = "epoll_wait_old",
-	[216] = "remap_file_pages",
-	[217] = "getdents64",
-	[218] = "set_tid_address",
-	[219] = "restart_syscall",
-	[220] = "semtimedop",
-	[221] = "fadvise64",
-	[222] = "timer_create",
-	[223] = "timer_settime",
-	[224] = "timer_gettime",
-	[225] = "timer_getoverrun",
-	[226] = "timer_delete",
-	[227] = "clock_settime",
-	[228] = "clock_gettime",
-	[229] = "clock_getres",
-	[230] = "clock_nanosleep",
-	[231] = "exit_group",
-	[232] = "epoll_wait",
-	[233] = "epoll_ctl",
-	[234] = "tgkill",
-	[235] = "utimes",
-	[236] = "vserver",
-	[237] = "mbind",
-	[238] = "set_mempolicy",
-	[239] = "get_mempolicy",
-	[240] = "mq_open",
-	[241] = "mq_unlink",
-	[242] = "mq_timedsend",
-	[243] = "mq_timedreceive",
-	[244] = "mq_notify",
-	[245] = "mq_getsetattr",
-	[246] = "kexec_load",
-	[247] = "waitid",
-	[248] = "add_key",
-	[249] = "request_key",
-	[250] = "keyctl",
-	[251] = "ioprio_set",
-	[252] = "ioprio_get",
-	[253] = "inotify_init",
-	[254] = "inotify_add_watch",
-	[255] = "inotify_rm_watch",
-	[256] = "migrate_pages",
-	[257] = "openat",
-	[258] = "mkdirat",
-	[259] = "mknodat",
-	[260] = "fchownat",
-	[261] = "futimesat",
-	[262] = "newfstatat",
-	[263] = "unlinkat",
-	[264] = "renameat",
-	[265] = "linkat",
-	[266] = "symlinkat",
-	[267] = "readlinkat",
-	[268] = "fchmodat",
-	[269] = "faccessat",
-	[270] = "pselect6",
-	[271] = "ppoll",
-	[272] = "unshare",
-	[273] = "set_robust_list",
-	[274] = "get_robust_list",
-	[275] = "splice",
-	[276] = "tee",
-	[277] = "sync_file_range",
-	[278] = "vmsplice",
-	[279] = "move_pages",
-	[280] = "utimensat",
-	[281] = "epoll_pwait",
-	[282] = "signalfd",
-	[283] = "timerfd_create",
-	[284] = "eventfd",
-	[285] = "fallocate",
-	[286] = "timerfd_settime",
-	[287] = "timerfd_gettime",
-	[288] = "accept4",
-	[289] = "signalfd4",
-	[290] = "eventfd2",
-	[291] = "epoll_create1",
-	[292] = "dup3",
-	[293] = "pipe2",
-	[294] = "inotify_init1",
-	[295] = "preadv",
-	[296] = "pwritev",
-	[297] = "rt_tgsigqueueinfo",
-	[298] = "perf_event_open",
-	[299] = "recvmmsg",
-	[300] = "fanotify_init",
-	[301] = "fanotify_mark",
-	[302] = "prlimit64",
-	[303] = "name_to_handle_at",
-	[304] = "open_by_handle_at",
-	[305] = "clock_adjtime",
-	[306] = "syncfs",
-	[307] = "sendmmsg",
-	[308] = "setns",
-	[309] = "getcpu",
-	[310] = "process_vm_readv",
-	[311] = "process_vm_writev",
-	[312] = "kcmp",
-	[313] = "finit_module",
-	[314] = "sched_setattr",
-	[315] = "sched_getattr",
-	[316] = "renameat2",
-	[317] = "seccomp",
-	[318] = "getrandom",
-	[319] = "memfd_create",
-	[320] = "kexec_file_load",
-	[321] = "bpf",
-	[322] = "execveat",
-	[323] = "userfaultfd",
-	[324] = "membarrier",
-	[325] = "mlock2",
-	[326] = "copy_file_range",
-	[327] = "preadv2",
-	[328] = "pwritev2",
-	[329] = "pkey_mprotect",
-	[330] = "pkey_alloc",
-	[331] = "pkey_free",
-	[332] = "statx",
-	[333] = "io_pgetevents",
-	[334] = "rseq",
-	[424] = "pidfd_send_signal",
-	[425] = "io_uring_setup",
-	[426] = "io_uring_enter",
-	[427] = "io_uring_register",
-	[428] = "open_tree",
-	[429] = "move_mount",
-	[430] = "fsopen",
-	[431] = "fsconfig",
-	[432] = "fsmount",
-	[433] = "fspick",
-	[434] = "pidfd_open",
-	[435] = "clone3",
-	[437] = "openat2",
-	[438] = "pidfd_getfd",
-};
-size_t syscall_names_x86_64_size = sizeof(syscall_names_x86_64)/sizeof(char*);
-#endif
-
-void syscall_name(unsigned n, char *buf, size_t size)
-{
-	const char *name = NULL;
-
-	if (n < syscall_names_size)
-		name = syscall_names[n];
-#ifdef __x86_64__
-	else if (n < syscall_names_x86_64_size)
-		name = syscall_names_x86_64[n];
-#endif
-
-	if (name)
-		strncpy(buf, name, size-1);
-	else
-		snprintf(buf, size, "[unknown: %u]", n);
-}
-
-int list_syscalls(void)
-{
-	const char **list = syscall_names;
-	size_t size = syscall_names_size;
-
-#ifdef __x86_64__
-	if (!size) {
-		size = syscall_names_x86_64_size;
-		list = syscall_names_x86_64;
-	}
-#endif
-
-	for (size_t i = 0; i < size; i++) {
-		if (list[i])
-			printf("%3zd: %s\n", i, list[i]);
-	}
-
-	return (!list || !size);
-}
-

execsnoop-libbpf/syscall_helpers.h

@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
-#ifndef __SYSCALL_HELPERS_H
-#define __SYSCALL_HELPERS_H
-
-#include <stddef.h>
-
-void init_syscall_names(void);
-void free_syscall_names(void);
-void list_syscalls(void);
-void syscall_name(unsigned n, char *buf, size_t size);
-
-#endif /* __SYSCALL_HELPERS_H */

execsnoop-libbpf/trace_helpers.c

@@ -1,234 +0,0 @@
-/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdbool.h>
-#include <sys/resource.h>
-#include <time.h>
-#include "trace_helpers.h"
-
-#define min(x, y) ({				 \
-	typeof(x) _min1 = (x);			 \
-	typeof(y) _min2 = (y);			 \
-	(void) (&_min1 == &_min2);		 \
-	_min1 < _min2 ? _min1 : _min2; })
-
-struct ksyms {
-	struct ksym *syms;
-	int syms_sz;
-	int syms_cap;
-	char *strs;
-	int strs_sz;
-	int strs_cap;
-};
-
-static int ksyms__add_symbol(struct ksyms *ksyms, const char *name, unsigned long addr)
-{
-	size_t new_cap, name_len = strlen(name) + 1;
-	struct ksym *ksym;
-	void *tmp;
-
-	if (ksyms->strs_sz + name_len > ksyms->strs_cap) {
-		new_cap = ksyms->strs_cap * 4 / 3;
-		if (new_cap < ksyms->strs_sz + name_len)
-			new_cap = ksyms->strs_sz + name_len;
-		if (new_cap < 1024)
-			new_cap = 1024;
-		tmp = realloc(ksyms->strs, new_cap);
-		if (!tmp)
-			return -1;
-		ksyms->strs = tmp;
-		ksyms->strs_cap = new_cap;
-	}
-	if (ksyms->syms_sz + 1 > ksyms->syms_cap) {
-		new_cap = ksyms->syms_cap * 4 / 3;
-		if (new_cap < 1024)
-			new_cap = 1024;
-		tmp = realloc(ksyms->syms, sizeof(*ksyms->syms) * new_cap);
-		if (!tmp)
-			return -1;
-		ksyms->syms = tmp;
-		ksyms->syms_cap = new_cap;
-	}
-
-	ksym = &ksyms->syms[ksyms->syms_sz];
-	/* while constructing, re-use pointer as just a plain offset */
-	ksym->name = (void *)(unsigned long)ksyms->strs_sz;
-	ksym->addr = addr;
-
-	memcpy(ksyms->strs + ksyms->strs_sz, name, name_len);
-	ksyms->strs_sz += name_len;
-	ksyms->syms_sz++;
-
-	return 0;
-}
-
-static int ksym_cmp(const void *p1, const void *p2)
-{
-	const struct ksym *s1 = p1, *s2 = p2;
-
-	if (s1->addr == s2->addr)
-		return strcmp(s1->name, s2->name);
-	return s1->addr < s2->addr ? -1 : 1;
-}
-
-struct ksyms *ksyms__load(void)
-{
-	char sym_type, sym_name[256];
-	struct ksyms *ksyms;
-	unsigned long sym_addr;
-	int i, ret;
-	FILE *f;
-
-	f = fopen("/proc/kallsyms", "r");
-	if (!f)
-		return NULL;
-
-	ksyms = calloc(1, sizeof(*ksyms));
-	if (!ksyms)
-		goto err_out;
-
-	while (true) {
-		ret = fscanf(f, "%lx %c %s%*[^\n]\n",
-			     &sym_addr, &sym_type, sym_name);
-		if (ret == EOF && feof(f))
-			break;
-		if (ret != 3)
-			goto err_out;
-		if (ksyms__add_symbol(ksyms, sym_name, sym_addr))
-			goto err_out;
-	}
-
-	/* now when strings are finalized, adjust pointers properly */
-	for (i = 0; i < ksyms->syms_sz; i++)
-		ksyms->syms[i].name += (unsigned long)ksyms->strs;
-
-	qsort(ksyms->syms, ksyms->syms_sz, sizeof(*ksyms->syms), ksym_cmp);
-
-	fclose(f);
-	return ksyms;
-
-err_out:
-	ksyms__free(ksyms);
-	fclose(f);
-	return NULL;
-}
-
-void ksyms__free(struct ksyms *ksyms)
-{
-	if (!ksyms)
-		return;
-
-	free(ksyms->syms);
-	free(ksyms->strs);
-	free(ksyms);
-}
-
-const struct ksym *ksyms__map_addr(const struct ksyms *ksyms,
-				   unsigned long addr)
-{
-	int start = 0, end = ksyms->syms_sz - 1, mid;
-	unsigned long sym_addr;
-
-	/* find largest sym_addr <= addr using binary search */
-	while (start < end) {
-		mid = start + (end - start + 1) / 2;
-		sym_addr = ksyms->syms[mid].addr;
-
-		if (sym_addr <= addr)
-			start = mid;
-		else
-			end = mid - 1;
-	}
-
-	if (start == end && ksyms->syms[start].addr <= addr)
-		return &ksyms->syms[start];
-	return NULL;
-}
-
-const struct ksym *ksyms__get_symbol(const struct ksyms *ksyms,
-				     const char *name)
-{
-	int i;
-
-	for (i = 0; i < ksyms->syms_sz; i++) {
-		if (strcmp(ksyms->syms[i].name, name) == 0)
-			return &ksyms->syms[i];
-	}
-
-	return NULL;
-}
-
-static void print_stars(unsigned int val, unsigned int val_max, int width)
-{
-	int num_stars, num_spaces, i;
-	bool need_plus;
-
-	num_stars = min(val, val_max) * width / val_max;
-	num_spaces = width - num_stars;
-	need_plus = val > val_max;
-
-	for (i = 0; i < num_stars; i++)
-		printf("*");
-	for (i = 0; i < num_spaces; i++)
-		printf(" ");
-	if (need_plus)
-		printf("+");
-}
-
-void print_log2_hist(unsigned int *vals, int vals_size, char *val_type)
-{
-	int stars_max = 40, idx_max = -1;
-	unsigned int val, val_max = 0;
-	unsigned long long low, high;
-	int stars, width, i;
-
-	for (i = 0; i < vals_size; i++) {
-		val = vals[i];
-		if (val > 0)
-			idx_max = i;
-		if (val > val_max)
-			val_max = val;
-	}
-
-	if (idx_max < 0)
-		return;
-
-	printf("%*s%-*s : count    distribution\n", idx_max <= 32 ? 5 : 15, "",
-		idx_max <= 32 ? 19 : 29, val_type);
-
-	if (idx_max <= 32)
-		stars = stars_max;
-	else
-		stars = stars_max / 2;
-
-	for (i = 0; i <= idx_max; i++) {
-		low = (1ULL << (i + 1)) >> 1;
-		high = (1ULL << (i + 1)) - 1;
-		if (low == high)
-			low -= 1;
-		val = vals[i];
-		width = idx_max <= 32 ? 10 : 20;
-		printf("%*lld -> %-*lld : %-8d |", width, low, width, high, val);
-		print_stars(val, val_max, stars);
-		printf("|\n");
-	}
-}
-
-unsigned long long get_ktime_ns(void)
-{
-	struct timespec ts;
-
-	clock_gettime(CLOCK_MONOTONIC, &ts);
-	return ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec;
-}
-
-int bump_memlock_rlimit(void)
-{
-	struct rlimit rlim_new = {
-		.rlim_cur	= RLIM_INFINITY,
-		.rlim_max	= RLIM_INFINITY,
-	};
-
-	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
-}

execsnoop-libbpf/trace_helpers.h

@@ -1,26 +0,0 @@
-/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
-#ifndef __TRACE_HELPERS_H
-#define __TRACE_HELPERS_H
-
-#define NSEC_PER_SEC		1000000000ULL
-
-struct ksym {
-	const char *name;
-	unsigned long addr;
-};
-
-struct ksyms;
-
-struct ksyms *ksyms__load(void);
-void ksyms__free(struct ksyms *ksyms);
-const struct ksym *ksyms__map_addr(const struct ksyms *ksyms,
-				   unsigned long addr);
-const struct ksym *ksyms__get_symbol(const struct ksyms *ksyms,
-				     const char *name);
-
-void print_log2_hist(unsigned int *vals, int vals_size, char *val_type);
-
-unsigned long long get_ktime_ns(void);
-int bump_memlock_rlimit(void);
-
-#endif /* __TRACE_HELPERS_H */

man/cgnoproxy.1

@@ -4,7 +4,9 @@
 cgnoproxy \- Run program without proxy
 .SH SYNOPSIS
 cgnoproxy --help
+.br
 cgnoproxy [--debug] <CMD>
+.br
 cgnoproxy [--debug] --pid <PID>
 .SH ALIAS
 cgnoproxy = cgproxy --noproxy

man/cgproxy.1

@@ -4,7 +4,9 @@
 cgproxy \- Run program with proxy
 .SH SYNOPSIS
 cgproxy --help
+.br
 cgproxy [--debug] <CMD>
+.br
 cgproxy [--debug] --pid <PID>
 .SH DESCRIPTION
 cgproxy send current running process pid or specified pid to cgproxyd through unix socket, then pid is attached to proxied cgroup 

man/cgproxyd.1

@@ -15,7 +15,7 @@ enable execsnoop to support program level proxy, need bcc installed to actually
 .B port 
 tproxy listenning port
 .br
-program level proxy controll, need `bcc` installed to work:
+program level proxy controll, only work when execsnoop enabled:
 .br
 .RS
 .B program_proxy

pack/CMakeLists.txt

@@ -4,9 +4,13 @@ set(CPACK_PACKAGE_NAME "cgproxy")
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "cgproxy will transparent proxy anything running in specific cgroup.It aslo supports global transparent proxy and gateway proxy")
 
 ## deb pack
+execute_process(COMMAND dpkg --print-architecture 
+    OUTPUT_VARIABLE DEBIAN_ARCH 
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+set(CPACK_DEBIAN_FILE_NAME ${CPACK_PACKAGE_NAME}_${CMAKE_PROJECT_VERSION}_${DEBIAN_ARCH}.deb)
 set(CPACK_DEBIAN_PACKAGE_NAME "cgproxy")
-set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "amd64")
-set(CPACK_DEBIAN_PACKAGE_DEPENDS "systemd, libbpf0")
+# set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "amd64")
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "systemd")
 set(CPACK_DEBIAN_PACKAGE_SECTION "network")
 set(CPACK_DEBIAN_PACKAGE_PRIORITY "Optional")
 set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/springzfx/cgproxy")
@@ -14,8 +18,12 @@ set(CPACK_DEBIAN_PACKAGE_MAINTAINER "springzfx@gmail.com")
 set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/postinst;${CMAKE_CURRENT_SOURCE_DIR}/prerm")
 
 ## rpm pack
-set(CPACK_RPM_PACKAGE_ARCHITECTURE, "x86_64")
-set(CPACK_RPM_PACKAGE_REQUIRES "systemd, libbpf")
+execute_process(COMMAND uname -m 
+        OUTPUT_VARIABLE RPM_ARCH 
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+set(CPACK_RPM_FILE_NAME ${CPACK_PACKAGE_NAME}_${CMAKE_PROJECT_VERSION}_${RPM_ARCH}.rpm)
+# set(CPACK_RPM_PACKAGE_ARCHITECTURE, "x86_64")
+set(CPACK_RPM_PACKAGE_REQUIRES "systemd")
 set(CPACK_RPM_PACKAGE_GROUP "network")
 set(CPACK_RPM_PACKAGE_URL "https://github.com/springzfx/cgproxy")
 set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/postinst")

readme.md

@@ -20,7 +20,7 @@ Main feature:
       * [Introduction](#introduction)
       * [Contents](#contents)
       * [Prerequest](#prerequest)
-      * [How to install](#how-to-install)
+      * [How to build and install](#how-to-build-and-install)
       * [Default usage](#default-usage)
       * [Configuration](#configuration)
       * [Global transparent proxy](#global-transparent-proxy)
@@ -29,7 +29,7 @@ Main feature:
       * [NOTES](#notes)
       * [TIPS](#tips)
       * [Licences](#licences)
-      * [Known Issus](#known-issus)
+      * [Known Issues](#known-issues)
 
 <!-- Added by: fancy, at: Sat 04 Jul 2020 03:52:07 PM CST -->
 
@@ -53,25 +53,33 @@ Main feature:
 
   ubuntu 16.04, debian 9, fedora 27 and later are desired
 
-## How to install
+## How to build and install
 
-### main depency
+### distro install
 
-| os        | build depency                                                | runtime depency                                       | download                                                     |
-| --------- | ------------------------------------------------------------ | ----------------------------------------------------- | ------------------------------------------------------------ |
-| Archlinux | clang, nlohmann-json, libbpf                                 | libbpf                                                | [archlinux AUR](https://aur.archlinux.org/packages/?K=cgproxy) |
-| Ubuntu    | clang, nlohmann-json, [libbpf-dev](https://packages.ubuntu.com/groovy/libbpf-dev) | [libbpf0](https://packages.ubuntu.com/groovy/libbpf0) | [Release page](https://github.com/springzfx/cgproxy/releases) |
-| Fedora    | clang, nlohmann-json,[libbpf](https://src.fedoraproject.org/rpms/libbpf) | [libbpf](https://src.fedoraproject.org/rpms/libbpf)   | [Release page](https://github.com/springzfx/cgproxy/releases) |
+- For debian and redhat series, download from [Release page](https://github.com/springzfx/cgproxy/releases)
 
-tested in arch and ubuntu 20.04.
+- For archlinux series, already in archlinuxcn repo, or see [archlinux AUR](https://aur.archlinux.org/packages/?K=cgproxy)
+
+- **Tested on  archlinux, fedora 32, ubuntu 18.04, ubuntu 20.04,  deepin 15.11, deepin v20 beta**
 
 ### build
 
+- before build, install depencies: clang, nlohmann-json, libbpf, bpf(bpftool)
+- then cmake standard build
+
 ```bash
-# 
-mkdir build && cd build
-# cmake
-cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .. && make install
+# ready build dir
+mkdir build
+cd build
+# generate
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=/usr \
+      -Dbuild_execsnoop_dl=ON \
+      -Dbuild_static=OFF \
+      ..
+# compile
+make
 ```
 
 ## Default usage
@@ -117,15 +125,14 @@ Config file: **/etc/cgproxy/config.json**
     "enable_ipv4": true,
     "enable_ipv6": true,
     "table": 10007,
-    "fwmark": 39283,
-    "mark_newin": 39271
+    "fwmark": 39283
 }
 
 ```
 
 - **port** tproxy listenning port
 
-- program level proxy control:
+- program level proxy control, need execsnoop enabled:
 
   - **program_proxy**  program need to be proxied
   - **program_noproxy** program that won't be proxied
@@ -147,14 +154,14 @@ Config file: **/etc/cgproxy/config.json**
 
 - **enable_ipv6**
 
-- **table**, **fwmark**, **mark_newin**  you can specify iptables and route table related parameter in case conflict.
+- **table**, **fwmark** you can specify iptables and route table related parameter in case conflict.
 
 - options priority
 
   ```
   program_noproxy > program_proxy > cgroup_noproxy > cgroup_proxy
-  enable_ipv6 > enable_ipv4 > enable_tcp > enable_udp > enable_dns
-  ommand cgproxy and cgnoproxy always have highest priority
+  enable_ipv6 = enable_ipv4 > enable_dns > enable_tcp = enable_udp
+  command cgproxy and cgnoproxy always have highest priority
   ```
 
 **Note**: cgroup in configuration need to be exist, otherwise ignored
@@ -204,13 +211,13 @@ sudo systemctl restart cgproxy.service
 
 ## NOTES
 
-- v2ray TPROXY need root or special permission, use [service](https://github.com/springzfx/cgproxy/blob/v3.x/v2ray_config/v2ray.service) or
+- v2ray TPROXY need root or special permission, use [service](/v2ray_config/v2ray.service) or
   
   ```bash
   sudo setcap "cap_net_admin,cap_net_bind_service=ep" /usr/lib/v2ray/v2ray
   ```
 
-- Why not outbound mark solution, because in v2ray [when `"localhost"` is used, out-going DNS traffic is not controlled by V2Ray](https://www.v2fly.org/en/configuration/dns.html), so no mark at all, that's pity.
+- Why not outbound mark solution, because in v2ray [when `"localhost"` is used, out-going DNS traffic is not controlled by V2Ray](https://www.v2fly.org/config/dns.html#dnsobject), so no mark at all, that's pity.
 
 ## TIPS
 
@@ -220,12 +227,22 @@ sudo systemctl restart cgproxy.service
 - Offer you qv2ray config example
   
 
-![Qv2ray config example](https://i.loli.net/2020/06/12/6ltax93bv7NCTHU.png)
+![Qv2ray config example](https://i.loli.net/2020/08/17/P6y5SfLoUwGjaxM.png)
 
 ## Licences
 
 cgproxy is licenced under [![License: GPL v3](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/gpl-2.0) 
 
-## Known Issus
+## Known Issues
 
-- docker breaks cgroup path match, add kernel parameter `cgroup_no_v1=net_cls,net_prio` to resolve, see [issue #3](https://github.com/springzfx/cgproxy/issues/3) for detail
+- docker breaks cgroup v2 path match, add kernel parameter `cgroup_no_v1=net_cls,net_prio` to resolve, see [issue #3](https://github.com/springzfx/cgproxy/issues/3) for detail
+
+- docker load `br_netfilter` module due to [hairpin nat](https://wiki.mikrotik.com/wiki/Hairpin_NAT),  which is not a big deal,  see [commit](https://github.com/moby/moby/pull/13162). 
+
+  It enables data link layer packet to go through iptables and only once. However TPROXY do not accept this kind of packets. So to get it working, set following parameter to disable this behavior or unload br_netfilter module manualy. see [issue #10](https://github.com/springzfx/cgproxy/issues/10) for detail.
+
+  ```
+  sudo sysctl -w net.bridge.bridge-nf-call-iptables=0
+  sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=0
+  sudo sysctl -w net.bridge.bridge-nf-call-arptables = 0
+  ```

src/CMakeLists.txt

@@ -1,23 +1,26 @@
-find_package(Threads REQUIRED)
 find_package(nlohmann_json REQUIRED)
 include_directories(${PROJECT_SOURCE_DIR})
-include_directories(${PROJECT_SOURCE_DIR}/execsnoop-kernel/)
+include_directories(${PROJECT_SOURCE_DIR}/execsnoop-libbpf/)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-add_executable(main main.cpp
-                    common.cpp config.cpp cgroup_attach.cpp
-                    socket_client.cpp socket_server.cpp)
-target_link_libraries(main PRIVATE nlohmann_json::nlohmann_json Threads::Threads ${CMAKE_DL_LIBS})
-set_target_properties(main PROPERTIES LINKER_LANGUAGE CXX)
+configure_file(common.cmake.h ${CMAKE_CURRENT_SOURCE_DIR}/common.h)
+
+if (build_execsnoop_dl)
+    add_definitions(-DBUIlD_EXECSNOOP_DL)
+    set(DL_LIB "-ldl")
+    set(EXECSNOOP_LIB "")
+else()
+    set(EXECSNOOP_LIB "execsnoop")
+endif()
+
+add_executable(main main.cpp common.cpp config.cpp cgroup_attach.cpp socket_client.cpp socket_server.cpp)
+target_link_libraries(main PRIVATE nlohmann_json::nlohmann_json ${DL_LIB} ${EXECSNOOP_LIB})
 set_target_properties(main PROPERTIES OUTPUT_NAME cgproxy)
-install(TARGETS main RUNTIME)
+install(TARGETS main DESTINATION ${CMAKE_INSTALL_FULL_BINDIR})
 
-# # execsnoop related
-# set(execsnoop ${PROJECT_SOURCE_DIR}/execsnoop-libbpf/libexecsnoop.so)
-# add_custom_command(OUTPUT ${execsnoop}
-#     COMMAND make CFLAGS=\"-O2 -Wall -s -DNDEBUG\" libexecsnoop.so
-#     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/execsnoop-libbpf
-#     BYPRODUCTS ${PROJECT_SOURCE_DIR}/execsnoop-libbpf/build
-# )
-# add_custom_target(execsnoop ALL DEPENDS ${execsnoop})
-# install(PROGRAMS ${execsnoop} DESTINATION ${CMAKE_INSTALL_LIBDIR}/cgproxy/)
+if (build_static)
+    target_link_libraries(main PRIVATE -static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive)
+else()
+    target_link_libraries(main PRIVATE -lpthread)
+endif()

src/cgproxy.hpp

@@ -79,6 +79,7 @@ int main(int argc, char *argv[]) {
   if (attach_pid) return 0;
 
   string s = join2str(argc - shift, argv + shift, ' ');
+  debug("executing: %s", s.c_str());
   return system(s.c_str());
 }
 } // namespace CGPROXY::CGPROXY

src/cgproxyd.hpp

@@ -26,6 +26,7 @@ using namespace ::CGPROXY::CONFIG;
 using namespace ::CGPROXY::CGROUP;
 // using namespace ::CGPROXY::EXECSNOOP;
 
+#ifdef BUIlD_EXECSNOOP_DL
 namespace CGPROXY::EXECSNOOP {
 bool loadExecsnoopLib() {
   try {
@@ -48,6 +49,7 @@ bool loadExecsnoopLib() {
   }
 }
 } // namespace CGPROXY::EXECSNOOP
+#endif
 
 namespace CGPROXY::CGPROXYD {
 
@@ -232,14 +234,21 @@ class cgproxyd {
   }
 
   void startExecsnoopThread() {
+    #ifdef BUIlD_EXECSNOOP_DL
     if (!EXECSNOOP::loadExecsnoopLib() || EXECSNOOP::_startThread == NULL) {
       error("execsnoop not ready to start, maybe missing libbpf");
       return;
     }
+    #endif
 
     promise<void> status;
     future<void> status_f = status.get_future();
+    #ifdef BUIlD_EXECSNOOP_DL
     thread th(EXECSNOOP::_startThread, handle_pid_static, move(status));
+    #else
+    thread th(EXECSNOOP::startThread, handle_pid_static, move(status));
+    #endif
+
     execsnoop_thread = move(th);
 
     future_status fstatus = status_f.wait_for(chrono::seconds(THREAD_TIMEOUT));
@@ -262,7 +271,7 @@ class cgproxyd {
         }
         if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
             belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
-          debug("already in preserverd cgroup, leave alone: %d %s", pid, path.c_str());
+          info("already in preserverd cgroup, leave alone: %d %s", pid, path.c_str());
           continue;
         }
         if (!belongToCgroup(cg, config.cgroup_noproxy)) {
@@ -279,7 +288,7 @@ class cgproxyd {
         }
         if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
             belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
-          debug("already in preserverd cgroup, leave alone: %d %s", pid, path.c_str());
+          info("already in preserverd cgroup, leave alone: %d %s", pid, path.c_str());
           continue;
         }
         if (!belongToCgroup(cg, config.cgroup_proxy)) {

src/cgroup_attach.cpp

@@ -14,19 +14,8 @@
 
 namespace CGPROXY::CGROUP {
 
-string cgroup2_mount_point = get_cgroup2_mount_point();
+string cgroup2_mount_point = CGROUP2_MOUNT_POINT;
 
-string get_cgroup2_mount_point() {
-  stringstream buffer;
-  unique_ptr<FILE, decltype(&pclose)> fp(popen("findmnt -t cgroup2 -n -o TARGET", "r"),
-                                         &pclose);
-  if (!fp) return "";
-  char buf[READ_SIZE_MAX];
-  while (fgets(buf, READ_SIZE_MAX, fp.get()) != NULL) { buffer << buf; }
-  string s = buffer.str();
-  if (!s.empty()) s.pop_back(); // remove newline character
-  return s;
-}
 
 bool validate(string pid, string cgroup) {
   bool pid_v = validPid(pid);
@@ -63,7 +52,11 @@ int attach(const string pid, const string cgroup_target) {
     // return_error
   }
 
-  if (getCgroup(pid) == cgroup_target) {
+  string cg;
+
+  cg = getCgroup(pid);
+  if (cg.empty()) return_success;
+  if (cg == cgroup_target) {
     debug("%s already in %s", pid.c_str(), cgroup_target.c_str());
     return_success;
   }
@@ -73,7 +66,9 @@ int attach(const string pid, const string cgroup_target) {
 
   // wait for small period and check again
   this_thread::sleep_for(std::chrono::milliseconds(100));
-  if (getCgroup(pid) != cgroup_target && write2procs(pid, cgroup_target_procs) != 0)
+  cg = getCgroup(pid);
+  if (cg.empty()) return_success;
+  if (cg != cgroup_target && write2procs(pid, cgroup_target_procs) != 0)
     return_error;
   return_success;
 }
@@ -89,7 +84,7 @@ int write2procs(string pid, string procspath) {
 
   // maybe there some write error, for example process pid may not exist
   if (!procs) {
-    error("write %s to %s failed, maybe process %s not exist", pid.c_str(),
+    error("write %s to %s failed, maybe process %s live too short", pid.c_str(),
           procspath.c_str(), pid.c_str());
     return_error;
   }

src/cgroup_attach.h

@@ -8,7 +8,6 @@ using namespace std;
 namespace CGPROXY::CGROUP {
 extern string cgroup2_mount_point;
 bool validate(string pid, string cgroup);
-string get_cgroup2_mount_point();
 int attach(const string pid, const string cgroup_target);
 int attach(const int pid, const string cgroup_target);
 int write2procs(string pid, string procspath);

src/common.cmake.h

@@ -7,14 +7,15 @@
 #include <vector>
 using namespace std;
 
-#define TPROXY_IPTABLS_START "/usr/share/cgproxy/scripts/cgroup-tproxy.sh"
-#define TPROXY_IPTABLS_CLEAN "/usr/share/cgproxy/scripts/cgroup-tproxy.sh stop"
+#define TPROXY_IPTABLS_START "@CMAKE_INSTALL_FULL_DATADIR@/cgproxy/scripts/cgroup-tproxy.sh"
+#define TPROXY_IPTABLS_CLEAN "@CMAKE_INSTALL_FULL_DATADIR@/cgproxy/scripts/cgroup-tproxy.sh stop"
 
-#define LIBEXECSNOOP_SO "/usr/lib/cgproxy/libexecsnoop.so"
+#define LIBEXECSNOOP_SO "@CMAKE_INSTALL_FULL_LIBDIR@/cgproxy/libexecsnoop.so"
+#define CGROUP2_MOUNT_POINT "/var/run/cgproxy/cgroup2"
 #define PID_LOCK_FILE "/var/run/cgproxyd.pid"
 #define SOCKET_PATH "/tmp/cgproxy_unix_socket"
 #define LISTEN_BACKLOG 64
-#define DEFAULT_CONFIG_FILE "/etc/cgproxy/config.json"
+#define DEFAULT_CONFIG_FILE "@CMAKE_INSTALL_FULL_SYSCONFDIR@/cgproxy/config.json"
 #define READ_SIZE_MAX 128
 
 #define CGROUP_PROXY_PRESVERED "/proxy.slice"

src/config.cpp

@@ -21,6 +21,7 @@ using json = nlohmann::json;
 namespace CGPROXY::CONFIG {
 
 void Config::toEnv() {
+  setenv("cgroup_mount_point", CGROUP2_MOUNT_POINT, 1);
   setenv("program_proxy", join2str(program_proxy, ':').c_str(), 1);
   setenv("program_noproxy", join2str(program_noproxy, ':').c_str(), 1);
   setenv("cgroup_proxy", join2str(cgroup_proxy, ':').c_str(), 1);

tools/CMakeLists.txt

@@ -2,11 +2,4 @@ include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/src)
 
 add_executable(cgattach cgattach.cpp ../src/cgroup_attach.cpp ../src/common.cpp)
-install(TARGETS cgattach  DESTINATION /usr/bin PERMISSIONS ${basic_permission})
-
-if (with_execsnoop)
-add_executable(execsnoop_exec execsnoop.cpp ../src/common.cpp ../src/execsnoop.cpp)
-set_target_properties(execsnoop_exec PROPERTIES OUTPUT_NAME execsnoop)
-target_link_libraries(execsnoop_exec bcc)
-install(TARGETS execsnoop_exec  DESTINATION /usr/bin PERMISSIONS ${basic_permission})
-endif()
+install(TARGETS cgattach  DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} PERMISSIONS ${basic_permission})

tools/execsnoop.cpp

@@ -1,24 +0,0 @@
-#include "execsnoop.h"
-#include "common.h"
-#include <unistd.h>
-using namespace std;
-using namespace CGPROXY::EXECSNOOP;
-
-#define PATH_MAX_LEN 128
-
-int handle_pid(int pid) {
-  char path[PATH_MAX_LEN];
-  auto size = readlink(to_str("/proc/", pid, "/exe").c_str(), path, PATH_MAX_LEN);
-  if (size == -1) error("readlink: %s", to_str("/proc/", pid, "/exe").c_str());
-  path[size] = '\0';
-  info("%d %s", pid, path);
-  return 0;
-}
-
-int main() {
-  enable_debug = true;
-  enable_info = true;
-  callback = handle_pid;
-  execsnoop();
-  return 0;
-}

v2ray_config/readme.md

@@ -1,8 +1,8 @@
 ## Usage
-- Fill `06_outbounds_myproxy.json` with your vmess proxy config with tag `outBound_PROXY`.
+- Fill `06_outbounds_myproxy.json` with your VMess proxy config with tag `outBound_PROXY`.
 - Start with  `sudo v2ray -confdir .`
 
 ## Reference
 
-- [v2ray multi-file config](https://www.v2fly.org/chapter_02/multiple_config.html)
+- [v2ray multi-file config](https://www.v2fly.org/config/multiple_config.html)