version 0.20

If cgroup is empty, then the process is not exsit any more.
So can be ignored.

.gitignore

@@ -1,7 +1,10 @@
-build
+build*
+.cache
 .directory
-.vscode
 .clangd
 v2ray_config/proxy
 v2ray_config/06_outbounds_proxy.json
 aur-*
+gh-md-toc
+add_toc.sh
+compile_commands.json

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "execsnoop-libbpf/libbpf"]
+	path = execsnoop-libbpf/libbpf
+	url = https://github.com/libbpf/libbpf.git

.vscode/c_cpp_properties.json

@@ -0,0 +1,15 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [],
+            "defines": ["USE_VMLINUX"],
+            "compilerPath": "/usr/bin/gcc",
+            "cStandard": "c18",
+            "cppStandard": "gnu++17",
+            "intelliSenseMode": "gcc-x64",
+            "configurationProvider": "ms-vscode.cmake-tools"
+        }
+    ],
+    "version": 4
+}

.vscode/gdb-root

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec sudo -E /usr/bin/gdb "$@"

.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+    "configurations": [
+    {
+        "name": "(gdb) cgproxyd",
+        "type": "cppdbg",
+        "request": "launch",
+        "program": "${workspaceFolder}/build/src/cgproxy",
+        "args": ["--daemon", "--debug", "--execsnoop"],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "miDebuggerPath": "${workspaceFolder}/.vscode/gdb-root",
+        "setupCommands": [
+            {
+                "description": "Enable pretty-printing for gdb",
+                "text": "-enable-pretty-printing",
+                "ignoreFailures": true
+            }
+        ]
+    }]
+}

.vscode/settings.json

@@ -0,0 +1,9 @@
+{
+    "cmake.configureOnOpen": true,
+    "cmake.configureArgs": [
+        "-Dbuild_static=OFF",
+        "-Dbuild_execsnoop_dl=ON",
+        "-Dbuild_tools=ON",
+        "-Dbuild_test=ON"
+    ]
+}

CMakeLists.txt

@@ -1,17 +1,24 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.14)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-project(cgproxy VERSION 0.14)
-set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -Wno-unused-result")
+project(cgproxy VERSION 0.20)
 
-option(with_execsnoop "enable program level proxy control feature, need bcc installed" ON)
+include(GNUInstallDirs)
+
+add_compile_options(-Wall -Wextra -Wpedantic -Wno-unused-result -Wno-unused-parameter -Wno-overlength-strings)
+
+# for clangd
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+# option(with_execsnoop "enable program level proxy control feature, need bcc installed" ON)
+option(build_execsnoop_dl "build libexecsnoop.so which will be dynamic loaded, otherwise built directly into cgproxy" ON)
+option(build_static "build with static link prefered" OFF)
 option(build_tools OFF)
-option(build_test  OFF)
-
-set(basic_permission OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+option(build_test "for develop" OFF)
 
 add_subdirectory(src)
+add_subdirectory(execsnoop-libbpf)
 add_subdirectory(pack)
 if (build_tools)
     add_subdirectory(tools)
@@ -20,23 +27,23 @@ if (build_test)
     add_subdirectory(test)
 endif()
 
-install(FILES cgproxyd DESTINATION /usr/bin PERMISSIONS ${basic_permission})
-install(FILES cgnoproxy DESTINATION /usr/bin PERMISSIONS ${basic_permission})
-install(FILES cgproxy.service DESTINATION /usr/lib/systemd/system/)
-install(FILES config.json DESTINATION /etc/cgproxy/)
-install(FILES cgroup-tproxy.sh DESTINATION /usr/share/cgproxy/scripts/ PERMISSIONS ${basic_permission})
-install(FILES readme.md DESTINATION /usr/share/doc/cgproxy/)
+configure_file(cgnoproxy.cmake cgnoproxy)
+configure_file(cgproxyd.cmake cgproxyd)
+configure_file(cgproxy.service.cmake cgproxy.service)
+
+# instal scripts and other things
+install(PROGRAMS ${CMAKE_BINARY_DIR}/cgproxyd DESTINATION ${CMAKE_INSTALL_FULL_BINDIR})
+install(PROGRAMS ${CMAKE_BINARY_DIR}/cgnoproxy DESTINATION ${CMAKE_INSTALL_FULL_BINDIR})
+install(PROGRAMS cgroup-tproxy.sh DESTINATION ${CMAKE_INSTALL_FULL_DATADIR}/cgproxy/scripts)
+install(FILES ${CMAKE_BINARY_DIR}/cgproxy.service DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/systemd/system)
+install(FILES config.json DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/cgproxy)
+install(FILES readme.md DESTINATION ${CMAKE_INSTALL_FULL_DOCDIR})
 
 # man pages
-set(man_gz
-${PROJECT_BINARY_DIR}/cgproxyd.1.gz
-${PROJECT_BINARY_DIR}/cgproxy.1.gz
-${PROJECT_BINARY_DIR}/cgnoproxy.1.gz
-)
-add_custom_target(man
+set(man_gz ${PROJECT_SOURCE_DIR}/man/cgproxyd.1.gz ${PROJECT_SOURCE_DIR}/man/cgproxy.1.gz ${PROJECT_SOURCE_DIR}/man/cgnoproxy.1.gz)
+add_custom_command(OUTPUT ${man_gz}
     COMMAND gzip -fk cgproxyd.1 cgproxy.1 cgnoproxy.1
-    COMMAND mv *.gz ${PROJECT_BINARY_DIR}
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/man
 )
-add_dependencies(main man)
-install(FILES ${man_gz} DESTINATION /usr/share/man/man1/)
+add_custom_target(man ALL DEPENDS ${man_gz})
+install(FILES ${man_gz} DESTINATION ${CMAKE_INSTALL_FULL_MANDIR}/man1/)

cgnoproxy

@@ -1,2 +0,0 @@
-#!/bin/sh
-exec /usr/bin/cgproxy --noproxy $@

cgnoproxy.cmake

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec @CMAKE_INSTALL_FULL_BINDIR@/cgproxy --noproxy $@

cgproxy.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=cgproxy service
-After=network.target
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/cgproxyd --execsnoop
-
-[Install]
-WantedBy=multi-user.target

cgproxy.service.cmake

@@ -0,0 +1,10 @@
+[Unit]
+Description=cgproxy service
+After=network.target network-online.target
+
+[Service]
+Type=simple
+ExecStart=@CMAKE_INSTALL_FULL_BINDIR@/cgproxyd --execsnoop
+
+[Install]
+WantedBy=multi-user.target

cgproxyd

@@ -1,2 +0,0 @@
-#!/bin/sh
-exec /usr/bin/cgproxy --daemon $@

cgproxyd.cmake

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec @CMAKE_INSTALL_FULL_BINDIR@/cgproxy --daemon $@

cgroup-tproxy.sh

@@ -1,37 +1,39 @@
 #!/bin/bash
+### This script will proxy/noproxy anything running in specific cgroup
+### need cgroup2 support, and iptables cgroup2 path match support
+### 
+### script usage:
+###     cgroup-tproxy.sh [--help|--config|stop]
+### options:
+###     --config=FILE   load config from file
+###            --help   show help info
+###              stop   clean then stop. Variables may change when stopping, which should be avoid
+###                     so always stop first in the last context before start new context
+###
+### available variables with default value:
+###     cgroup_noproxy="/noproxy.slice" 
+###     cgroup_proxy="/proxy.slice" 
+###     port=12345
+###     enable_dns=true
+###     enable_udp=true
+###     enable_tcp=true
+###     enable_ipv4=true
+###     enable_ipv6=true
+###     enable_gateway=false
+###     table=10007
+###     fwmark=0x9973
+###     cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET | head -n 1)
+###
+### semicolon to seperate multi cgroup:
+###     cgroup_noproxy="/noproxy1.slice:/noproxy2.slice"
+###     cgroup_proxy="/proxy1.slice:/proxy2.slice"
+
 print_help(){
-cat << 'DOC'
-#############################################################################
-# 
-# 1. This script need cgroup v2 
-# 
-# 2. Listening port is expected to accept iptables TPROXY, while REDIRECT 
-#    will not work in this script, because REDIRECT only support tcp/ipv4
-# 
-# 3. TPROXY need root or special capability whatever process is listening on port
-#    v2ray as example: 
-#       sudo setcap "cap_net_bind_service=+ep cap_net_admin=+ep" /usr/lib/v2ray/v2ray
-# 
-# 4. this script will proxy anything running in specific cgroup
-# 
-# script usage:
-#     cgroup-tproxy.sh [--help|--config|stop]
-#     --config=FILE
-#           load config from file
-#     --help
-#           show help info
-#     stop
-#           clean then stop
-# 
-# proxy usage:
-#     cgproxy <program>
-# 
-#############################################################################
-DOC
+    sed -rn 's/^### ?//;T;p' "$0"
 }
 
 ## check root
-[ ! $(id -u) -eq 0 ] && { >&2 echo "need root to modify iptables";exit -1; }
+[ ! $(id -u) -eq 0 ] && { >&2 echo "iptables: need root to modify iptables";exit -1; }
 
 ## any process in this cgroup will be proxied
 if [ -z ${cgroup_proxy+x} ]; then  
@@ -47,56 +49,75 @@ else
     IFS=':' read -r -a cgroup_noproxy   <<< "$cgroup_noproxy"
 fi
 
-# allow as gateway for local network
-[ -z ${enable_gateway+x} ] && enable_gateway=false
-
-## some variables
+## tproxy listening port
 [ -z ${port+x} ] && port=12345
 
-## some options
+## controll options
 [ -z ${enable_dns+x} ]  && enable_dns=true
-[ -z ${enable_tcp+x} ]  && enable_tcp=true
 [ -z ${enable_udp+x} ]  && enable_udp=true
+[ -z ${enable_tcp+x} ]  && enable_tcp=true
 [ -z ${enable_ipv4+x} ] && enable_ipv4=true
 [ -z ${enable_ipv6+x} ] && enable_ipv6=true
+[ -z ${enable_gateway+x} ] && enable_gateway=false
 
-## do not modify this if you don't known what you are doing
-table=100
-fwmark=0x01
-make_newin=0x02
+## mark/route things
+[ -z ${table+x} ]           && table=10007
+[ -z ${fwmark+x} ]          && fwmark=0x9973
+[ -z ${table_reroute+x} ]   && table_reroute=$table
+[ -z ${table_tproxy+x} ]    && table_tproxy=$table
+[ -z ${fwmark_reroute+x} ]  && fwmark_reroute=$fwmark
+[ -z ${fwmark_tproxy+x} ]   && fwmark_tproxy=$fwmark
 
-## cgroup things
-cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET)
-cgroup_type="cgroup2"
-cgroup_procs_file="cgroup.procs"
+## cgroup mount point things
+[ -z ${cgroup_mount_point+x} ] && cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET | head -n 1)
 
 
 stop(){
-    iptables -t mangle -L TPROXY_PRE &> /dev/null || return
-    echo "cleaning tproxy iptables"
-    iptables -t mangle -D PREROUTING -j TPROXY_PRE
-    iptables -t mangle -D OUTPUT -j TPROXY_OUT
-    iptables -t mangle -F TPROXY_PRE
-    iptables -t mangle -F TPROXY_OUT
-    iptables -t mangle -F TPROXY_ENT
-    iptables -t mangle -X TPROXY_PRE
-    iptables -t mangle -X TPROXY_OUT
-    iptables -t mangle -X TPROXY_ENT
-    ip6tables -t mangle -D PREROUTING -j TPROXY_PRE
-    ip6tables -t mangle -D OUTPUT -j TPROXY_OUT
-    ip6tables -t mangle -F TPROXY_PRE
-    ip6tables -t mangle -F TPROXY_OUT
-    ip6tables -t mangle -F TPROXY_ENT
-    ip6tables -t mangle -X TPROXY_PRE
-    ip6tables -t mangle -X TPROXY_OUT
-    ip6tables -t mangle -X TPROXY_ENT
-    ip rule delete fwmark $fwmark lookup $table
-    ip route flush table $table
-    ip -6 rule delete fwmark $fwmark lookup $table
-    ip -6 route flush table $table
+    iptables -w 60 -t mangle -L TPROXY_ENT &> /dev/null || return
+    echo "iptables: cleaning tproxy iptables"
+
+    iptables -w 60 -t mangle -D PREROUTING -j TPROXY_PRE
+    iptables -w 60 -t mangle -D OUTPUT -j TPROXY_OUT
+
+    iptables -w 60 -t mangle -F TPROXY_PRE
+    iptables -w 60 -t mangle -F TPROXY_ENT
+    iptables -w 60 -t mangle -F TPROXY_OUT
+    iptables -w 60 -t mangle -F TPROXY_MARK
+
+    iptables -w 60 -t mangle -X TPROXY_PRE
+    iptables -w 60 -t mangle -X TPROXY_ENT
+    iptables -w 60 -t mangle -X TPROXY_OUT
+    iptables -w 60 -t mangle -X TPROXY_MARK
+
+    ip rule delete fwmark $fwmark_tproxy lookup $table_tproxy
+    ip rule delete fwmark $fwmark_reroute lookup $table_reroute &> /dev/null
+    ip route flush table $table_tproxy
+    ip route flush table $table_reroute &> /dev/null
+
+    ip6tables -w 60 -t mangle -D PREROUTING -j TPROXY_PRE
+    ip6tables -w 60 -t mangle -D OUTPUT -j TPROXY_OUT
+
+    ip6tables -w 60 -t mangle -F TPROXY_PRE
+    ip6tables -w 60 -t mangle -F TPROXY_OUT
+    ip6tables -w 60 -t mangle -F TPROXY_ENT
+    ip6tables -w 60 -t mangle -F TPROXY_MARK
+
+    ip6tables -w 60 -t mangle -X TPROXY_PRE
+    ip6tables -w 60 -t mangle -X TPROXY_OUT
+    ip6tables -w 60 -t mangle -X TPROXY_ENT
+    ip6tables -w 60 -t mangle -X TPROXY_MARK
+
+    ip -6 rule delete fwmark $fwmark_tproxy lookup $table_tproxy
+    ip -6 rule delete fwmark $fwmark_reroute lookup $table_reroute &> /dev/null
+    ip -6 route flush table $table_tproxy
+    ip -6 route flush table $table_reroute &> /dev/null
+
     ## may not exist, just ignore, and tracking their existence is not reliable
-    iptables -t nat -D POSTROUTING -m owner ! --socket-exists -j MASQUERADE &> /dev/null
-    ip6tables -t nat -D POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE &> /dev/null
+    iptables -w 60 -t nat -D POSTROUTING -m owner ! --socket-exists -j MASQUERADE &> /dev/null
+    ip6tables -w 60 -t nat -D POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE &> /dev/null
+
+    ## unmount cgroup2
+    [ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" = "cgroup2" ] && umount $cgroup_mount_point
 }
 
 ## parse parameter
@@ -115,120 +136,139 @@ case $i in
         print_help
         exit 0
         ;;
+    *)
+        print_help
+        exit 0
+        ;;
 esac
 done
 
-## TODO cgroup need to exists before using in iptables since 5.6.5, maybe it's bug
+
+## check cgroup_mount_point, create and mount if necessary
+[ -z $cgroup_mount_point ] && { >&2 echo "iptables: no cgroup2 mount point available"; exit -1; }
+[ ! -d $cgroup_mount_point ] && mkdir -p $cgroup_mount_point
+[ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" != "cgroup2" ] && mount -t cgroup2 none $cgroup_mount_point
+[ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" != "cgroup2" ] && { >&2 echo "iptables: mount $cgroup_mount_point failed"; exit -1; }
+
 ## only create the first one in arrary
 test -d $cgroup_mount_point$cgroup_proxy    || mkdir $cgroup_mount_point$cgroup_proxy   || exit -1; 
 test -d $cgroup_mount_point$cgroup_noproxy  || mkdir $cgroup_mount_point$cgroup_noproxy || exit -1; 
 
-
-echo "applying tproxy iptables"
-## use TPROXY
-#ipv4#
-ip rule add fwmark $fwmark table $table
-ip route add local default dev lo table $table
-iptables -t mangle -N TPROXY_ENT
-iptables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
-iptables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
-
-iptables -t mangle -N TPROXY_PRE
-iptables -t mangle -A TPROXY_PRE -m socket --transparent -j MARK --set-mark $fwmark
-iptables -t mangle -A TPROXY_PRE -m socket --transparent -j RETURN
-iptables -t mangle -A TPROXY_PRE -p icmp -j RETURN
-iptables -t mangle -A TPROXY_PRE -p udp --dport 53 -j TPROXY_ENT
-iptables -t mangle -A TPROXY_PRE -p tcp --dport 53 -j TPROXY_ENT
-iptables -t mangle -A TPROXY_PRE -m addrtype --dst-type LOCAL -j RETURN
-iptables -t mangle -A TPROXY_PRE -m addrtype ! --dst-type UNICAST -j RETURN
-iptables -t mangle -A TPROXY_PRE -j TPROXY_ENT
-iptables -t mangle -A PREROUTING -j TPROXY_PRE
-
-iptables -t mangle -N TPROXY_OUT
-iptables -t mangle -A TPROXY_OUT -p icmp -j RETURN
-iptables -t mangle -A TPROXY_OUT -m connmark --mark  $make_newin -j RETURN
-iptables -t mangle -A TPROXY_OUT -m addrtype --dst-type LOCAL -j RETURN
-iptables -t mangle -A TPROXY_OUT -m addrtype ! --dst-type UNICAST -j RETURN
+## filter cgroup that not exist
+_cgroup_noproxy=()
 for cg in ${cgroup_noproxy[@]}; do
-iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
+    test -d $cgroup_mount_point$cg && _cgroup_noproxy+=($cg) || { >&2 echo "iptables: $cg not exist, ignore";}
 done
+unset cgroup_noproxy && cgroup_noproxy=${_cgroup_noproxy[@]}
+
+## filter cgroup that not exist
+_cgroup_proxy=()
 for cg in ${cgroup_proxy[@]}; do
-iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark
+    test -d $cgroup_mount_point$cg && _cgroup_proxy+=($cg) || { >&2 echo "iptables: $cg not exist, ignore";}
 done
-iptables -t mangle -A OUTPUT -j TPROXY_OUT
+unset cgroup_proxy && cgroup_proxy=${_cgroup_proxy[@]}
 
-#ipv6#
-ip -6 rule add fwmark $fwmark table $table
-ip -6 route add local default dev lo table $table
-ip6tables -t mangle -N TPROXY_ENT
-ip6tables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
-ip6tables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
 
-ip6tables -t mangle -N TPROXY_PRE
-ip6tables -t mangle -A TPROXY_PRE -m socket --transparent -j MARK --set-mark $fwmark
-ip6tables -t mangle -A TPROXY_PRE -m socket --transparent -j RETURN
-ip6tables -t mangle -A TPROXY_PRE -p icmpv6 -j RETURN
-ip6tables -t mangle -A TPROXY_PRE -p udp --dport 53 -j TPROXY_ENT
-ip6tables -t mangle -A TPROXY_PRE -p tcp --dport 53 -j TPROXY_ENT
-ip6tables -t mangle -A TPROXY_PRE -m addrtype --dst-type LOCAL -j RETURN
-ip6tables -t mangle -A TPROXY_PRE -m addrtype ! --dst-type UNICAST -j RETURN
-ip6tables -t mangle -A TPROXY_PRE -j TPROXY_ENT
-ip6tables -t mangle -A PREROUTING -j TPROXY_PRE
+## ipv4 #########################################################################
+echo "iptables: applying tproxy iptables"
+## mangle prerouting
+ip rule add fwmark $fwmark_tproxy table $table_tproxy
+ip route add local default dev lo table $table_tproxy
+# core
+iptables -w 60 -t mangle -N TPROXY_ENT
+iptables -w 60 -t mangle -A TPROXY_ENT -m socket -j MARK --set-mark $fwmark_tproxy
+iptables -w 60 -t mangle -A TPROXY_ENT -m socket -j ACCEPT
+iptables -w 60 -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark_tproxy
+iptables -w 60 -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark_tproxy
+# filter
+iptables -w 60 -t mangle -N TPROXY_PRE
+iptables -w 60 -t mangle -A TPROXY_PRE -m addrtype --dst-type LOCAL -j RETURN
+iptables -w 60 -t mangle -A TPROXY_PRE -m addrtype ! --dst-type UNICAST -j RETURN
+$enable_gateway  || iptables -w 60 -t mangle -A TPROXY_PRE -m addrtype ! --src-type LOCAL -j RETURN
+$enable_dns && iptables -w 60 -t mangle -A TPROXY_PRE -p udp --dport 53 -j TPROXY_ENT
+$enable_udp && iptables -w 60 -t mangle -A TPROXY_PRE -p udp -j TPROXY_ENT
+$enable_tcp && iptables -w 60 -t mangle -A TPROXY_PRE -p tcp -j TPROXY_ENT
+# hook
+iptables -w 60 -t mangle -A PREROUTING -j TPROXY_PRE
 
-ip6tables -t mangle -N TPROXY_OUT
-ip6tables -t mangle -A TPROXY_OUT -p icmpv6 -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m connmark --mark  $make_newin -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m addrtype --dst-type LOCAL -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m addrtype ! --dst-type UNICAST -j RETURN
-for cg in ${cgroup_noproxy[@]}; do
-ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
-done
-for cg in ${cgroup_proxy[@]}; do
-ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark
-done
-ip6tables -t mangle -A OUTPUT -j TPROXY_OUT
-
-## allow to disable, order is important
-$enable_dns     || iptables  -t mangle -I TPROXY_OUT -p udp --dport 53 -j RETURN
-$enable_dns     || ip6tables -t mangle -I TPROXY_OUT -p udp --dport 53 -j RETURN
-$enable_udp     || iptables  -t mangle -I TPROXY_OUT -p udp -j RETURN
-$enable_udp     || ip6tables -t mangle -I TPROXY_OUT -p udp -j RETURN
-$enable_tcp     || iptables  -t mangle -I TPROXY_OUT -p tcp -j RETURN
-$enable_tcp     || ip6tables -t mangle -I TPROXY_OUT -p tcp -j RETURN
-$enable_ipv4    || iptables  -t mangle -I TPROXY_OUT -j RETURN
-$enable_ipv6    || ip6tables -t mangle -I TPROXY_OUT -j RETURN
-
-if $enable_gateway; then
-$enable_dns     || iptables  -t mangle -I TPROXY_PRE -p udp --dport 53 -j RETURN
-$enable_dns     || ip6tables -t mangle -I TPROXY_PRE -p udp --dport 53 -j RETURN
-$enable_udp     || iptables  -t mangle -I TPROXY_PRE -p udp -j RETURN
-$enable_udp     || ip6tables -t mangle -I TPROXY_PRE -p udp -j RETURN
-$enable_tcp     || iptables  -t mangle -I TPROXY_PRE -p tcp -j RETURN
-$enable_tcp     || ip6tables -t mangle -I TPROXY_PRE -p tcp -j RETURN
-$enable_ipv4    || iptables  -t mangle -I TPROXY_PRE -j RETURN
-$enable_ipv6    || ip6tables -t mangle -I TPROXY_PRE -j RETURN
+## mangle output
+if [ $fwmark_reroute != $fwmark_tproxy ]; then
+ip rule add fwmark $fwmark_reroute table $table_reroute
+ip route add local default dev lo table $table_reroute
 fi
+# filter
+iptables -w 60 -t mangle -N TPROXY_MARK
+iptables -w 60 -t mangle -A TPROXY_MARK -m addrtype ! --dst-type UNICAST -j RETURN
+$enable_dns && iptables -w 60 -t mangle -A TPROXY_MARK -p udp --dport 53 -j MARK --set-mark $fwmark_reroute
+$enable_udp && iptables -w 60 -t mangle -A TPROXY_MARK -p udp -j MARK --set-mark $fwmark_reroute
+$enable_tcp && iptables -w 60 -t mangle -A TPROXY_MARK -p tcp -j MARK --set-mark $fwmark_reroute
+# cgroup
+iptables -w 60 -t mangle -N TPROXY_OUT
+iptables -w 60 -t mangle -A TPROXY_OUT -m conntrack --ctdir REPLY -j RETURN
+for cg in ${cgroup_noproxy[@]}; do
+iptables -w 60 -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
+done
+for cg in ${cgroup_proxy[@]}; do
+iptables -w 60 -t mangle -A TPROXY_OUT -m cgroup --path $cg -j TPROXY_MARK
+done
+# hook
+$enable_ipv4 && iptables -w 60 -t mangle -A OUTPUT -j TPROXY_OUT
 
-## do not handle local device connection through tproxy if gateway is not enabled
-$enable_gateway || iptables  -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -j RETURN
-$enable_gateway || ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -j RETURN
+## ipv6 #########################################################################
+## mangle prerouting
+ip -6 rule add fwmark $fwmark_tproxy table $table_tproxy
+ip -6 route add local default dev lo table $table_tproxy
+# core
+ip6tables -w 60 -t mangle -N TPROXY_ENT
+ip6tables -w 60 -t mangle -A TPROXY_ENT -m socket -j MARK --set-mark $fwmark_tproxy
+ip6tables -w 60 -t mangle -A TPROXY_ENT -m socket -j ACCEPT
+ip6tables -w 60 -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark_tproxy
+ip6tables -w 60 -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark_tproxy
+# filter
+ip6tables -w 60 -t mangle -N TPROXY_PRE
+ip6tables -w 60 -t mangle -A TPROXY_PRE -m addrtype --dst-type LOCAL -j RETURN
+ip6tables -w 60 -t mangle -A TPROXY_PRE -m addrtype ! --dst-type UNICAST -j RETURN
+$enable_gateway  || ip6tables -w 60 -t mangle -A TPROXY_PRE -m addrtype ! --src-type LOCAL -j RETURN
+$enable_dns && ip6tables -w 60 -t mangle -A TPROXY_PRE -p udp --dport 53 -j TPROXY_ENT
+$enable_udp && ip6tables -w 60 -t mangle -A TPROXY_PRE -p udp -j TPROXY_ENT
+$enable_tcp && ip6tables -w 60 -t mangle -A TPROXY_PRE -p tcp -j TPROXY_ENT
+# hook
+ip6tables -w 60 -t mangle -A PREROUTING -j TPROXY_PRE
 
-## make sure following rules are the first in chain TPROXY_PRE to mark new incoming connection or gateway proxy connection
-## so must put at last to insert first
-iptables  -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $make_newin
-ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $make_newin
+## mangle output
+if [ $fwmark_reroute != $fwmark_tproxy ]; then
+ip -6 rule add fwmark $fwmark_reroute table $table_reroute
+ip -6 route add local default dev lo table $table_reroute
+fi
+# filter
+ip6tables -w 60 -t mangle -N TPROXY_MARK
+ip6tables -w 60 -t mangle -A TPROXY_MARK -m addrtype ! --dst-type UNICAST -j RETURN
+$enable_dns && ip6tables -w 60 -t mangle -A TPROXY_MARK -p udp --dport 53 -j MARK --set-mark $fwmark_reroute
+$enable_udp && ip6tables -w 60 -t mangle -A TPROXY_MARK -p udp -j MARK --set-mark $fwmark_reroute
+$enable_tcp && ip6tables -w 60 -t mangle -A TPROXY_MARK -p tcp -j MARK --set-mark $fwmark_reroute
+# cgroup
+ip6tables -w 60 -t mangle -N TPROXY_OUT
+ip6tables -w 60 -t mangle -A TPROXY_OUT -m conntrack --ctdir REPLY -j RETURN
+for cg in ${cgroup_noproxy[@]}; do
+ip6tables -w 60 -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
+done
+for cg in ${cgroup_proxy[@]}; do
+ip6tables -w 60 -t mangle -A TPROXY_OUT -m cgroup --path $cg -j TPROXY_MARK
+done
+# hook
+$enable_ipv6 && ip6tables -w 60 -t mangle -A OUTPUT -j TPROXY_OUT
+
+## forward #######################################################################
+if $enable_gateway; then
+    iptables  -t nat -A POSTROUTING -m owner ! --socket-exists -j MASQUERADE
+    ip6tables -w 60 -t nat -A POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE # only masquerade ipv6 private address
+    sysctl -w net.ipv4.ip_forward=1
+    sysctl -w net.ipv6.conf.all.forwarding=1
+    echo "iptables: gateway enabled"
+fi
 
 ## message for user
 cat << DOC
-noproxy cgroup: ${cgroup_noproxy[@]}
-proxied cgroup: ${cgroup_proxy[@]}
+iptables: noproxy cgroup: ${cgroup_noproxy[@]}
+iptables: proxied cgroup: ${cgroup_proxy[@]}
 DOC
-
-
-if $enable_gateway; then
-    iptables  -t nat -A POSTROUTING -m owner ! --socket-exists -j MASQUERADE
-    ip6tables -t nat -A POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE # only masquerade ipv6 private address
-    sysctl -w net.ipv4.ip_forward=1
-    sysctl -w net.ipv6.conf.all.forwarding=1
-    echo "gateway enabled"
-fi

config.json

@@ -1,4 +1,6 @@
 {
+    "comment":"For usage, see https://github.com/springzfx/cgproxy",
+
     "port": 12345,
     "program_noproxy": ["v2ray", "qv2ray"],
     "program_proxy": [],
@@ -9,5 +11,7 @@
     "enable_udp": true,
     "enable_tcp": true,
     "enable_ipv4": true,
-    "enable_ipv6": true
+    "enable_ipv6": true,
+    "table": 10007,
+    "fwmark": 39283
 }

execsnoop-bcc/CMakeLists.txt

@@ -0,0 +1,6 @@
+include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+add_library(execsnoop MODULE execsnoop.cpp ../src/common.cpp)
+target_link_libraries(execsnoop bcc)
+install(TARGETS execsnoop DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cgproxy/ PERMISSIONS ${basic_permission})

execsnoop-bcc/execsnoop.cpp

@@ -48,6 +48,7 @@ struct data_t {
 };
 
 function<int(int)> callback = NULL;
+promise<void> status;
 
 void handle_events(void *cb_cookie, void *data, int data_size) {
   auto event = static_cast<data_t *>(data);
@@ -59,8 +60,10 @@ void handle_events(void *cb_cookie, void *data, int data_size) {
 int execsnoop() {
   debug("starting execsnoop");
   ebpf::BPF bpf;
+
   auto init_res = bpf.init(BPF_PROGRAM);
   if (init_res.code() != 0) {
+    error("bpf init failed, maybe linux-headers not installed");
     std::cerr << init_res.msg() << std::endl;
     return 1;
   }
@@ -85,18 +88,17 @@ int execsnoop() {
     return 1;
   }
 
+  status.set_value();
+
   while (true) bpf.poll_perf_buffer("events");
 
   return 0;
 }
 
-void *startThread(void *arg) {
-  thread_arg *p = (thread_arg *)arg;
-  callback = p->handle_pid;
+void startThread(function<int(int)> c, promise<void> _status) {
+  status = move(_status);
+  callback = c;
   execsnoop();
-  return (void *)0;
 }
 
-} // namespace CGPROXY::EXECSNOOP
-
-extern "C" void *_startThread(void *arg) { return CGPROXY::EXECSNOOP::startThread(arg); }
+} // namespace CGPROXY::EXECSNOOP

execsnoop-bcc/execsnoop.h

@@ -2,6 +2,7 @@
 #define EXECSNOOP_HPP 1
 
 #include <functional>
+#include <future>
 #include <string>
 using namespace std;
 
@@ -13,10 +14,10 @@ extern function<int(int)> callback;
 void handle_events(void *cb_cookie, void *data, int data_size);
 int execsnoop();
 
-struct thread_arg {
-  function<int(int)> handle_pid;
-};
-void *startThread(void *arg);
+extern "C" void startThread(function<int(int)> c, promise<void> _status);
+// typedef void startThread_t(function<int(int)>, promise<void>);
+using startThread_t=decltype(startThread);
+startThread_t *_startThread; // only for dlsym()
 
 } // namespace CGPROXY::EXECSNOOP
 #endif

execsnoop-bcc/readme.md

@@ -0,0 +1,3 @@
+- depend [bcc](https://github.com/iovisor/bcc) 
+
+- huge memory usage, at least 50M

execsnoop-kernel/CMakeLists.txt

@@ -0,0 +1,15 @@
+# find libbpf
+if (build_static)
+    find_library(LIBBPF libbpf.a REQUIRED)
+else()
+    find_library(LIBBPF bpf REQUIRED)
+endif()
+
+if (build_execsnoop_dl)
+    add_library(execsnoop MODULE execsnoop_share.cpp)
+    install(TARGETS execsnoop DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cgproxy/)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF} -lelf -lz)
+else()
+    add_library(execsnoop STATIC execsnoop_share.cpp)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF} -l:libelf.a -l:libz.a)
+endif()

execsnoop-kernel/aarch64/execsnoop_kern_skel.h

@@ -0,0 +1,216 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+
+/* THIS FILE IS AUTOGENERATED! */
+#ifndef __EXECSNOOP_KERN_SKEL_H__
+#define __EXECSNOOP_KERN_SKEL_H__
+
+#include <stdlib.h>
+#include <bpf/libbpf.h>
+
+struct execsnoop_kern {
+	struct bpf_object_skeleton *skeleton;
+	struct bpf_object *obj;
+	struct {
+		struct bpf_map *perf_events;
+		struct bpf_map *records;
+	} maps;
+	struct {
+		struct bpf_program *syscall_enter_execve;
+		struct bpf_program *syscall_exit_execve;
+	} progs;
+	struct {
+		struct bpf_link *syscall_enter_execve;
+		struct bpf_link *syscall_exit_execve;
+	} links;
+};
+
+static void
+execsnoop_kern__destroy(struct execsnoop_kern *obj)
+{
+	if (!obj)
+		return;
+	if (obj->skeleton)
+		bpf_object__destroy_skeleton(obj->skeleton);
+	free(obj);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj);
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_opts(const struct bpf_object_open_opts *opts)
+{
+	struct execsnoop_kern *obj;
+
+	obj = (typeof(obj))calloc(1, sizeof(*obj));
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__create_skeleton(obj))
+		goto err;
+	if (bpf_object__open_skeleton(obj->skeleton, opts))
+		goto err;
+
+	return obj;
+err:
+	execsnoop_kern__destroy(obj);
+	return NULL;
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open(void)
+{
+	return execsnoop_kern__open_opts(NULL);
+}
+
+static inline int
+execsnoop_kern__load(struct execsnoop_kern *obj)
+{
+	return bpf_object__load_skeleton(obj->skeleton);
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_and_load(void)
+{
+	struct execsnoop_kern *obj;
+
+	obj = execsnoop_kern__open();
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__load(obj)) {
+		execsnoop_kern__destroy(obj);
+		return NULL;
+	}
+	return obj;
+}
+
+static inline int
+execsnoop_kern__attach(struct execsnoop_kern *obj)
+{
+	return bpf_object__attach_skeleton(obj->skeleton);
+}
+
+static inline void
+execsnoop_kern__detach(struct execsnoop_kern *obj)
+{
+	return bpf_object__detach_skeleton(obj->skeleton);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj)
+{
+	struct bpf_object_skeleton *s;
+
+	s = (typeof(s))calloc(1, sizeof(*s));
+	if (!s)
+		return -1;
+	obj->skeleton = s;
+
+	s->sz = sizeof(*s);
+	s->name = "execsnoop_kern";
+	s->obj = &obj->obj;
+
+	/* maps */
+	s->map_cnt = 2;
+	s->map_skel_sz = sizeof(*s->maps);
+	s->maps = (typeof(s->maps))calloc(s->map_cnt, s->map_skel_sz);
+	if (!s->maps)
+		goto err;
+
+	s->maps[0].name = "perf_events";
+	s->maps[0].map = &obj->maps.perf_events;
+
+	s->maps[1].name = "records";
+	s->maps[1].map = &obj->maps.records;
+
+	/* programs */
+	s->prog_cnt = 2;
+	s->prog_skel_sz = sizeof(*s->progs);
+	s->progs = (typeof(s->progs))calloc(s->prog_cnt, s->prog_skel_sz);
+	if (!s->progs)
+		goto err;
+
+	s->progs[0].name = "syscall_enter_execve";
+	s->progs[0].prog = &obj->progs.syscall_enter_execve;
+	s->progs[0].link = &obj->links.syscall_enter_execve;
+
+	s->progs[1].name = "syscall_exit_execve";
+	s->progs[1].prog = &obj->progs.syscall_exit_execve;
+	s->progs[1].link = &obj->links.syscall_exit_execve;
+
+	s->data_sz = 2024;
+	s->data = (void *)"\
+\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\x28\x05\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0b\0\
+\x01\0\x85\0\0\0\x0e\0\0\0\xbf\x06\0\0\0\0\0\0\x63\x6a\xfc\xff\0\0\0\0\x85\0\0\
+\0\x0f\0\0\0\xbf\x07\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\
+\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\
+\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
+\xd0\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb7\x04\0\0\x01\0\0\0\x85\
+\0\0\0\x02\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x19\0\0\0\0\0\
+\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\0\x15\x08\x12\0\0\0\0\0\x77\x06\0\
+\0\x20\0\0\0\x61\xa1\xfc\xff\0\0\0\0\x63\x78\x1c\0\0\0\0\0\x63\x68\x14\0\0\0\0\
+\0\x63\x18\x10\0\0\0\0\0\x85\0\0\0\x23\0\0\0\x07\0\0\0\x78\x04\0\0\xbf\xa1\0\0\
+\0\0\0\0\x07\x01\0\0\xf0\xff\xff\xff\xb7\x02\0\0\x08\0\0\0\xbf\x03\0\0\0\0\0\0\
+\x85\0\0\0\x04\0\0\0\x07\x08\0\0\x18\0\0\0\x79\xa3\xf0\xff\0\0\0\0\x07\x03\0\0\
+\x6c\x04\0\0\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\
+\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\xbf\x16\0\0\0\0\0\0\x85\0\0\0\x0e\0\0\0\x63\
+\x0a\xfc\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x15\x07\x13\
+\0\0\0\0\0\x79\x61\x10\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\x6d\x12\x0b\0\0\0\0\0\xbf\
+\x71\0\0\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x85\0\0\0\x10\0\0\0\xbf\x61\0\0\0\0\0\0\
+\x18\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\x03\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\
+\0\xbf\x74\0\0\0\0\0\0\xb7\x05\0\0\x20\0\0\0\x85\0\0\0\x19\0\0\0\xbf\xa2\0\0\0\
+\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\
+\0\x03\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x01\0\0\0\x04\0\0\0\x20\0\0\0\
+\0\x28\0\0\0\0\0\0\x04\0\0\0\x04\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\
+\0\x06\x07\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xac\0\0\0\x04\
+\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdb\0\0\0\0\0\x03\0\x70\x01\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\xd4\0\0\0\0\0\x05\0\xc0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xcd\
+\0\0\0\0\0\x05\0\xe8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa3\0\0\0\x11\0\x08\0\0\0\0\
+\0\0\0\0\0\x04\0\0\0\0\0\0\0\x20\0\0\0\x11\0\x09\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\
+\0\0\0\x07\0\0\0\x11\0\x07\0\x14\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x18\0\0\0\x11\
+\0\x07\0\0\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x8e\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\
+\0\x80\x01\0\0\0\0\0\0\x51\0\0\0\x12\0\x05\0\0\0\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\
+\x70\0\0\0\0\0\0\0\x01\0\0\0\x08\0\0\0\xb8\0\0\0\0\0\0\0\x01\0\0\0\x08\0\0\0\
+\x28\0\0\0\0\0\0\0\x01\0\0\0\x08\0\0\0\x88\0\0\0\0\0\0\0\x01\0\0\0\x07\0\0\0\
+\xd0\0\0\0\0\0\0\0\x01\0\0\0\x08\0\0\0\0\x2e\x74\x65\x78\x74\0\x70\x65\x72\x66\
+\x5f\x65\x76\x65\x6e\x74\x73\0\x6d\x61\x70\x73\0\x72\x65\x63\x6f\x72\x64\x73\0\
+\x5f\x76\x65\x72\x73\x69\x6f\x6e\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\x6f\
+\x69\x6e\x74\x2f\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\x78\
+\x69\x74\x5f\x65\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\x65\x78\
+\x69\x74\x5f\x65\x78\x65\x63\x76\x65\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\
+\x6f\x69\x6e\x74\x2f\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\
+\x6e\x74\x65\x72\x5f\x65\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\
+\x65\x6e\x74\x65\x72\x5f\x65\x78\x65\x63\x76\x65\0\x5f\x6c\x69\x63\x65\x6e\x73\
+\x65\0\x65\x78\x65\x63\x73\x6e\x6f\x6f\x70\x5f\x6b\x65\x72\x6e\x2e\x63\0\x2e\
+\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x4c\x42\x42\x31\x5f\
+\x34\0\x4c\x42\x42\x31\x5f\x33\0\x4c\x42\x42\x30\x5f\x33\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbd\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\x40\x04\0\0\0\0\0\0\xe2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\x69\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\
+\0\x80\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x65\0\
+\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf0\x03\0\0\0\0\0\0\x20\0\0\0\0\
+\0\0\0\x0a\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x2d\0\0\0\x01\0\
+\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc0\x01\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x29\0\0\0\x09\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\x10\x04\0\0\0\0\0\0\x30\0\0\0\0\0\0\0\x0a\0\0\0\x05\0\0\
+\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x13\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\xb8\x02\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\xa4\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xe0\x02\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\x21\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe4\x02\0\0\0\
+\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc5\
+\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe8\x02\0\0\0\0\0\0\x08\x01\0\
+\0\0\0\0\0\x01\0\0\0\x05\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
+
+	return 0;
+err:
+	bpf_object__destroy_skeleton(s);
+	return -1;
+}
+
+#endif /* __EXECSNOOP_KERN_SKEL_H__ */

execsnoop-kernel/arm64.md

@@ -0,0 +1,82 @@
+##  Cross Compile
+
+```bash
+docker pull debian:buster
+
+# in container
+dpkg --add-architecture arm64
+apt update
+apt install gcc-8-aarch64-linux-gnu # cross compile toolchain
+apt install libelf-dev:arm64 # target depency library
+...
+```
+
+## Emulation
+
+### Register qemu-user-static
+
+- before register binfmt
+
+```bash
+docker run --rm -t arm64/ubuntu uname -m
+```
+
+- register binfmt
+
+```bash
+# through docker
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+# or through systemd
+pacman -S qemu-user-static binfmt-qemu-static
+systemctl restart systemd-binfmt.service
+# test
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
+```
+
+- after register binfmt
+
+```bash
+docker run --rm -t arm64/ubuntu uname -m
+```
+
+### M1: Docker
+
+```bash
+# start container background
+docker run -dit --name arm64 -v /home/fancy/workspace-xps:/data arm64v8/ubuntu
+
+# enter container
+docker exec -it arm64 bash
+```
+
+### M2: Chroot
+
+download image [ubuntu-base-20.04-base-arm64.tar.gz](http://cdimage.ubuntu.com/ubuntu-base/releases/20.04/release/ubuntu-base-20.04-base-arm64.tar.gz), extract and chroot to it.
+
+```bash
+sudo arch-chroot ubuntu-base-20.04-base-arm64
+```
+
+### Refer
+
+- https://www.stereolabs.com/docs/docker/building-arm-container-on-x86/
+
+- https://github.com/junaruga/fedora-workshop-multiarch/blob/master/slides/Lets-add-Fedora-multiarch-to-CI.pdf
+
+- https://wiki.debian.org/QemuUserEmulation
+
+### Compile
+
+ready some depencies.
+
+```bash
+# maybe repository: https://mirrors.tuna.tsinghua.edu.cn/help/ubuntu/
+# install in container for kernel bpf build
+apt install dialog apt-utils
+apt install build-essential gcc clang llvm
+apt install bison flex bc rsync libssl-dev binutils-dev libreadline-dev libelf-dev
+apt install make cmake
+# for cgproxy
+apt install nlohmann-json3-dev rpm
+```
+

execsnoop-kernel/execsnoop_kern.c

@@ -0,0 +1,97 @@
+#ifdef USE_VMLINUX
+#include "vmlinux.h"
+#else
+#include "linux/sched.h"
+#include <linux/ptrace.h>
+#include <uapi/linux/bpf.h>
+#endif
+#include <linux/version.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+/* /sys/kernel/debug/tracing/events/syscalls/sys_enter_execve/format */
+struct syscalls_enter_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	const char filename_ptr;
+    const char *const * argv;
+    const char *const * envp;
+};
+
+/* /sys/kernel/debug/tracing/events/syscalls/sys_exit_execve/format */
+struct syscalls_exit_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	long ret;
+};
+
+struct bpf_map_def SEC("maps") records = {
+    .type = BPF_MAP_TYPE_HASH,
+    .key_size = sizeof(pid_t),
+    .value_size = sizeof(struct event),
+    .max_entries = 10240,
+};
+struct bpf_map_def SEC("maps") perf_events = {
+    .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
+    .key_size = sizeof(u32),
+    .value_size = sizeof(u32),
+    .max_entries = 128,
+};
+
+SEC("tracepoint/syscalls/sys_enter_execve")
+int syscall_enter_execve(struct syscalls_enter_execve_args *ctx){
+    pid_t pid, tgid; uid_t uid;
+	struct event *event;
+	struct task_struct *task, *task_p;
+
+    u64 id = bpf_get_current_pid_tgid();
+	pid = (pid_t)id;
+	tgid = id >> 32;
+	uid = (u32)bpf_get_current_uid_gid();
+
+	struct event empty_event={};
+	if (bpf_map_update_elem(&records, &pid, &empty_event, BPF_NOEXIST)!=0) return 0;
+	event = bpf_map_lookup_elem(&records, &pid);
+	if (!event) return 0;
+
+	event->pid = pid;
+	event->tgid = tgid;
+	event->uid = uid;
+
+	/* ppid is not reliable here, normal in arch, but become 0 in ubuntu 20.04 */
+	task = (struct task_struct*)bpf_get_current_task();
+	bpf_probe_read(&task_p, sizeof(struct task_struct*),&task->real_parent);
+	bpf_probe_read(&event->ppid,sizeof(pid_t),&task_p->tgid);
+
+    return 0;
+}
+
+SEC("tracepoint/syscalls/sys_exit_execve")
+int syscall_exit_execve(struct syscalls_exit_execve_args *ctx){
+	pid_t pid;
+	struct event *event;
+
+	pid = (pid_t)bpf_get_current_pid_tgid();
+	event = bpf_map_lookup_elem(&records,&pid);
+	if (!event) return 0;
+	if (ctx->ret < 0) goto cleanup;
+
+	/* get comm */
+	bpf_get_current_comm(&event->comm,sizeof(event->comm));
+	bpf_perf_event_output(ctx, &perf_events, BPF_F_CURRENT_CPU, event, sizeof(struct event));
+cleanup:
+	bpf_map_delete_elem(&records, &pid);
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
+u32 _version SEC("version") = LINUX_VERSION_CODE;

execsnoop-kernel/execsnoop_share.cpp

@@ -0,0 +1,96 @@
+#include "execsnoop_share.h"
+
+#include <errno.h>
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include <sys/resource.h>
+
+#if defined(__x86_64__)
+	#include "x86_64/execsnoop_kern_skel.h"
+#elif defined(__aarch64__)
+	#include "aarch64/execsnoop_kern_skel.h"
+#endif
+
+namespace CGPROXY::EXECSNOOP {
+
+#define PERF_BUFFER_PAGES 64
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+function<int(int)> callback = NULL;
+promise<void> status;
+
+static void handle_event(void *ctx, int cpu, void *data, __u32 size) {
+  	auto e = static_cast<event*>(data);
+  	if (callback) callback(e->pid);
+}
+
+void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) {
+	fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
+}
+
+int bump_memlock_rlimit(void) {
+	struct rlimit rlim_new = { RLIM_INFINITY, RLIM_INFINITY };
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
+int execsnoop() {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int err;
+	bool notified=false;
+	
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+	
+	struct execsnoop_kern *obj=execsnoop_kern__open_and_load();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	err = execsnoop_kern__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		return err;
+	}
+
+main_loop:
+	pb_opts.sample_cb = handle_event;
+	pb_opts.lost_cb = handle_lost_events;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	err = libbpf_get_error(pb);
+	if (err) {
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
+	}
+
+	// notify
+  	if (!notified) {status.set_value(); notified=true;}
+
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	perf_buffer__free(pb);
+	/* handle Interrupted system call when sleep */
+	if (err == -EINTR) goto main_loop;
+
+	perror("perf_buffer__poll");
+	kill(0, SIGINT);
+	return err;
+}
+
+void startThread(function<int(int)> c, promise<void> _status) {
+  status = move(_status);
+  callback = c;
+  execsnoop();
+}
+
+}

execsnoop-kernel/execsnoop_share.h

@@ -0,0 +1,20 @@
+#ifndef EXECSNOOP_SHARE_HPP
+#define EXECSNOOP_SHARE_HPP 1
+
+#include <functional>
+#include <future>
+#include <string>
+using namespace std;
+
+namespace CGPROXY::EXECSNOOP {
+
+extern "C" void startThread(function<int(int)> c, promise<void> _status);
+
+#ifdef BUIlD_EXECSNOOP_DL
+// only for dlsym()
+using startThread_t=decltype(startThread);
+startThread_t *_startThread;
+#endif
+
+} // namespace CGPROXY::EXECSNOOP
+#endif

execsnoop-kernel/execsnoop_user.c

@@ -0,0 +1,40 @@
+
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include "bpf_load.h"
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+static void print_bpf_output(void *ctx, int cpu, void *data, __u32 size) {
+	struct event *e=data;
+    printf("comm: %s, pid: %d, tgid: %d, ppid: %d, uid: %d\n",e->comm,e->pid,e->tgid,e->ppid,e->uid);
+}
+
+int main(int argc, char **argv) {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int ret;
+
+	if (load_bpf_file("execsnoop_kern.o")!=0) {
+		printf("%s", bpf_log_buf);
+		return 1;
+	}
+
+	pb_opts.sample_cb = print_bpf_output;
+	pb = perf_buffer__new(map_fd[1], 8, &pb_opts);
+	ret = libbpf_get_error(pb);
+	if (ret) {
+		printf("failed to setup perf_buffer: %d\n", ret);
+		return 1;
+	}
+
+	while ((ret = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return ret;
+}

execsnoop-kernel/execsnoop_user_1.c

@@ -0,0 +1,72 @@
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include <sys/resource.h>
+
+#if defined(__x86_64__)
+	#include "x86_64/execsnoop_kern_skel.h"
+#elif defined(__aarch64__)
+	#include "aarch64/execsnoop_kern_skel.h"
+#endif
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+#define PERF_BUFFER_PAGES   64
+
+static void print_bpf_output(void *ctx, int cpu, void *data, __u32 size) {
+	struct event *e=data;
+    printf("comm: %s, pid: %d, tgid: %d, ppid: %d, uid: %d\n",e->comm,e->pid,e->tgid,e->ppid,e->uid);
+}
+
+int bump_memlock_rlimit(void)
+{
+	struct rlimit rlim_new = {
+		.rlim_cur	= RLIM_INFINITY,
+		.rlim_max	= RLIM_INFINITY,
+	};
+
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
+
+int main(int argc, char **argv) {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int err;
+	
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+	
+	struct execsnoop_kern *obj=execsnoop_kern__open_and_load();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	err = execsnoop_kern__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		return err;
+	}
+
+	pb_opts.sample_cb = print_bpf_output;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	err = libbpf_get_error(pb);
+	if (err) {
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
+	}
+
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return err;
+}

execsnoop-kernel/readme.md

@@ -0,0 +1,198 @@
+
+## Prons and cons
+
+- use stable execve tracepoint, so build once and should work everywhere
+- build in kernel tree or build with VMLINUX
+
+
+
+## Build `execsnoop_kern.o`
+
+### M1: Build in kernel tree
+
+- download kernel source code
+- ready and config kernel tree
+
+```bash
+# kernel config
+#gunzip -c /proc/config.gz > .config
+#make oldconfig && make prepare
+make defconfig && make prepare
+# install headers to ./usr/include
+make headers_install -j8
+# build samples/bpf
+make M=samples/bpf -j8
+# build bpftool
+make tools/bpf -j8
+```
+
+- put or link `execsnoop_kern.c` and `execsnoop_user.c` to *samples/bpf/*
+- edit  *samples/bpf/makefile*
+
+```makefile
+# in samples/bpf/makefile
+tprogs-y += execsnoop
+execsnoop-objs := bpf_load.o execsnoop_user.o $(TRACE_HELPERS)
+always-y += execsnoop_kern.o
+```
+
+- compile again
+
+```
+make M=samples/bpf -j8
+```
+
+- run test
+
+```bash
+cd samples/bpf
+sudo bash -c "ulimit -l unlimited && ./execsnoop"
+```
+
+
+**Detail build command**
+
+using `make V=1 M=samples/bpf | tee -a log.txt` to get and filter following command
+
+- build `execsnoop_kern.o`
+
+
+```bash
+clang -nostdinc \
+	-isystem /usr/lib/gcc/x86_64-pc-linux-gnu/10.1.0/include \
+	-I./arch/x86/include \
+	-I./arch/x86/include/generated  \
+	-I./include \
+	-I./arch/x86/include/uapi \
+	-I./arch/x86/include/generated/uapi \
+	-I./include/uapi \
+	-I./include/generated/uapi \
+	-include ./include/linux/kconfig.h \
+	-I./samples/bpf \
+	-I./tools/testing/selftests/bpf/ \
+	-I./tools/lib/ \
+	-include asm_goto_workaround.h \
+	-D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \
+	-D__TARGET_ARCH_x86 -Wno-compare-distinct-pointer-types \
+	-Wno-gnu-variable-sized-type-not-at-end \
+	-Wno-address-of-packed-member -Wno-tautological-compare \
+	-Wno-unknown-warning-option  \
+	-fno-stack-protector \
+	-O2 -emit-llvm -c samples/bpf/execsnoop_kern.c \
+	-o - | llc -march=bpf -filetype=obj -o samples/bpf/execsnoop_kern.o
+```
+
+
+
+### M2: Build with VMLINUX
+
+- get `vmlinux.h`
+
+```
+bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
+```
+
+- compile
+
+  note `-g` is needed if with BPF CO-RE
+
+```
+clang -O2 -target bpf -DUSE_VMLINUX -c execsnoop_kern.c -o execsnoop_kern.o
+```
+
+## Generate `execsnoop_kern_skel.h`
+
+- generate `execsnoop_kern_skel.h`
+
+```
+bpftool gen skeleton execsnoop_kern.o > execsnoop_kern_skel.h
+```
+
+- build execsnoop
+
+```
+gcc -Wall -O2 execsnoop_user_1.c -o execsnoop -lbpf
+```
+
+
+
+## Multiarch build
+
+- Cross compile, fast, but library link can be mess
+  - aarch64-linux-gnu-gcc
+- Emulation, the easist, but with perfomance cost
+  - qemu-user-static + binfmt-qemu-static + docker/chroot
+- see `arm64.md` to see how to setup
+
+
+
+```bash
+# if cross compile
+export ARCH=arm64
+export CROSS_COMPILE=aarch64-linux-gnu-
+export SYSROOT=/home/fancy/workspace-xps/linux/ArchLinuxARM-aarch64-latest
+export C_INCLUDE_PATH=$SYSROOT/usr/include
+```
+
+-  edit `tools/lib/bpf/makefile` #192 to:
+
+```makefile
+$(OUTPUT)libbpf.so.$(LIBBPF_VERSION): $(BPF_IN_SHARED)
+     $(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS)
+```
+
+- make
+
+```bash
+# clean
+make mrproper
+make clean
+make -C tools clean
+make -C samples/bpf clean
+# make
+make defconfig && make prepare
+make headers_install -j8
+# build samples/bpf
+make M=samples/bpf -j8
+# build bpftool
+make tools/bpf -j8
+```
+
+- detail build `execsnoop_kern.o`
+
+
+```bash
+clang  -nostdinc \
+	-isystem /usr/lib/gcc/aarch64-linux-gnu/9/include \
+	-I./arch/arm64/include -I./arch/arm64/include/generated  \
+	-I./include -I./arch/arm64/include/uapi \
+	-I./arch/arm64/include/generated/uapi \
+	-I./include/uapi \
+	-I./include/generated/uapi \
+	-include ./include/linux/kconfig.h \
+    -I./samples/bpf \
+    -I./tools/testing/selftests/bpf/ \
+    -I./tools/lib/ \
+    -include asm_goto_workaround.h \
+    -D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \
+    -D__TARGET_ARCH_arm64 -Wno-compare-distinct-pointer-types \
+    -Wno-gnu-variable-sized-type-not-at-end \
+    -Wno-address-of-packed-member -Wno-tautological-compare \
+    -Wno-unknown-warning-option  \
+    -fno-stack-protector \
+    -O2 -emit-llvm -c samples/bpf/execsnoop_kern.c \
+    -o -| llc -march=bpf  -filetype=obj -o samples/bpf/execsnoop_kern.o
+```
+
+- generate
+
+```
+bpftool gen skeleton execsnoop_kern.o > aarch64/execsnoop_kern_skel.h
+```
+
+
+
+## Refer
+
+- [A thorough introduction to eBPF](https://lwn.net/Articles/740157/)
+

execsnoop-kernel/x86_64/execsnoop_kern_skel.h

@@ -0,0 +1,225 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+
+/* THIS FILE IS AUTOGENERATED! */
+#ifndef __EXECSNOOP_KERN_SKEL_H__
+#define __EXECSNOOP_KERN_SKEL_H__
+
+#include <stdlib.h>
+#include <bpf/libbpf.h>
+
+struct execsnoop_kern {
+	struct bpf_object_skeleton *skeleton;
+	struct bpf_object *obj;
+	struct {
+		struct bpf_map *perf_events;
+		struct bpf_map *records;
+	} maps;
+	struct {
+		struct bpf_program *syscall_enter_execve;
+		struct bpf_program *syscall_exit_execve;
+	} progs;
+	struct {
+		struct bpf_link *syscall_enter_execve;
+		struct bpf_link *syscall_exit_execve;
+	} links;
+};
+
+static void
+execsnoop_kern__destroy(struct execsnoop_kern *obj)
+{
+	if (!obj)
+		return;
+	if (obj->skeleton)
+		bpf_object__destroy_skeleton(obj->skeleton);
+	free(obj);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj);
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_opts(const struct bpf_object_open_opts *opts)
+{
+	struct execsnoop_kern *obj;
+
+	obj = (typeof(obj))calloc(1, sizeof(*obj));
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__create_skeleton(obj))
+		goto err;
+	if (bpf_object__open_skeleton(obj->skeleton, opts))
+		goto err;
+
+	return obj;
+err:
+	execsnoop_kern__destroy(obj);
+	return NULL;
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open(void)
+{
+	return execsnoop_kern__open_opts(NULL);
+}
+
+static inline int
+execsnoop_kern__load(struct execsnoop_kern *obj)
+{
+	return bpf_object__load_skeleton(obj->skeleton);
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_and_load(void)
+{
+	struct execsnoop_kern *obj;
+
+	obj = execsnoop_kern__open();
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__load(obj)) {
+		execsnoop_kern__destroy(obj);
+		return NULL;
+	}
+	return obj;
+}
+
+static inline int
+execsnoop_kern__attach(struct execsnoop_kern *obj)
+{
+	return bpf_object__attach_skeleton(obj->skeleton);
+}
+
+static inline void
+execsnoop_kern__detach(struct execsnoop_kern *obj)
+{
+	return bpf_object__detach_skeleton(obj->skeleton);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj)
+{
+	struct bpf_object_skeleton *s;
+
+	s = (typeof(s))calloc(1, sizeof(*s));
+	if (!s)
+		return -1;
+	obj->skeleton = s;
+
+	s->sz = sizeof(*s);
+	s->name = "execsnoop_kern";
+	s->obj = &obj->obj;
+
+	/* maps */
+	s->map_cnt = 2;
+	s->map_skel_sz = sizeof(*s->maps);
+	s->maps = (typeof(s->maps))calloc(s->map_cnt, s->map_skel_sz);
+	if (!s->maps)
+		goto err;
+
+	s->maps[0].name = "perf_events";
+	s->maps[0].map = &obj->maps.perf_events;
+
+	s->maps[1].name = "records";
+	s->maps[1].map = &obj->maps.records;
+
+	/* programs */
+	s->prog_cnt = 2;
+	s->prog_skel_sz = sizeof(*s->progs);
+	s->progs = (typeof(s->progs))calloc(s->prog_cnt, s->prog_skel_sz);
+	if (!s->progs)
+		goto err;
+
+	s->progs[0].name = "syscall_enter_execve";
+	s->progs[0].prog = &obj->progs.syscall_enter_execve;
+	s->progs[0].link = &obj->links.syscall_enter_execve;
+
+	s->progs[1].name = "syscall_exit_execve";
+	s->progs[1].prog = &obj->progs.syscall_exit_execve;
+	s->progs[1].link = &obj->links.syscall_exit_execve;
+
+	s->data_sz = 2320;
+	s->data = (void *)"\
+\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\xd0\x05\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0d\0\
+\x01\0\x85\0\0\0\x0e\0\0\0\xbf\x06\0\0\0\0\0\0\x63\x6a\xfc\xff\0\0\0\0\x85\0\0\
+\0\x0f\0\0\0\xbf\x07\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\
+\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\
+\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
+\xd0\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb7\x04\0\0\x01\0\0\0\x85\
+\0\0\0\x02\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x19\0\0\0\0\0\
+\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\0\x15\x08\x12\0\0\0\0\0\x77\x06\0\
+\0\x20\0\0\0\x61\xa1\xfc\xff\0\0\0\0\x63\x78\x1c\0\0\0\0\0\x63\x68\x14\0\0\0\0\
+\0\x63\x18\x10\0\0\0\0\0\x85\0\0\0\x23\0\0\0\x07\0\0\0\xa0\x04\0\0\xbf\xa1\0\0\
+\0\0\0\0\x07\x01\0\0\xf0\xff\xff\xff\xb7\x02\0\0\x08\0\0\0\xbf\x03\0\0\0\0\0\0\
+\x85\0\0\0\x04\0\0\0\x07\x08\0\0\x18\0\0\0\x79\xa3\xf0\xff\0\0\0\0\x07\x03\0\0\
+\x94\x04\0\0\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\
+\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\xbf\x16\0\0\0\0\0\0\x85\0\0\0\x0e\0\0\0\x63\
+\x0a\xfc\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x15\x07\x13\
+\0\0\0\0\0\x79\x61\x10\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\x6d\x12\x0b\0\0\0\0\0\xbf\
+\x71\0\0\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x85\0\0\0\x10\0\0\0\xbf\x61\0\0\0\0\0\0\
+\x18\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\x03\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\
+\0\xbf\x74\0\0\0\0\0\0\xb7\x05\0\0\x20\0\0\0\x85\0\0\0\x19\0\0\0\xbf\xa2\0\0\0\
+\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\
+\0\x03\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x01\0\0\0\x04\0\0\0\x20\0\0\0\
+\0\x28\0\0\0\0\0\0\x04\0\0\0\x04\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\
+\0\x06\x07\x05\0\x10\0\0\0\0\0\0\0\x01\x7a\x52\0\x08\x7c\x0b\x01\x0c\0\0\0\x18\
+\0\0\0\x18\0\0\0\0\0\0\0\0\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\x34\0\0\
+\0\0\0\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\xba\0\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xe9\0\0\0\0\0\x03\0\x70\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe2\0\0\0\0\0\x05\0\
+\xc0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdb\0\0\0\0\0\x05\0\xe8\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\
+\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa3\0\0\0\x11\0\x08\0\0\0\0\0\0\0\0\0\
+\x04\0\0\0\0\0\0\0\x20\0\0\0\x11\0\x09\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\x07\
+\0\0\0\x11\0\x07\0\x14\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x18\0\0\0\x11\0\x07\0\0\
+\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x8e\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\0\x80\x01\
+\0\0\0\0\0\0\x51\0\0\0\x12\0\x05\0\0\0\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\x70\0\0\0\
+\0\0\0\0\x01\0\0\0\x0a\0\0\0\xb8\0\0\0\0\0\0\0\x01\0\0\0\x0a\0\0\0\x28\0\0\0\0\
+\0\0\0\x01\0\0\0\x0a\0\0\0\x88\0\0\0\0\0\0\0\x01\0\0\0\x09\0\0\0\xd0\0\0\0\0\0\
+\0\0\x01\0\0\0\x0a\0\0\0\x1c\0\0\0\0\0\0\0\x01\0\0\0\x05\0\0\0\x38\0\0\0\0\0\0\
+\0\x01\0\0\0\x06\0\0\0\0\x2e\x74\x65\x78\x74\0\x70\x65\x72\x66\x5f\x65\x76\x65\
+\x6e\x74\x73\0\x6d\x61\x70\x73\0\x72\x65\x63\x6f\x72\x64\x73\0\x5f\x76\x65\x72\
+\x73\x69\x6f\x6e\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\x6f\x69\x6e\x74\x2f\
+\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\x78\x69\x74\x5f\x65\
+\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\x65\x78\x69\x74\x5f\x65\
+\x78\x65\x63\x76\x65\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\x6f\x69\x6e\x74\
+\x2f\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\x6e\x74\x65\x72\
+\x5f\x65\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\x65\x6e\x74\x65\
+\x72\x5f\x65\x78\x65\x63\x76\x65\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x72\
+\x65\x6c\x2e\x65\x68\x5f\x66\x72\x61\x6d\x65\0\x65\x78\x65\x63\x73\x6e\x6f\x6f\
+\x70\x5f\x6b\x65\x72\x6e\x2e\x63\0\x2e\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\
+\x6d\x74\x61\x62\0\x4c\x42\x42\x31\x5f\x34\0\x4c\x42\x42\x31\x5f\x33\0\x4c\x42\
+\x42\x30\x5f\x33\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xcb\0\0\0\
+\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe0\x04\0\0\0\0\0\0\xf0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x69\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\x40\0\0\0\0\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x65\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x70\
+\x04\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x0c\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\
+\0\0\0\0\0\0\x2d\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc0\x01\0\0\
+\0\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x29\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x04\0\0\0\0\0\0\x30\0\
+\0\0\0\0\0\0\x0c\0\0\0\x05\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x13\0\0\0\
+\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x02\0\0\0\0\0\0\x28\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa4\0\0\0\x01\0\0\0\x03\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe0\x02\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x21\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\xe4\x02\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\0\0\0\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\xe8\x02\0\0\0\0\0\0\x50\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\xac\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc0\x04\0\0\0\
+\0\0\0\x20\0\0\0\0\0\0\0\x0c\0\0\0\x0a\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\
+\0\xd3\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x38\x03\0\0\0\0\0\0\x38\
+\x01\0\0\0\0\0\0\x01\0\0\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
+
+	return 0;
+err:
+	bpf_object__destroy_skeleton(s);
+	return -1;
+}
+
+#endif /* __EXECSNOOP_KERN_SKEL_H__ */

execsnoop-libbpf/CMakeLists.txt

@@ -0,0 +1,21 @@
+find_library(LIBBPF bpf REQUIRED)
+
+# generate execsnoop.skel.h
+if (EXISTS /sys/kernel/btf/vmlinux)
+add_custom_command(OUTPUT execsnoop.skel.h
+    COMMAND bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
+    COMMAND clang -O2 -g -target bpf -c execsnoop.bpf.c -o execsnoop.bpf.o
+    COMMAND bpftool gen skeleton execsnoop.bpf.o > execsnoop.skel.h
+    DEPENDS execsnoop.bpf.c
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+endif()
+
+if (build_execsnoop_dl)
+    add_library(execsnoop MODULE execsnoop_share.cpp execsnoop.skel.h)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF})
+    install(TARGETS execsnoop DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cgproxy/)
+else()
+    add_library(execsnoop execsnoop_share.cpp execsnoop.skel.h)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF})
+endif()

execsnoop-libbpf/execsnoop.bpf.c

@@ -0,0 +1,92 @@
+#include "vmlinux.h"
+#include <linux/version.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+/* /sys/kernel/debug/tracing/events/syscalls/sys_enter_execve/format */
+struct syscalls_enter_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	const char filename_ptr;
+    const char *const * argv;
+    const char *const * envp;
+};
+
+/* /sys/kernel/debug/tracing/events/syscalls/sys_exit_execve/format */
+struct syscalls_exit_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	long ret;
+};
+
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 10240);
+    __type(key, pid_t);
+    __type(value, struct event);
+} records SEC(".maps");
+
+struct {
+    __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
+    __uint(max_entries, 128);
+    __type(key, u32);
+    __type(value, u32);
+} perf_events SEC(".maps");
+
+SEC("tracepoint/syscalls/sys_enter_execve")
+int syscall_enter_execve(struct syscalls_enter_execve_args *ctx){
+    pid_t pid, tgid; uid_t uid;
+	struct event *event;
+	struct task_struct *task, *task_p;
+
+    u64 id = bpf_get_current_pid_tgid();
+	pid = (pid_t)id;
+	tgid = id >> 32;
+	uid = (u32)bpf_get_current_uid_gid();
+
+	struct event empty_event={};
+	if (bpf_map_update_elem(&records, &pid, &empty_event, BPF_NOEXIST)!=0) return 0;
+	event = bpf_map_lookup_elem(&records, &pid);
+	if (!event) return 0;
+
+	event->pid = pid;
+	event->tgid = tgid;
+	event->uid = uid;
+
+	/* ppid is not reliable here, normal in arch, but become 0 in ubuntu 20.04 */
+	task = (struct task_struct*)bpf_get_current_task();
+	bpf_probe_read(&task_p, sizeof(struct task_struct*),&task->real_parent);
+	bpf_probe_read(&event->ppid,sizeof(pid_t),&task_p->tgid);
+
+    return 0;
+}
+
+SEC("tracepoint/syscalls/sys_exit_execve")
+int syscall_exit_execve(struct syscalls_exit_execve_args *ctx){
+	pid_t pid;
+	struct event *event;
+
+	pid = (pid_t)bpf_get_current_pid_tgid();
+	event = bpf_map_lookup_elem(&records,&pid);
+	if (!event) return 0;
+	if (ctx->ret < 0) goto cleanup;
+
+	/* get comm */
+	bpf_get_current_comm(&event->comm,sizeof(event->comm));
+	bpf_perf_event_output(ctx, &perf_events, BPF_F_CURRENT_CPU, event, sizeof(struct event));
+cleanup:
+	bpf_map_delete_elem(&records, &pid);
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
+u32 _version SEC("version") = LINUX_VERSION_CODE;

execsnoop-libbpf/execsnoop.c

@@ -0,0 +1,71 @@
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include <sys/resource.h>
+#include "execsnoop.skel.h"
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+#define PERF_BUFFER_PAGES   64
+
+static void handle_event(void *ctx, int cpu, void *data, __u32 size) {
+  	auto e = static_cast<event*>(data);
+    printf("comm: %s, pid: %d, tgid: %d, ppid: %d, uid: %d\n",e->comm,e->pid,e->tgid,e->ppid,e->uid);
+}
+
+void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) {
+	fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
+}
+
+int bump_memlock_rlimit(void) {
+	struct rlimit rlim_new = { RLIM_INFINITY, RLIM_INFINITY };
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
+int main(int argc, char **argv) {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int err;
+	
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+	
+	struct execsnoop_bpf *obj=execsnoop_bpf__open_and_load();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	err = execsnoop_bpf__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		return err;
+	}
+
+	#if LIBBPF_MAJOR_VERSION == 0
+	pb_opts.sample_cb = handle_event;
+	pb_opts.lost_cb = handle_lost_events;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	#else
+	pb_opts.sz = sizeof(pb_opts);
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, handle_event, handle_lost_events, nullptr, &pb_opts);
+	#endif
+	err = libbpf_get_error(pb);
+	if (err) {
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
+	}
+
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return err;
+}

execsnoop-libbpf/execsnoop_share.cpp

@@ -0,0 +1,96 @@
+#include "execsnoop_share.h"
+
+#include <errno.h>
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include <sys/resource.h>
+#include "execsnoop.skel.h"
+
+namespace CGPROXY::EXECSNOOP {
+
+#define PERF_BUFFER_PAGES 64
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+function<int(int)> callback = NULL;
+promise<void> status;
+
+static void handle_event(void *ctx, int cpu, void *data, __u32 size) {
+  	auto e = static_cast<event*>(data);
+  	if (callback) callback(e->pid);
+}
+
+void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) {
+	fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
+}
+
+int bump_memlock_rlimit(void) {
+	struct rlimit rlim_new = { RLIM_INFINITY, RLIM_INFINITY };
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
+int execsnoop() {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int err;
+	bool notified=false;
+	
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+	
+	struct execsnoop_bpf *obj=execsnoop_bpf__open_and_load();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	err = execsnoop_bpf__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		return err;
+	}
+
+main_loop:
+	#if LIBBPF_MAJOR_VERSION == 0
+	pb_opts.sample_cb = handle_event;
+	pb_opts.lost_cb = handle_lost_events;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	#else
+	pb_opts.sz = sizeof(pb_opts);
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, handle_event, handle_lost_events, nullptr, &pb_opts);
+	#endif
+	err = libbpf_get_error(pb);
+	if (err) {
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
+	}
+
+	// notify
+  	if (!notified) {status.set_value(); notified=true;}
+
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	perf_buffer__free(pb);
+	/* handle Interrupted system call when sleep */
+	if (err == -EINTR) goto main_loop;
+
+	perror("perf_buffer__poll");
+	kill(0, SIGINT);
+	return err;
+}
+
+void startThread(function<int(int)> c, promise<void> _status) {
+  status = move(_status);
+  callback = c;
+  execsnoop();
+}
+
+}

execsnoop-libbpf/execsnoop_share.h

@@ -0,0 +1,20 @@
+#ifndef EXECSNOOP_SHARE_HPP
+#define EXECSNOOP_SHARE_HPP 1
+
+#include <functional>
+#include <future>
+#include <string>
+using namespace std;
+
+namespace CGPROXY::EXECSNOOP {
+
+extern "C" void startThread(function<int(int)> c, promise<void> _status);
+
+#ifdef BUIlD_EXECSNOOP_DL
+// only for dlsym()
+using startThread_t=decltype(startThread);
+startThread_t *_startThread;
+#endif
+
+} // namespace CGPROXY::EXECSNOOP
+#endif

execsnoop-libbpf/readme.md

@@ -0,0 +1,18 @@
+generate `vmlinux.h`
+
+```shell
+bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
+```
+
+compiled into BPF ELF file
+
+```shell
+clang -O2 -g -target bpf -c execsnoop.bpf.c -o execsnoop.bpf.o
+```
+
+generate BPF skeleton .skel.h
+
+```shell
+bpftool gen skeleton execsnoop.bpf.o > execsnoop.skel.h
+```
+

man/cgnoproxy.1

@@ -4,7 +4,9 @@
 cgnoproxy \- Run program without proxy
 .SH SYNOPSIS
 cgnoproxy --help
+.br
 cgnoproxy [--debug] <CMD>
+.br
 cgnoproxy [--debug] --pid <PID>
 .SH ALIAS
 cgnoproxy = cgproxy --noproxy

man/cgproxy.1

@@ -4,7 +4,9 @@
 cgproxy \- Run program with proxy
 .SH SYNOPSIS
 cgproxy --help
+.br
 cgproxy [--debug] <CMD>
+.br
 cgproxy [--debug] --pid <PID>
 .SH DESCRIPTION
 cgproxy send current running process pid or specified pid to cgproxyd through unix socket, then pid is attached to proxied cgroup 

man/cgproxyd.1

@@ -8,14 +8,14 @@ cgproxyd [--help] [--debug] [--execsnoop]
 cgproxyd = cgproxy --daemon
 .SH OPTIONS
 .B  --execsnoop
-enable execsnoop to support program level proxy, need python-bcc installed to actually work
+enable execsnoop to support program level proxy, need bcc installed to actually work
 .SH CONFIGURATION
 .I /etc/cgproxy/config.json
 .br
 .B port 
 tproxy listenning port
 .br
-program level proxy controll, need `python-bcc` installed to work:
+program level proxy controll, only work when execsnoop enabled:
 .br
 .RS
 .B program_proxy

pack/CMakeLists.txt

@@ -4,10 +4,13 @@ set(CPACK_PACKAGE_NAME "cgproxy")
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "cgproxy will transparent proxy anything running in specific cgroup.It aslo supports global transparent proxy and gateway proxy")
 
 ## deb pack
+execute_process(COMMAND dpkg --print-architecture 
+    OUTPUT_VARIABLE DEBIAN_ARCH 
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+set(CPACK_DEBIAN_FILE_NAME ${CPACK_PACKAGE_NAME}_${CMAKE_PROJECT_VERSION}_${DEBIAN_ARCH}.deb)
 set(CPACK_DEBIAN_PACKAGE_NAME "cgproxy")
-set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "x86_64")
+# set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "amd64")
 set(CPACK_DEBIAN_PACKAGE_DEPENDS "systemd")
-set(CPACK_DEBIAN_PACKAGE_SUGGESTS "python-bcc")
 set(CPACK_DEBIAN_PACKAGE_SECTION "network")
 set(CPACK_DEBIAN_PACKAGE_PRIORITY "Optional")
 set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/springzfx/cgproxy")
@@ -15,9 +18,12 @@ set(CPACK_DEBIAN_PACKAGE_MAINTAINER "springzfx@gmail.com")
 set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/postinst;${CMAKE_CURRENT_SOURCE_DIR}/prerm")
 
 ## rpm pack
-set(CPACK_RPM_PACKAGE_ARCHITECTURE, "x86_64")
+execute_process(COMMAND uname -m 
+        OUTPUT_VARIABLE RPM_ARCH 
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+set(CPACK_RPM_FILE_NAME ${CPACK_PACKAGE_NAME}_${CMAKE_PROJECT_VERSION}_${RPM_ARCH}.rpm)
+# set(CPACK_RPM_PACKAGE_ARCHITECTURE, "x86_64")
 set(CPACK_RPM_PACKAGE_REQUIRES "systemd")
-set(CPACK_RPM_PACKAGE_SUGGESTS "python-bcc")
 set(CPACK_RPM_PACKAGE_GROUP "network")
 set(CPACK_RPM_PACKAGE_URL "https://github.com/springzfx/cgproxy")
 set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/postinst")

readme.md

@@ -1,6 +1,6 @@
 
 
-# Transparent Proxy powered with cgroup v2
+# Transparent Proxy powered by cgroup v2
 
 
 
@@ -16,11 +16,11 @@ Main feature:
 ## Contents
 
 <!--ts-->
-
-   * [Transparent Proxy with cgroup v2](#transparent-proxy-with-cgroup-v2)
+   * [Transparent Proxy powered by cgroup v2](#transparent-proxy-powered-by-cgroup-v2)
       * [Introduction](#introduction)
+      * [Contents](#contents)
       * [Prerequest](#prerequest)
-      * [How to install](#how-to-install)
+      * [How to build and install](#how-to-build-and-install)
       * [Default usage](#default-usage)
       * [Configuration](#configuration)
       * [Global transparent proxy](#global-transparent-proxy)
@@ -29,8 +29,9 @@ Main feature:
       * [NOTES](#notes)
       * [TIPS](#tips)
       * [Licences](#licences)
+      * [Known Issues](#known-issues)
 
-<!-- Added by: fancy, at: Sat 16 May 2020 03:12:07 PM HKT -->
+<!-- Added by: fancy, at: Sat 04 Jul 2020 03:52:07 PM CST -->
 
 <!--te-->
 
@@ -52,16 +53,35 @@ Main feature:
 
   ubuntu 16.04, debian 9, fedora 27 and later are desired
 
-## How to install
+## How to build and install
+
+### distro install
+
+- For debian and redhat series, download from [Release page](https://github.com/springzfx/cgproxy/releases)
+
+- For archlinux series, already in archlinuxcn repo, or see [archlinux AUR](https://aur.archlinux.org/packages/?K=cgproxy)
+
+- **Tested on  archlinux, fedora 32, ubuntu 18.04, ubuntu 20.04,  deepin 15.11, deepin v20 beta**
+
+### build
+
+- before build, install depencies: clang, nlohmann-json, libbpf, bpf(bpftool)
+- then cmake standard build
 
 ```bash
-mkdir build && cd build && cmake .. && make && make install
+# ready build dir
+mkdir build
+cd build
+# generate
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=/usr \
+      -Dbuild_execsnoop_dl=ON \
+      -Dbuild_static=OFF \
+      ..
+# compile
+make
 ```
 
-- It is alreay in [archlinux AUR](https://aur.archlinux.org/packages/?K=cgproxy). 
-
-- DEB and RPM are packaged in [release page](https://github.com/springzfx/cgproxy/releases).
-
 ## Default usage
 
 - First enable and start service
@@ -95,21 +115,24 @@ Config file: **/etc/cgproxy/config.json**
 {
     "port": 12345,
     "program_noproxy": ["v2ray", "qv2ray"],
-    "program_proxy": [ ],
+    "program_proxy": [],
     "cgroup_noproxy": ["/system.slice/v2ray.service"],
-    "cgroup_proxy": [ ],
+    "cgroup_proxy": [],
     "enable_gateway": false,
     "enable_dns": true,
     "enable_udp": true,
     "enable_tcp": true,
     "enable_ipv4": true,
-    "enable_ipv6": true
+    "enable_ipv6": true,
+    "table": 10007,
+    "fwmark": 39283
 }
+
 ```
 
 - **port** tproxy listenning port
 
-- program level proxy control, need `python-bcc` installed to work
+- program level proxy control, need execsnoop enabled:
 
   - **program_proxy**  program need to be proxied
   - **program_noproxy** program that won't be proxied
@@ -131,11 +154,14 @@ Config file: **/etc/cgproxy/config.json**
 
 - **enable_ipv6**
 
+- **table**, **fwmark** you can specify iptables and route table related parameter in case conflict.
+
 - options priority
 
   ```
   program_noproxy > program_proxy > cgroup_noproxy > cgroup_proxy
-  enable_ipv6 > enable_ipv4 > enable_tcp > enable_udp > enable_dns
+  enable_ipv6 = enable_ipv4 > enable_dns > enable_tcp = enable_udp
+  command cgproxy and cgnoproxy always have highest priority
   ```
 
 **Note**: cgroup in configuration need to be exist, otherwise ignored
@@ -181,27 +207,17 @@ sudo systemctl restart cgproxy.service
   cgnoproxy [--debug] --pid <PID>
   ```
   
-- `cgattach` attach specific process pid to specific cgroup which will create if not exist , cgroup can be only one level down exist cgroup, otherwise created fail. 
-
-  You need to set `set(build_tools ON)` in *CmakeLists.txt* to build this.
-  
-  ```bash
-  cgattch <pid> <cgroup>
-  # example
-  cgattch 9999 /proxy.slice
-  ```
-  
 - For more detail command usage, see `man cgproxyd`  `man cgproxy`  `man cgnoproxy` 
 
 ## NOTES
 
-- v2ray TPROXY need root or special permission, use [service](https://github.com/springzfx/cgproxy/blob/v3.x/v2ray_config/v2ray.service) or
+- v2ray TPROXY need root or special permission, use [service](/v2ray_config/v2ray.service) or
   
   ```bash
   sudo setcap "cap_net_admin,cap_net_bind_service=ep" /usr/lib/v2ray/v2ray
   ```
 
-- Why not outbound mark solution, because in v2ray [when `"localhost"` is used, out-going DNS traffic is not controlled by V2Ray](https://www.v2fly.org/en/configuration/dns.html), so no mark at all, that's pity.
+- Why not outbound mark solution, because in v2ray [when `"localhost"` is used, out-going DNS traffic is not controlled by V2Ray](https://www.v2fly.org/config/dns.html#dnsobject), so no mark at all, that's pity.
 
 ## TIPS
 
@@ -211,8 +227,22 @@ sudo systemctl restart cgproxy.service
 - Offer you qv2ray config example
   
 
-![Qv2ray config example](https://i.loli.net/2020/04/28/bdQBzUD37FOgfvt.png)
+![Qv2ray config example](https://i.loli.net/2020/08/17/P6y5SfLoUwGjaxM.png)
 
 ## Licences
 
 cgproxy is licenced under [![License: GPL v3](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/gpl-2.0) 
+
+## Known Issues
+
+- docker breaks cgroup v2 path match, add kernel parameter `cgroup_no_v1=net_cls,net_prio` to resolve, see [issue #3](https://github.com/springzfx/cgproxy/issues/3) for detail
+
+- docker load `br_netfilter` module due to [hairpin nat](https://wiki.mikrotik.com/wiki/Hairpin_NAT),  which is not a big deal,  see [commit](https://github.com/moby/moby/pull/13162). 
+
+  It enables data link layer packet to go through iptables and only once. However TPROXY do not accept this kind of packets. So to get it working, set following parameter to disable this behavior or unload br_netfilter module manualy. see [issue #10](https://github.com/springzfx/cgproxy/issues/10) for detail.
+
+  ```
+  sudo sysctl -w net.bridge.bridge-nf-call-iptables=0
+  sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=0
+  sudo sysctl -w net.bridge.bridge-nf-call-arptables = 0
+  ```

src/CMakeLists.txt

@@ -1,19 +1,26 @@
-find_package(Threads REQUIRED)
 find_package(nlohmann_json REQUIRED)
 include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${PROJECT_SOURCE_DIR}/execsnoop-libbpf/)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-if (with_execsnoop)
-add_library(execsnoop MODULE execsnoop.cpp common.cpp)
-target_link_libraries(execsnoop bcc)
-install(TARGETS execsnoop DESTINATION /usr/lib/cgproxy/ PERMISSIONS ${basic_permission})
+configure_file(common.cmake.h ${CMAKE_CURRENT_SOURCE_DIR}/common.h)
+
+if (build_execsnoop_dl)
+    add_definitions(-DBUIlD_EXECSNOOP_DL)
+    set(DL_LIB "-ldl")
+    set(EXECSNOOP_LIB "")
+else()
+    set(EXECSNOOP_LIB "execsnoop")
 endif()
 
-add_executable(main main.cpp
-                    common.cpp config.cpp cgroup_attach.cpp
-                    socket_client.cpp socket_server.cpp)
-target_link_libraries(main PRIVATE nlohmann_json::nlohmann_json Threads::Threads dl)
-set_target_properties(main PROPERTIES LINKER_LANGUAGE CXX)
+add_executable(main main.cpp common.cpp config.cpp cgroup_attach.cpp socket_client.cpp socket_server.cpp)
+target_link_libraries(main PRIVATE nlohmann_json::nlohmann_json ${DL_LIB} ${EXECSNOOP_LIB})
 set_target_properties(main PROPERTIES OUTPUT_NAME cgproxy)
+install(TARGETS main DESTINATION ${CMAKE_INSTALL_FULL_BINDIR})
 
-install(TARGETS main DESTINATION /usr/bin PERMISSIONS ${basic_permission})
+if (build_static)
+    target_link_libraries(main PRIVATE -static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive)
+else()
+    target_link_libraries(main PRIVATE -lpthread)
+endif()

src/cgproxy.hpp

@@ -5,8 +5,8 @@
 #include <nlohmann/json.hpp>
 #include <unistd.h>
 using json = nlohmann::json;
-using namespace CGPROXY;
-using namespace CGPROXY::CONFIG;
+using namespace ::CGPROXY;
+using namespace ::CGPROXY::CONFIG;
 
 namespace CGPROXY::CGPROXY {
 
@@ -79,6 +79,7 @@ int main(int argc, char *argv[]) {
   if (attach_pid) return 0;
 
   string s = join2str(argc - shift, argv + shift, ' ');
+  debug("executing: %s", s.c_str());
   return system(s.c_str());
 }
-} // namespace CGPROXY::CGPROXY
+} // namespace CGPROXY::CGPROXY

src/cgproxyd.hpp

@@ -4,7 +4,7 @@
 #include "cgroup_attach.h"
 #include "common.h"
 #include "config.h"
-#include "execsnoop.h"
+#include "execsnoop_share.h"
 #include "socket_server.h"
 #include <algorithm>
 #include <csignal>
@@ -13,8 +13,8 @@
 #include <exception>
 #include <fstream>
 #include <functional>
+#include <future>
 #include <nlohmann/json.hpp>
-#include <pthread.h>
 #include <sched.h>
 #include <sys/file.h>
 #include <unistd.h>
@@ -26,9 +26,8 @@ using namespace ::CGPROXY::CONFIG;
 using namespace ::CGPROXY::CGROUP;
 // using namespace ::CGPROXY::EXECSNOOP;
 
+#ifdef BUIlD_EXECSNOOP_DL
 namespace CGPROXY::EXECSNOOP {
-typedef void *(*startThread_t)(void *arg);
-startThread_t _startThread;
 bool loadExecsnoopLib() {
   try {
     info("loading %s", LIBEXECSNOOP_SO);
@@ -37,16 +36,20 @@ bool loadExecsnoopLib() {
       error("dlopen %s failed: %s", LIBEXECSNOOP_SO, dlerror());
       return false;
     }
-    _startThread = reinterpret_cast<startThread_t>(dlsym(handle_dl, "_startThread"));
+    _startThread = reinterpret_cast<startThread_t *>(dlsym(handle_dl, "startThread"));
     if (_startThread == NULL) {
-      error("dlsym startThread failed: %s", dlerror());
+      error("dlsym startThread func failed: %s", dlerror());
       return false;
     }
-    info("dlsym startThread success");
+    info("dlsym startThread func success");
     return true;
-  } catch (exception &e) { return false; }
+  } catch (exception &e) {
+    debug("exception: %s", e.what());
+    return false;
+  }
 }
 } // namespace CGPROXY::EXECSNOOP
+#endif
 
 namespace CGPROXY::CGPROXYD {
 
@@ -55,11 +58,8 @@ bool enable_socketserver = true;
 bool enable_execsnoop = false;
 
 class cgproxyd {
-  SOCKET::thread_arg socketserver_thread_arg;
-  pthread_t socket_thread_id = THREAD_UNDEF;
-
-  EXECSNOOP::thread_arg execsnoop_thread_arg;
-  pthread_t execsnoop_thread_id = THREAD_UNDEF;
+  thread socketserver_thread;
+  thread execsnoop_thread;
 
   Config config;
 
@@ -81,28 +81,63 @@ class cgproxyd {
   }
 
   int handle_pid(int pid) {
-    auto path = realpath(to_str("/proc/", pid, "/exe").c_str(), NULL);
+    unique_ptr<char[], decltype(&free)> path(
+        realpath(to_str("/proc/", pid, "/exe").c_str(), NULL), &free);
     if (path == NULL) {
-      debug("pid %d live life too short", pid);
+      debug("execsnoop: pid %d live life too short", pid);
       return 0;
     }
-    debug("execsnoop: %d %s", pid, path);
+    debug("execsnoop: %d %s", pid, path.get());
 
     vector<string> v;
 
     v = config.program_noproxy;
-    if (find(v.begin(), v.end(), path) != v.end()) {
-      info("execsnoop noproxy: %d %s", pid, path);
-      free(path);
-      return attach(pid, config.cgroup_noproxy_preserved);
+    if (find(v.begin(), v.end(), path.get()) != v.end()) {
+      string cg = getCgroup(pid);
+      if (cg.empty()) {
+        debug("execsnoop: cgroup get failed, ignore: %d %s", pid, path.get());
+        return 0;
+      }
+      if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
+          belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
+        info("execsnoop: already in preserverd cgroup, leave alone: %d %s", pid,
+             path.get());
+        return 0;
+      }
+      if (!belongToCgroup(cg, config.cgroup_noproxy)) {
+        int res = attach(pid, config.cgroup_noproxy_preserved);
+        if (res == 0) {
+          info("execsnoop; noproxy: %d %s", pid, path.get());
+        } else {
+          info("execsnoop; noproxy failed: %d %s", pid, path.get());
+        }
+        return res;
+      }
     }
+
     v = config.program_proxy;
-    if (find(v.begin(), v.end(), path) != v.end()) {
-      info("execsnoop proxied: %d %s", pid, path);
-      free(path);
-      return attach(pid, config.cgroup_proxy_preserved);
+    if (find(v.begin(), v.end(), path.get()) != v.end()) {
+      string cg = getCgroup(pid);
+      if (cg.empty()) {
+        debug("execsnoop: cgroup get failed, ignore: %d %s", pid, path.get());
+        return 0;
+      }
+      if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
+          belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
+        info("execsnoop: already in preserverd cgroup, leave alone: %d %s", pid,
+             path.get());
+        return 0;
+      }
+      if (!belongToCgroup(cg, config.cgroup_proxy)) {
+        int res = attach(pid, config.cgroup_proxy_preserved);
+        if (res == 0) {
+          info("execsnoop: proxied: %d %s", pid, path.get());
+        } else {
+          info("execsnoop: proxied failed: %d %s", pid, path.get());
+        }
+        return res;
+      }
     }
-    free(path);
     return 0;
   }
 
@@ -146,78 +181,120 @@ class cgproxyd {
       return MSG_ERROR;
     }
 
-    int type, status;
-    int pid, cgroup_target;
+    int type, status, pid;
     try {
       type = j.at("type").get<int>();
       switch (type) {
       case MSG_TYPE_CONFIG_JSON:
         status = config.loadFromJsonStr(j.at("data").dump());
+        info("process received config json msg");
         if (status == SUCCESS) status = applyConfig();
         return status;
         break;
       case MSG_TYPE_CONFIG_PATH:
         status = config.loadFromFile(j.at("data").get<string>());
+        info("process received config path msg");
         if (status == SUCCESS) status = applyConfig();
         return status;
         break;
       case MSG_TYPE_PROXY_PID:
         pid = j.at("data").get<int>();
+        info("process proxy pid msg: %d", pid);
         status = attach(pid, config.cgroup_proxy_preserved);
         return status;
         break;
       case MSG_TYPE_NOPROXY_PID:
         pid = j.at("data").get<int>();
+        info("process noproxy pid msg: %d", pid);
         status = attach(pid, config.cgroup_noproxy_preserved);
         return status;
         break;
-      default: return MSG_ERROR; break;
+      default:
+        error("unknown msg");
+        return MSG_ERROR;
+        break;
       };
     } catch (out_of_range &e) { return MSG_ERROR; } catch (exception &e) {
       return ERROR;
     }
   }
 
-  pthread_t startSocketListeningThread() {
-    socketserver_thread_arg.handle_msg = &handle_msg_static;
-    pthread_t thread_id;
-    int status =
-        pthread_create(&thread_id, NULL, &SOCKET::startThread, &socketserver_thread_arg);
-    if (status != 0) {
-      error("socket thread create failed");
-      return THREAD_UNDEF;
+  void startSocketListeningThread() {
+    promise<void> status;
+    future<void> status_f = status.get_future();
+    thread th(SOCKET::startThread, handle_msg_static, move(status));
+    socketserver_thread = move(th);
+
+    future_status fstatus = status_f.wait_for(chrono::seconds(THREAD_TIMEOUT));
+    if (fstatus == std::future_status::ready) {
+      info("socketserver thread started");
+    } else {
+      error("socketserver thread timeout, maybe failed");
     }
-    return thread_id;
   }
 
-  pthread_t startExecsnoopThread() {
+  void startExecsnoopThread() {
+    #ifdef BUIlD_EXECSNOOP_DL
     if (!EXECSNOOP::loadExecsnoopLib() || EXECSNOOP::_startThread == NULL) {
-      error("execsnoop start failed, maybe bcc not installed");
-      return THREAD_UNDEF;
+      error("execsnoop not ready to start, maybe missing libbpf");
+      return;
     }
+    #endif
 
-    execsnoop_thread_arg.handle_pid = &handle_pid_static;
-    pthread_t thread_id;
-    int status =
-        pthread_create(&thread_id, NULL, EXECSNOOP::_startThread, &execsnoop_thread_arg);
-    if (status != 0) {
-      error("execsnoop thread create failed");
-      return THREAD_UNDEF;
+    promise<void> status;
+    future<void> status_f = status.get_future();
+    #ifdef BUIlD_EXECSNOOP_DL
+    thread th(EXECSNOOP::_startThread, handle_pid_static, move(status));
+    #else
+    thread th(EXECSNOOP::startThread, handle_pid_static, move(status));
+    #endif
+
+    execsnoop_thread = move(th);
+
+    future_status fstatus = status_f.wait_for(chrono::seconds(THREAD_TIMEOUT));
+    if (fstatus == std::future_status::ready) {
+      info("execsnoop thread started");
+      processRunningProgram();
+    } else {
+      error("execsnoop thread timeout, maybe failed");
     }
-    return thread_id;
   }
 
   void processRunningProgram() {
-    debug("process running program") for (auto &path :
-                                          config.program_noproxy) for (auto &pid :
-                                                                       bash_pidof(path)) {
-      int status = attach(pid, config.cgroup_noproxy_preserved);
-      if (status == 0) info("noproxy running process %d %s", pid, path.c_str());
-    }
+    debug("process running program");
+    for (auto &path : config.program_noproxy)
+      for (auto &pid : bash_pidof(path)) {
+        string cg = getCgroup(pid);
+        if (cg.empty()) {
+          debug("cgroup get failed, ignore: %d %s", pid, path.c_str());
+          continue;
+        }
+        if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
+            belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
+          info("already in preserverd cgroup, leave alone: %d %s", pid, path.c_str());
+          continue;
+        }
+        if (!belongToCgroup(cg, config.cgroup_noproxy)) {
+          int status = attach(pid, config.cgroup_noproxy_preserved);
+          if (status == 0) info("noproxy running process %d %s", pid, path.c_str());
+        }
+      }
     for (auto &path : config.program_proxy)
       for (auto &pid : bash_pidof(path)) {
-        int status = attach(pid, config.cgroup_proxy_preserved);
-        if (status == 0) info("proxied running process %d %s", pid, path.c_str());
+        string cg = getCgroup(pid);
+        if (cg.empty()) {
+          debug("cgroup get failed, ignore: %d %s", pid, path.c_str());
+          continue;
+        }
+        if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
+            belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
+          info("already in preserverd cgroup, leave alone: %d %s", pid, path.c_str());
+          continue;
+        }
+        if (!belongToCgroup(cg, config.cgroup_proxy)) {
+          int status = attach(pid, config.cgroup_proxy_preserved);
+          if (status == 0) info("proxied running process %d %s", pid, path.c_str());
+        }
       }
   }
 
@@ -232,24 +309,21 @@ public:
 
     assignStaticInstance();
 
-    config.loadFromFile(DEFAULT_CONFIG_FILE);
+    if (config.loadFromFile(DEFAULT_CONFIG_FILE)!=SUCCESS) {
+      error("load config file failed");
+      return -1;
+    }
     applyConfig();
-    processRunningProgram();
 
-    if (enable_socketserver) {
-      socket_thread_id = startSocketListeningThread();
-      if (socket_thread_id > 0) info("socket server listening thread started");
-    }
-    if (enable_execsnoop) {
-      execsnoop_thread_id = startExecsnoopThread();
-      if (execsnoop_thread_id > 0) info("execsnoop thread started");
-    }
-    cout << flush;
+    if (enable_socketserver) startSocketListeningThread();
+    if (enable_execsnoop) startExecsnoopThread();
+
+    if (socketserver_thread.joinable()) socketserver_thread.join();
+    if (execsnoop_thread.joinable()) execsnoop_thread.join();
 
-    pthread_join(socket_thread_id, NULL);
-    pthread_join(execsnoop_thread_id, NULL);
     return 0;
   }
+
   int applyConfig() {
     system(TPROXY_IPTABLS_CLEAN);
     config.print_summary();
@@ -286,7 +360,6 @@ void processArgs(const int argc, char *argv[]) {
 }
 
 int main(int argc, char *argv[]) {
-  setbuf(stdout, NULL);
   processArgs(argc, argv);
   if (print_help) {
     print_usage();
@@ -302,4 +375,4 @@ int main(int argc, char *argv[]) {
   return d.start();
 }
 } // namespace CGPROXY::CGPROXYD
-#endif
+#endif

src/cgroup_attach.cpp

@@ -9,29 +9,13 @@
 #include <string>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <thread>
 #include <unistd.h>
 
 namespace CGPROXY::CGROUP {
 
-string cgroup2_mount_point = get_cgroup2_mount_point();
+string cgroup2_mount_point = CGROUP2_MOUNT_POINT;
 
-bool exist(string path) {
-  struct stat st;
-  if (stat(path.c_str(), &st) != -1) { return S_ISDIR(st.st_mode); }
-  return false;
-}
-
-string get_cgroup2_mount_point() {
-  stringstream buffer;
-  FILE *fp = popen("findmnt -t cgroup2 -n -o TARGET", "r");
-  if (!fp) return "";
-  char buf[64];
-  while (fgets(buf, 64, fp) != NULL) { buffer << buf; }
-  pclose(fp);
-  string s = buffer.str();
-  s.pop_back(); // remove newline character
-  return s;
-}
 
 bool validate(string pid, string cgroup) {
   bool pid_v = validPid(pid);
@@ -50,14 +34,13 @@ int attach(const string pid, const string cgroup_target) {
 
   debug("attaching %s to %s", pid.c_str(), cgroup_target.c_str());
 
-  int status;
   if (!validate(pid, cgroup_target)) return_error;
   if (cgroup2_mount_point.empty()) return_error;
   string cgroup_target_path = cgroup2_mount_point + cgroup_target;
   string cgroup_target_procs = cgroup_target_path + "/cgroup.procs";
 
   // check if exist, we will create it if not exist
-  if (!exist(cgroup_target_path)) {
+  if (!dirExist(cgroup_target_path)) {
     if (mkdir(cgroup_target_path.c_str(),
               S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH) == 0) {
       debug("created cgroup %s success", cgroup_target.c_str());
@@ -69,10 +52,31 @@ int attach(const string pid, const string cgroup_target) {
     // return_error
   }
 
+  string cg;
+
+  cg = getCgroup(pid);
+  if (cg.empty()) return_success;
+  if (cg == cgroup_target) {
+    debug("%s already in %s", pid.c_str(), cgroup_target.c_str());
+    return_success;
+  }
+
   // put pid to target cgroup
-  ofstream procs(cgroup_target_procs, ofstream::app);
+  if (write2procs(pid, cgroup_target_procs) != 0) return_error;
+
+  // wait for small period and check again
+  this_thread::sleep_for(std::chrono::milliseconds(100));
+  cg = getCgroup(pid);
+  if (cg.empty()) return_success;
+  if (cg != cgroup_target && write2procs(pid, cgroup_target_procs) != 0)
+    return_error;
+  return_success;
+}
+
+int write2procs(string pid, string procspath) {
+  ofstream procs(procspath, ofstream::app);
   if (!procs.is_open()) {
-    error("open file %s failed", cgroup_target_procs.c_str());
+    error("open file %s failed", procspath.c_str());
     return_error;
   }
   procs << pid.c_str() << endl;
@@ -80,8 +84,8 @@ int attach(const string pid, const string cgroup_target) {
 
   // maybe there some write error, for example process pid may not exist
   if (!procs) {
-    error("write %s to %s failed, maybe process %s not exist", pid.c_str(),
-          cgroup_target_procs.c_str(), pid.c_str());
+    error("write %s to %s failed, maybe process %s live too short", pid.c_str(),
+          procspath.c_str(), pid.c_str());
     return_error;
   }
   return_success;
@@ -91,4 +95,4 @@ int attach(const int pid, const string cgroup_target) {
   return attach(to_str(pid), cgroup_target);
 }
 
-} // namespace CGPROXY::CGROUP
+} // namespace CGPROXY::CGROUP

src/cgroup_attach.h

@@ -7,12 +7,11 @@ using namespace std;
 
 namespace CGPROXY::CGROUP {
 extern string cgroup2_mount_point;
-bool exist(string path);
 bool validate(string pid, string cgroup);
-string get_cgroup2_mount_point();
 int attach(const string pid, const string cgroup_target);
 int attach(const int pid, const string cgroup_target);
+int write2procs(string pid, string procspath);
 
 } // namespace CGPROXY::CGROUP
 
-#endif
+#endif

src/common.cmake.h

@@ -7,19 +7,21 @@
 #include <vector>
 using namespace std;
 
-#define TPROXY_IPTABLS_START "/usr/share/cgproxy/scripts/cgroup-tproxy.sh"
-#define TPROXY_IPTABLS_CLEAN "/usr/share/cgproxy/scripts/cgroup-tproxy.sh stop"
+#define TPROXY_IPTABLS_START "@CMAKE_INSTALL_FULL_DATADIR@/cgproxy/scripts/cgroup-tproxy.sh"
+#define TPROXY_IPTABLS_CLEAN "@CMAKE_INSTALL_FULL_DATADIR@/cgproxy/scripts/cgroup-tproxy.sh stop"
 
-#define LIBEXECSNOOP_SO "/usr/lib/cgproxy/libexecsnoop.so"
+#define LIBEXECSNOOP_SO "@CMAKE_INSTALL_FULL_LIBDIR@/cgproxy/libexecsnoop.so"
+#define CGROUP2_MOUNT_POINT "/var/run/cgproxy/cgroup2"
 #define PID_LOCK_FILE "/var/run/cgproxyd.pid"
 #define SOCKET_PATH "/tmp/cgproxy_unix_socket"
 #define LISTEN_BACKLOG 64
-#define DEFAULT_CONFIG_FILE "/etc/cgproxy/config.json"
+#define DEFAULT_CONFIG_FILE "@CMAKE_INSTALL_FULL_SYSCONFDIR@/cgproxy/config.json"
+#define READ_SIZE_MAX 128
 
 #define CGROUP_PROXY_PRESVERED "/proxy.slice"
 #define CGROUP_NOPROXY_PRESVERED "/noproxy.slice"
 
-#define THREAD_UNDEF 0
+#define THREAD_TIMEOUT 5
 
 #define MSG_TYPE_CONFIG_JSON 1
 #define MSG_TYPE_CONFIG_PATH 2
@@ -45,6 +47,15 @@ extern bool enable_info;
     fprintf(stderr, "error: ");                                                          \
     fprintf(stderr, __VA_ARGS__);                                                        \
     fprintf(stderr, "\n");                                                               \
+    fflush(stderr);                                                                      \
+  }
+
+#define warning(...)                                                                       \
+  {                                                                                      \
+    fprintf(stderr, "warning: ");                                                          \
+    fprintf(stderr, __VA_ARGS__);                                                        \
+    fprintf(stderr, "\n");                                                               \
+    fflush(stderr);                                                                      \
   }
 
 #define debug(...)                                                                       \
@@ -52,6 +63,7 @@ extern bool enable_info;
     fprintf(stdout, "debug: ");                                                          \
     fprintf(stdout, __VA_ARGS__);                                                        \
     fprintf(stdout, "\n");                                                               \
+    fflush(stdout);                                                                      \
   }
 
 #define info(...)                                                                        \
@@ -59,6 +71,7 @@ extern bool enable_info;
     fprintf(stdout, "info: ");                                                           \
     fprintf(stdout, __VA_ARGS__);                                                        \
     fprintf(stdout, "\n");                                                               \
+    fflush(stdout);                                                                      \
   }
 
 #define return_error return -1
@@ -74,6 +87,7 @@ template <typename... T> string to_str(T... args) {
 
 string join2str(const vector<string> t, const char delm = ' ');
 string join2str(const int argc, char **argv, const char delm = ' ');
+bool startWith(string prefix);
 
 bool validCgroup(const string cgroup);
 bool validCgroup(const vector<string> cgroup);
@@ -87,4 +101,12 @@ string bash_which(const string &name);
 string bash_readlink(const string &path);
 string getRealExistPath(const string &name);
 
-#endif
+/**
+ * whether cg1 belongs to cg2
+ */
+bool belongToCgroup(string cg1, string cg2);
+bool belongToCgroup(string cg1, vector<string> cg2);
+string getCgroup(const pid_t &pid);
+string getCgroup(const string &pid);
+
+#endif

src/common.cpp

@@ -1,7 +1,5 @@
 #include "common.h"
 #include <fstream>
-#include <limits.h>
-#include <linux/limits.h>
 #include <regex>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -24,6 +22,8 @@ string join2str(const int argc, char **argv, const char delm) {
   return s;
 }
 
+bool startWith(string s, string prefix) { return s.rfind(prefix, 0) == 0; }
+
 bool validCgroup(const string cgroup) {
   return regex_match(cgroup, regex("^/[a-zA-Z0-9\\-_./@]*$"));
 }
@@ -51,36 +51,35 @@ bool dirExist(const string &path) {
 
 vector<int> bash_pidof(const string &path) {
   vector<int> pids;
-  FILE *fp = popen(to_str("pidof ", path).c_str(), "r");
+  unique_ptr<FILE, decltype(&pclose)> fp(popen(to_str("pidof ", path).c_str(), "r"),
+                                         &pclose);
   if (!fp) return pids;
   int pid;
-  char buf[64];
-  while (fscanf(fp, "%d", &pid) != EOF) { pids.push_back(pid); }
-  pclose(fp);
+  while (fscanf(fp.get(), "%d", &pid) != EOF) { pids.push_back(pid); }
   return pids;
 }
 
 string bash_which(const string &name) {
   stringstream buffer;
-  FILE *fp = popen(to_str("which ", name).c_str(), "r");
+  unique_ptr<FILE, decltype(&pclose)> fp(popen(to_str("which ", name).c_str(), "r"),
+                                         &pclose);
   if (!fp) return "";
-  char buf[64];
-  while (fgets(buf, 64, fp) != NULL) { buffer << buf; }
-  pclose(fp);
+  char buf[READ_SIZE_MAX];
+  while (fgets(buf, READ_SIZE_MAX, fp.get()) != NULL) { buffer << buf; }
   string s = buffer.str();
-  s.pop_back(); // remove newline character
+  if (!s.empty()) s.pop_back(); // remove newline character
   return s;
 }
 
 string bash_readlink(const string &path) {
   stringstream buffer;
-  FILE *fp = popen(to_str("readlink -e ", path).c_str(), "r");
+  unique_ptr<FILE, decltype(&pclose)> fp(popen(to_str("readlink -e ", path).c_str(), "r"),
+                                         &pclose);
   if (!fp) return "";
-  char buf[64];
-  while (fgets(buf, 64, fp) != NULL) { buffer << buf; }
-  pclose(fp);
+  char buf[READ_SIZE_MAX];
+  while (fgets(buf, READ_SIZE_MAX, fp.get()) != NULL) { buffer << buf; }
   string s = buffer.str();
-  s.pop_back(); // remove newline character
+  if (!s.empty()) s.pop_back(); // remove newline character
   return s;
 }
 
@@ -92,4 +91,34 @@ string getRealExistPath(const string &name) {
   path = bash_readlink(path);
   if (!fileExist(path)) return "";
   return path;
-}
+}
+
+bool belongToCgroup(string cg1, string cg2) { return startWith(cg1 + '/', cg2 + '/'); }
+
+bool belongToCgroup(string cg1, vector<string> cg2) {
+  for (const auto &s : cg2) {
+    if (startWith(cg1 + '/', s + '/')) return true;
+  }
+  return false;
+}
+
+string getCgroup(const pid_t &pid) { return getCgroup(to_str(pid)); }
+
+string getCgroup(const string &pid) {
+  string cgroup_f = to_str("/proc/", pid, "/cgroup");
+  if (!fileExist(cgroup_f)) return "";
+
+  string cgroup, line;
+  ifstream ifs(cgroup_f);
+  debug("prcessing file %s", cgroup_f.c_str());
+  while (ifs.good() && getline(ifs, line)) {
+    // debug("process line: %s", line.c_str());
+    if (line[0] == '0') {
+      cgroup = line.substr(3);
+      debug("get cgroup of %s: %s", pid.c_str(), cgroup.c_str());
+      break;
+    }
+  }
+  ifs.close();
+  return cgroup;
+}

src/config.cpp

@@ -21,7 +21,7 @@ using json = nlohmann::json;
 namespace CGPROXY::CONFIG {
 
 void Config::toEnv() {
-  mergeReserved();
+  setenv("cgroup_mount_point", CGROUP2_MOUNT_POINT, 1);
   setenv("program_proxy", join2str(program_proxy, ':').c_str(), 1);
   setenv("program_noproxy", join2str(program_noproxy, ':').c_str(), 1);
   setenv("cgroup_proxy", join2str(cgroup_proxy, ':').c_str(), 1);
@@ -33,6 +33,9 @@ void Config::toEnv() {
   setenv("enable_udp", to_str(enable_udp).c_str(), 1);
   setenv("enable_ipv4", to_str(enable_ipv4).c_str(), 1);
   setenv("enable_ipv6", to_str(enable_ipv6).c_str(), 1);
+  setenv("table", to_str(table).c_str(), 1);
+  setenv("fwmark", to_str(fwmark).c_str(), 1);
+  setenv("mark_newin", to_str(mark_newin).c_str(), 1);
 }
 
 int Config::saveToFile(const string f) {
@@ -57,6 +60,9 @@ string Config::toJsonStr() {
   add2json(enable_udp);
   add2json(enable_ipv4);
   add2json(enable_ipv6);
+  add2json(table);
+  add2json(fwmark);
+  add2json(mark_newin);
   return j.dump();
 }
 
@@ -90,11 +96,16 @@ int Config::loadFromJsonStr(const string js) {
   tryassign(enable_udp);
   tryassign(enable_ipv4);
   tryassign(enable_ipv6);
+  tryassign(table);
+  tryassign(fwmark);
+  tryassign(mark_newin);
 
   // e.g. v2ray -> /usr/bin/v2ray -> /usr/lib/v2ray/v2ray
   toRealProgramPath(program_noproxy);
   toRealProgramPath(program_proxy);
 
+  mergeReserved();
+
   return 0;
 }
 
@@ -108,7 +119,7 @@ bool Config::validateJsonStr(const string js) {
   bool status = true;
   const set<string> boolset = {"enable_gateway", "enable_dns",  "enable_tcp",
                                "enable_udp",     "enable_ipv4", "enable_ipv6"};
-  const set<string> allowset = {"program_proxy", "program_noproxy"};
+  const set<string> allowset = {"program_proxy", "program_noproxy", "comment", "table", "fwmark", "mark_newin"};
   for (auto &[key, value] : j.items()) {
     if (key == "cgroup_proxy" || key == "cgroup_noproxy") {
       if (value.is_string() && !validCgroup((string)value)) status = false;
@@ -138,6 +149,7 @@ void Config::print_summary() {
   info("proxied program: %s", join2str(program_proxy).c_str());
   info("noproxy cgroup: %s", join2str(cgroup_noproxy).c_str());
   info("proxied cgroup: %s", join2str(cgroup_proxy).c_str());
+  info("table: %d, fwmark: %d, mark_newin: %d", table, fwmark, mark_newin);
 }
 
 void Config::toRealProgramPath(vector<string> &v) {
@@ -146,9 +158,13 @@ void Config::toRealProgramPath(vector<string> &v) {
     auto rpath = getRealExistPath(p);
     if (!rpath.empty()) tmp.push_back(rpath);
     else
-      error("%s not exist or broken link", p.c_str());
+      warning("%s not exist or broken link", p.c_str());
   }
   v = tmp;
 }
 
+#undef tryassign
+#undef add2json
+#undef merge
+
 } // namespace CGPROXY::CONFIG

src/config.h

@@ -13,8 +13,8 @@ public:
   const string cgroup_proxy_preserved = CGROUP_PROXY_PRESVERED;
   const string cgroup_noproxy_preserved = CGROUP_NOPROXY_PRESVERED;
 
-  vector<string> program_proxy;
-  vector<string> program_noproxy;
+  vector<string> program_proxy = {cgroup_proxy_preserved};
+  vector<string> program_noproxy = {cgroup_noproxy_preserved};
   vector<string> cgroup_proxy;
   vector<string> cgroup_noproxy;
   bool enable_gateway = false;
@@ -25,6 +25,11 @@ public:
   bool enable_ipv4 = true;
   bool enable_ipv6 = true;
 
+  // for iptables
+  int table=10007;
+  int fwmark=0x9973;
+  int mark_newin=0x9967;
+
   void toEnv();
   int saveToFile(const string f);
   string toJsonStr();

src/socket_client.cpp

@@ -42,11 +42,7 @@ void send(const char *msg, int &status) {
 }
 
 void send(const string msg, int &status) {
-  int msg_len = msg.length();
-  char buff[msg_len];
-  msg.copy(buff, msg_len, 0);
-  buff[msg_len] = '\0';
-  send(buff, status);
+  send(msg.c_str(), status);
   debug("return status: %d", status);
 }
 

src/socket_server.cpp

@@ -1,6 +1,7 @@
 #include "socket_server.h"
 #include "common.h"
 #include <filesystem>
+#include <stdlib.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -9,7 +10,7 @@ namespace fs = std::filesystem;
 
 namespace CGPROXY::SOCKET {
 
-void SocketServer::socketListening(function<int(char *)> callback) {
+void SocketServer::socketListening(function<int(char *)> callback, promise<void> status) {
   debug("starting socket listening");
   sfd = socket(AF_UNIX, SOCK_STREAM, 0);
 
@@ -26,6 +27,8 @@ void SocketServer::socketListening(function<int(char *)> callback) {
   listen(sfd, LISTEN_BACKLOG);
   chmod(SOCKET_PATH, S_IRWXU | S_IRWXG | S_IRWXO);
 
+  status.set_value();
+
   while (true) {
     close(cfd);
     cfd = accept(sfd, NULL, NULL);
@@ -37,29 +40,28 @@ void SocketServer::socketListening(function<int(char *)> callback) {
     flag = read(cfd, &msg_len, sizeof(int));
     continue_if_error(flag, "read length");
     // read msg
-    char msg[msg_len];
+    auto msg = (char *)malloc(msg_len + 1);
     flag = read(cfd, msg, msg_len * sizeof(char));
     continue_if_error(flag, "read msg");
     msg[msg_len] = '\0';
     // handle msg
     int status = callback(msg);
+    free(msg);
     // send back flag
     flag = write(cfd, &status, sizeof(int));
     continue_if_error(flag, "write back");
   }
 }
 
-void *startThread(void *arg) {
-  thread_arg *p = (thread_arg *)arg;
-  SocketServer server;
-  server.socketListening(p->handle_msg);
-  return (void *)0;
-}
-
 SocketServer::~SocketServer() {
   close(sfd);
   close(cfd);
   unlink(SOCKET_PATH);
 }
 
+void startThread(function<int(char *)> callback, promise<void> status) {
+  SocketServer server;
+  server.socketListening(callback, move(status));
+}
+
 } // namespace CGPROXY::SOCKET

src/socket_server.h

@@ -2,6 +2,7 @@
 #define SOCKET_SERVER_H
 
 #include <functional>
+#include <future>
 #include <stdlib.h>
 #include <sys/un.h>
 using namespace std;
@@ -14,20 +15,17 @@ namespace CGPROXY::SOCKET {
     continue;                                                                            \
   }
 
-struct thread_arg {
-  function<int(char *)> handle_msg;
-};
-void *startThread(void *arg);
-
 class SocketServer {
 public:
   int sfd = -1, cfd = -1, flag = -1;
   struct sockaddr_un unix_socket;
 
-  void socketListening(function<int(char *)> callback);
+  void socketListening(function<int(char *)> callback, promise<void> status);
   ~SocketServer();
 };
 
+void startThread(function<int(char *)> callback, promise<void> status);
+
 } // namespace CGPROXY::SOCKET
 
 #endif

tools/CMakeLists.txt

@@ -2,11 +2,4 @@ include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/src)
 
 add_executable(cgattach cgattach.cpp ../src/cgroup_attach.cpp ../src/common.cpp)
-install(TARGETS cgattach  DESTINATION /usr/bin PERMISSIONS ${basic_permission})
-
-if (with_execsnoop)
-add_executable(execsnoop_exec execsnoop.cpp ../src/common.cpp ../src/execsnoop.cpp)
-set_target_properties(execsnoop_exec PROPERTIES OUTPUT_NAME execsnoop)
-target_link_libraries(execsnoop_exec bcc)
-install(TARGETS execsnoop_exec  DESTINATION /usr/bin PERMISSIONS ${basic_permission})
-endif()
+install(TARGETS cgattach  DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} PERMISSIONS ${basic_permission})

tools/execsnoop.cpp

@@ -1,24 +0,0 @@
-#include "execsnoop.h"
-#include "common.h"
-#include <unistd.h>
-using namespace std;
-using namespace CGPROXY::EXECSNOOP;
-
-#define PATH_MAX_LEN 128
-
-int handle_pid(int pid) {
-  char path[PATH_MAX_LEN];
-  auto size = readlink(to_str("/proc/", pid, "/exe").c_str(), path, PATH_MAX_LEN);
-  if (size == -1) error("readlink: %s", to_str("/proc/", pid, "/exe").c_str());
-  path[size] = '\0';
-  info("%d %s", pid, path);
-  return 0;
-}
-
-int main() {
-  enable_debug = true;
-  enable_info = true;
-  callback = handle_pid;
-  execsnoop();
-  return 0;
-}

v2ray_config/05_inbounds_04_socks5.json

@@ -5,6 +5,7 @@
       "port": 1080,
       "protocol": "socks",
       "settings": {
+        "udp": true,
         "auth": "noauth",
         "userLevel": 0
       },

v2ray_config/readme.md

@@ -1,8 +1,8 @@
 ## Usage
-- Fill `06_outbounds_myproxy.json` with your vmess proxy config with tag `outBound_PROXY`.
+- Fill `06_outbounds_myproxy.json` with your VMess proxy config with tag `outBound_PROXY`.
 - Start with  `sudo v2ray -confdir .`
 
 ## Reference
 
-- [v2ray multi-file config](https://www.v2fly.org/chapter_02/multiple_config.html)
+- [v2ray multi-file config](https://www.v2fly.org/config/multiple_config.html)