bump version

so make cgproxy and cgnoproxy command have highest priority

.gitignore

@@ -1,5 +1,9 @@
 build
 .directory
-.vscode
+.clangd
 v2ray_config/proxy
 v2ray_config/06_outbounds_proxy.json
+aur-*
+gh-md-toc
+add_toc.sh
+compile_commands.json

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "execsnoop-libbpf/libbpf"]
+	path = execsnoop-libbpf/libbpf
+	url = https://github.com/libbpf/libbpf.git

.vscode/gdb-root

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec sudo -E /usr/bin/gdb "$@"

.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+    "configurations": [
+    {
+        "name": "(gdb) cgproxyd",
+        "type": "cppdbg",
+        "request": "launch",
+        "program": "${workspaceFolder}/build/src/cgproxy",
+        "args": ["--daemon", "--debug", "--execsnoop"],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "miDebuggerPath": "${workspaceFolder}/.vscode/gdb-root",
+        "setupCommands": [
+            {
+                "description": "Enable pretty-printing for gdb",
+                "text": "-enable-pretty-printing",
+                "ignoreFailures": true
+            }
+        ]
+    }]
+}

CMakeLists.txt

@@ -1,54 +1,45 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.14)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-project(cgproxy VERSION 3.7)
-add_executable(cgattach cgattach.cpp)
+project(cgproxy VERSION 0.18)
 
-install(TARGETS cgattach DESTINATION /usr/bin 
-        PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE SETUID)
-install(FILES  cgproxy.sh DESTINATION /usr/bin 
-        RENAME cgproxy
-        PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
-install(FILES  cgnoproxy.sh DESTINATION /usr/bin 
-        RENAME cgnoproxy
-        PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
-# install(FILES  run_in_cgroup.sh DESTINATION /usr/bin 
-#         RENAME run_in_cgroup
-#        PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+include(GNUInstallDirs)
 
-install(FILES cgproxy.service 
-        DESTINATION /usr/lib/systemd/system/)
-install(FILES cgproxy.conf 
-        DESTINATION /etc/)
-install(FILES cgroup-tproxy.sh 
-        DESTINATION /usr/share/cgproxy/scripts/)
-        
-install(FILES readme.md
-        DESTINATION /share/doc/cgproxy/)
+add_compile_options(-Wall -Wextra -Wpedantic -Wno-unused-result -Wno-unused-parameter)
 
+# for clangd
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-## package for deb and rpm
-set(CPACK_GENERATOR "DEB;RPM")
-set(CPACK_PACKAGE_NAME "cgproxy")
-set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "cgproxy will transparent proxy anything running in specific cgroup.It aslo supports global transparent proxy and gateway proxy")
+# option(with_execsnoop "enable program level proxy control feature, need bcc installed" ON)
+option(build_execsnoop_dl "build libexecsnoop.so which will be dynamic loaded, otherwise built directly into cgproxy" ON)
+option(build_static "build with static link prefered" OFF)
+option(build_tools OFF)
+option(build_test "for develop" OFF)
 
-## deb pack
-set(CPACK_DEBIAN_PACKAGE_NAME "cgproxy")
-set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "x86_64")
-set(CPACK_DEBIAN_PACKAGE_DEPENDS "systemd")
-set(CPACK_DEBIAN_PACKAGE_SECTION "network")
-set(CPACK_DEBIAN_PACKAGE_PRIORITY "Optional")
-set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/springzfx/cgproxy")
-set(CPACK_DEBIAN_PACKAGE_MAINTAINER "springzfx@gmail.com")
-set(CONTROL_DIR ${CMAKE_SOURCE_DIR}/control)
-set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CONTROL_DIR}/postinst;${CONTROL_DIR}/prerm")
+add_subdirectory(src)
+add_subdirectory(execsnoop-kernel)
+add_subdirectory(pack)
+if (build_tools)
+    add_subdirectory(tools)
+endif()
+if (build_test)
+    add_subdirectory(test)
+endif()
 
-## rpm pack
-set(CPACK_RPM_PACKAGE_ARCHITECTURE, "x86_64")
-set(CPACK_RPM_PACKAGE_REQUIRES "systemd")
-set(CPACK_RPM_PACKAGE_GROUP "network")
-set(CPACK_RPM_PACKAGE_URL "https://github.com/springzfx/cgproxy")
-set(CONTROL_DIR ${CMAKE_SOURCE_DIR}/control)
-set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CONTROL_DIR}/postinst")
-set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CONTROL_DIR}/prerm")
+# instal scripts and other things
+install(PROGRAMS cgproxyd TYPE BIN)
+install(PROGRAMS cgnoproxy TYPE BIN)
+install(PROGRAMS cgroup-tproxy.sh DESTINATION ${CMAKE_INSTALL_DATADIR}/cgproxy/scripts)
+install(FILES cgproxy.service DESTINATION ${CMAKE_INSTALL_LIBDIR}/systemd/system)
+install(FILES config.json DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/cgproxy)
+install(FILES readme.md DESTINATION ${CMAKE_INSTALL_DOCDIR})
 
-include(CPack)
+# man pages
+set(man_gz ${PROJECT_SOURCE_DIR}/man/cgproxyd.1.gz ${PROJECT_SOURCE_DIR}/man/cgproxy.1.gz ${PROJECT_SOURCE_DIR}/man/cgnoproxy.1.gz)
+add_custom_command(OUTPUT ${man_gz}
+    COMMAND gzip -fk cgproxyd.1 cgproxy.1 cgnoproxy.1
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/man
+)
+add_custom_target(man ALL DEPENDS ${man_gz})
+install(FILES ${man_gz} DESTINATION ${CMAKE_INSTALL_MANDIR}/man1/)

_clang-format

@@ -0,0 +1,137 @@
+---
+Language:        Cpp
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -2
+AlignAfterOpenBracket: Align
+AlignConsecutiveMacros: false
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: true
+AllowAllConstructorInitializersOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: Always
+AllowShortCaseLabelsOnASingleLine: true
+AllowShortFunctionsOnASingleLine: All
+AllowShortLambdasOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Always
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: MultiLine
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterCaseLabel:  false
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     90
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DeriveLineEnding: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority:        2
+    SortPriority:    0
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
+    SortPriority:    0
+  - Regex:           '.*'
+    Priority:        1
+    SortPriority:    0
+IncludeIsMainRegex: '(Test)?$'
+IncludeIsMainSourceRegex: ''
+IndentCaseLabels: false
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInConditionalStatement: false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpaceBeforeSquareBrackets: false
+Standard:        Latest
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        8
+UseCRLF:         false
+UseTab:          Never
+...
+

cgattach.cpp

@@ -1,98 +0,0 @@
-#include <errno.h>
-#include <fstream>
-#include <iostream>
-#include <regex>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-using namespace std;
-
-void print_usage() { fprintf(stdout, "usage: cgattach <pid> <cgroup>\n"); }
-
-bool exist(string path) {
-  struct stat st;
-  if (stat(path.c_str(), &st) != -1) {
-    return S_ISDIR(st.st_mode);
-  }
-  return false;
-}
-
-bool validate(string pid, string cgroup) {
-  bool pid_v = regex_match(pid, regex("^[0-9]+$"));
-  bool cg_v = regex_match(cgroup, regex("^\\/[a-zA-Z0-9\\-_./@]*$"));
-  if (pid_v && cg_v)
-    return true;
- 
-  fprintf(stderr, "paramater validate error\n");
-  print_usage();
-  exit(EXIT_FAILURE);
-}
-
-string get_cgroup2_mount_point(){
-  char cgroup2_mount_point[100]="";
-  FILE* fp = popen("findmnt -t cgroup2 -n -o TARGET", "r");
-  int count=fscanf(fp,"%s",&cgroup2_mount_point);
-  fclose(fp);
-  if (count=0){
-    fprintf(stderr, "cgroup2 not supported\n");
-    exit(EXIT_FAILURE);
-  }
-  return cgroup2_mount_point;
-}
-
-int main(int argc, char *argv[]) {
-  setuid(0);
-  setgid(0);
-  if (getuid() != 0 || getgid() != 0) {
-    fprintf(stderr, "cgattach need suid sticky bit or run with root\n");
-    exit(EXIT_FAILURE);
-  }
-
-  if (argc != 3) {
-    fprintf(stderr, "only need 2 paramaters\n");
-    print_usage();
-    exit(EXIT_FAILURE);
-  }
-
-  string pid = string(argv[1]);
-  string cgroup_target = string(argv[2]);
-  validate(pid, cgroup_target);
-  // string cgroup_mount_point = "/sys/fs/cgroup";
-  string cgroup_mount_point = get_cgroup2_mount_point();
-  string cgroup_target_path = cgroup_mount_point + cgroup_target;
-  string cgroup_target_procs = cgroup_target_path + "/cgroup.procs";
-
-  // check if exist, we will create it if not exist
-  if (!exist(cgroup_target_path)) {
-    if (mkdir(cgroup_target_path.c_str(),
-              S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH) == 0) {
-      fprintf(stdout, "created cgroup %s success\n", cgroup_target.c_str());
-    } else {
-      fprintf(stderr, "created cgroup %s failed, errno %d\n",
-              cgroup_target.c_str(), errno);
-      exit(EXIT_FAILURE);
-    }
-    // fprintf(stderr, "cgroup %s not exist\n",cgroup_target.c_str());
-    // exit(EXIT_FAILURE);
-  }
-
-  // put pid to target cgroup
-  ofstream procs(cgroup_target_procs, ofstream::app);
-  if (!procs.is_open()) {
-    fprintf(stderr, "open file %s failed\n", cgroup_target_procs.c_str());
-    exit(EXIT_FAILURE);
-  }
-  procs << pid.c_str() << endl;
-  procs.close();
-
-  // maybe there some write error, for example process pid may not exist
-  if (!procs) {
-    fprintf(stderr, "write %s to %s failed, maybe process %s not exist\n",
-            pid.c_str(), cgroup_target_procs.c_str(), pid.c_str());
-    exit(EXIT_FAILURE);
-  }
-  return EXIT_SUCCESS;
-}

cgnoproxy

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/bin/cgproxy --noproxy $@

cgnoproxy.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-config="/etc/cgproxy.conf"
-source $config
-
-# test suid bit
-if [ -u "$(which cgattach)" ]; then 
-    cgattach $$ $cgroup_noproxy && attached=1
-else
-    sudo cgattach $$ $cgroup_noproxy && attached=1
-fi
-
-# test attach success or not
-[[ -z "$attached" ]] && echo "attach error" && exit 1
-
-exec "$@"

cgproxy.conf

@@ -1,27 +0,0 @@
-## cgroup transparent proxy
-## see how to configure, https://github.com/springzfx/cgproxy
-
-## any process in cgroup_proxy will be proxied, and cgroup_noproxy the opposite
-## note, cgroup must start with slash '/'
-# cgroup_proxy="/"  # for global tproxy
-# cgroup_noproxy="/system.slice/v2ray.service" # for v2ray service
-cgroup_proxy="/proxy.slice"
-cgroup_noproxy="/noproxy.slice"
-
-## allow as gateway for local network
-enable_gateway=false
-
-## listening port of another proxy process, for example v2ray 
-port=12345
-
-## if you set to false, it's traffic won't go through proxy, but still can go direct to internet 
-enable_dns=true
-enable_tcp=true
-enable_udp=true
-enable_ipv4=true
-enable_ipv6=true
-
-## do not modify this if you don't known what you are doing
-table=100
-fwmark=0x01
-mark_newin=0x02

cgproxy.service

@@ -1,11 +1,10 @@
 [Unit]
-Description=proxy cgroup
+Description=cgproxy service
 After=network.target
 
 [Service]
-ExecStart=sh /usr/share/cgproxy/scripts/cgroup-tproxy.sh --config=/etc/cgproxy.conf
-ExecStop= sh /usr/share/cgproxy/scripts/cgroup-tproxy.sh stop
-RemainAfterExit=1
+Type=simple
+ExecStart=/usr/bin/cgproxyd --execsnoop
 
 [Install]
 WantedBy=multi-user.target

cgproxy.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-config="/etc/cgproxy.conf"
-source $config
-
-# test suid bit
-if [ -u "$(which cgattach)" ]; then 
-    cgattach $$ $cgroup_proxy && attached=1
-else
-    sudo cgattach $$ $cgroup_proxy && attached=1
-fi
-
-# test attach success or not
-[[ -z "$attached" ]] && echo "attach error" && exit 1
-
-exec "$@"

cgproxyd

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/bin/cgproxy --daemon $@

cgroup-tproxy.sh

@@ -30,67 +30,100 @@ cat << 'DOC'
 DOC
 }
 
+## check root
+[ ! $(id -u) -eq 0 ] && { >&2 echo "iptables: need root to modify iptables";exit -1; }
+
 ## any process in this cgroup will be proxied
-cgroup_proxy="/proxy.slice"
-cgroup_noproxy="/noproxy.slice"
+if [ -z ${cgroup_proxy+x} ]; then  
+    cgroup_proxy="/proxy.slice"
+else
+    IFS=':' read -r -a cgroup_proxy     <<< "$cgroup_proxy"
+fi
+
+## any process in this cgroup will not be proxied
+if [ -z ${cgroup_noproxy+x} ]; then  
+    cgroup_noproxy="/noproxy.slice"
+else
+    IFS=':' read -r -a cgroup_noproxy   <<< "$cgroup_noproxy"
+fi
 
 # allow as gateway for local network
-enable_gateway=false
+[ -z ${enable_gateway+x} ] && enable_gateway=false
 
 ## some variables
-port=12345
+[ -z ${port+x} ] && port=12345
 
 ## some options
-enable_dns=true
-enable_tcp=true
-enable_udp=true
-enable_ipv4=true
-enable_ipv6=true
+[ -z ${enable_dns+x} ]  && enable_dns=true
+[ -z ${enable_tcp+x} ]  && enable_tcp=true
+[ -z ${enable_udp+x} ]  && enable_udp=true
+[ -z ${enable_ipv4+x} ] && enable_ipv4=true
+[ -z ${enable_ipv6+x} ] && enable_ipv6=true
 
-## do not modify this if you don't known what you are doing
-table=100
-fwmark=0x01
-make_newin=0x02
+##
+get_available_route_table(){
+    table=10007 
+    while true; do 
+        ip route show table $table &> /dev/null && ((table++)) || { echo $table && break; }
+    done
+}
+
+## mark/route things
+[ -z ${table+x} ]       && table=10007 # just a prime number
+[ -z ${fwmark+x} ]      && fwmark=0x9973
+[ -z ${mark_newin+x} ]  && mark_newin=0x9967
+
+
+# echo "table: $table fwmark: $fwmark, mark_newin: $mark_newin"
 
 ## cgroup things
-cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET)
-cgroup_type="cgroup2"
-cgroup_procs_file="cgroup.procs"
+[ -z ${cgroup_mount_point+x} ] && cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET | head -n 1)
+[ -z $cgroup_mount_point ] && { >&2 echo "iptables: no cgroup2 mount point available"; exit -1; }
+[ ! -d $cgroup_mount_point ] && mkdir -p $cgroup_mount_point
+[ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" != "cgroup2" ] && mount -t cgroup2 none $cgroup_mount_point
+[ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" != "cgroup2" ] && { >&2 echo "iptables: mount $cgroup_mount_point failed"; exit -1; }
+
+stop(){
+    iptables -t mangle -L TPROXY_PRE &> /dev/null || return
+    echo "iptables: cleaning tproxy iptables"
+    iptables -t mangle -D PREROUTING -j TPROXY_PRE
+    iptables -t mangle -D OUTPUT -j TPROXY_OUT
+    iptables -t mangle -F TPROXY_PRE
+    iptables -t mangle -F TPROXY_OUT
+    iptables -t mangle -F TPROXY_ENT
+    iptables -t mangle -X TPROXY_PRE
+    iptables -t mangle -X TPROXY_OUT
+    iptables -t mangle -X TPROXY_ENT
+    ip6tables -t mangle -D PREROUTING -j TPROXY_PRE
+    ip6tables -t mangle -D OUTPUT -j TPROXY_OUT
+    ip6tables -t mangle -F TPROXY_PRE
+    ip6tables -t mangle -F TPROXY_OUT
+    ip6tables -t mangle -F TPROXY_ENT
+    ip6tables -t mangle -X TPROXY_PRE
+    ip6tables -t mangle -X TPROXY_OUT
+    ip6tables -t mangle -X TPROXY_ENT
+    ip rule delete fwmark $fwmark lookup $table
+    ip route flush table $table
+    ip -6 rule delete fwmark $fwmark lookup $table
+    ip -6 route flush table $table
+    ## may not exist, just ignore, and tracking their existence is not reliable
+    iptables -t nat -D POSTROUTING -m owner ! --socket-exists -j MASQUERADE &> /dev/null
+    ip6tables -t nat -D POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE &> /dev/null
+    ## unmount cgroup2
+    [ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" = "cgroup2" ] && umount $cgroup_mount_point
+}
 
 ## parse parameter
 for i in "$@"
 do
 case $i in
     stop)
-        iptables -t mangle -D PREROUTING -j TPROXY_PRE
-        iptables -t mangle -D OUTPUT -j TPROXY_OUT
-        iptables -t mangle -F TPROXY_PRE
-        iptables -t mangle -F TPROXY_OUT
-        iptables -t mangle -F TPROXY_ENT
-        iptables -t mangle -X TPROXY_PRE
-        iptables -t mangle -X TPROXY_OUT
-        iptables -t mangle -X TPROXY_ENT
-        ip6tables -t mangle -D PREROUTING -j TPROXY_PRE
-        ip6tables -t mangle -D OUTPUT -j TPROXY_OUT
-        ip6tables -t mangle -F TPROXY_PRE
-        ip6tables -t mangle -F TPROXY_OUT
-        ip6tables -t mangle -F TPROXY_ENT
-        ip6tables -t mangle -X TPROXY_PRE
-        ip6tables -t mangle -X TPROXY_OUT
-        ip6tables -t mangle -X TPROXY_ENT
-        ip rule delete fwmark $fwmark lookup $table
-        ip route flush table $table
-        ip -6 rule delete fwmark $fwmark lookup $table
-        ip -6 route flush table $table
-        ## may not exist, just ignore, and tracking their existence is not reliable
-        iptables -t nat -D POSTROUTING -m owner ! --socket-exists -j MASQUERADE &> /dev/null
-        ip6tables -t nat -D POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE &> /dev/null
+        stop
         exit 0
         ;;
     --config=*)
         config=${i#*=}
         source $config
-        shift
         ;;
     --help)
         print_help
@@ -100,16 +133,33 @@ esac
 done
 
 ## TODO cgroup need to exists before using in iptables since 5.6.5, maybe it's bug
+## only create the first one in arrary
 test -d $cgroup_mount_point$cgroup_proxy    || mkdir $cgroup_mount_point$cgroup_proxy   || exit -1; 
 test -d $cgroup_mount_point$cgroup_noproxy  || mkdir $cgroup_mount_point$cgroup_noproxy || exit -1; 
 
+## filter cgroup that not exist
+_cgroup_noproxy=()
+for cg in ${cgroup_noproxy[@]}; do
+    test -d $cgroup_mount_point$cg && _cgroup_noproxy+=($cg) || { >&2 echo "iptables: $cg not exist, ignore";}
+done
+unset cgroup_noproxy && cgroup_noproxy=${_cgroup_noproxy[@]}
+
+## filter cgroup that not exist
+_cgroup_proxy=()
+for cg in ${cgroup_proxy[@]}; do
+    test -d $cgroup_mount_point$cg && _cgroup_proxy+=($cg) || { >&2 echo "iptables: $cg not exist, ignore";}
+done
+unset cgroup_proxy && cgroup_proxy=${_cgroup_proxy[@]}
+
+
+echo "iptables: applying tproxy iptables"
 ## use TPROXY
 #ipv4#
 ip rule add fwmark $fwmark table $table
 ip route add local default dev lo table $table
 iptables -t mangle -N TPROXY_ENT
-iptables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
-iptables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
+iptables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark
+iptables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark
 
 iptables -t mangle -N TPROXY_PRE
 iptables -t mangle -A TPROXY_PRE -m socket --transparent -j MARK --set-mark $fwmark
@@ -124,19 +174,23 @@ iptables -t mangle -A PREROUTING -j TPROXY_PRE
 
 iptables -t mangle -N TPROXY_OUT
 iptables -t mangle -A TPROXY_OUT -p icmp -j RETURN
-iptables -t mangle -A TPROXY_OUT -m connmark --mark  $make_newin -j RETURN
+iptables -t mangle -A TPROXY_OUT -m connmark --mark  $mark_newin -j RETURN
 iptables -t mangle -A TPROXY_OUT -m addrtype --dst-type LOCAL -j RETURN
 iptables -t mangle -A TPROXY_OUT -m addrtype ! --dst-type UNICAST -j RETURN
-iptables -t mangle -A TPROXY_OUT -m cgroup --path $cgroup_noproxy -j RETURN
-iptables -t mangle -A TPROXY_OUT -m cgroup --path $cgroup_proxy -j MARK --set-mark $fwmark
+for cg in ${cgroup_noproxy[@]}; do
+iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
+done
+for cg in ${cgroup_proxy[@]}; do
+iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark
+done
 iptables -t mangle -A OUTPUT -j TPROXY_OUT
 
 #ipv6#
 ip -6 rule add fwmark $fwmark table $table
 ip -6 route add local default dev lo table $table
 ip6tables -t mangle -N TPROXY_ENT
-ip6tables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
-ip6tables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
+ip6tables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark
+ip6tables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark
 
 ip6tables -t mangle -N TPROXY_PRE
 ip6tables -t mangle -A TPROXY_PRE -m socket --transparent -j MARK --set-mark $fwmark
@@ -151,11 +205,15 @@ ip6tables -t mangle -A PREROUTING -j TPROXY_PRE
 
 ip6tables -t mangle -N TPROXY_OUT
 ip6tables -t mangle -A TPROXY_OUT -p icmpv6 -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m connmark --mark  $make_newin -j RETURN
+ip6tables -t mangle -A TPROXY_OUT -m connmark --mark  $mark_newin -j RETURN
 ip6tables -t mangle -A TPROXY_OUT -m addrtype --dst-type LOCAL -j RETURN
 ip6tables -t mangle -A TPROXY_OUT -m addrtype ! --dst-type UNICAST -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cgroup_noproxy -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cgroup_proxy -j MARK --set-mark $fwmark
+for cg in ${cgroup_noproxy[@]}; do
+ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
+done
+for cg in ${cgroup_proxy[@]}; do
+ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark
+done
 ip6tables -t mangle -A OUTPUT -j TPROXY_OUT
 
 ## allow to disable, order is important
@@ -185,13 +243,13 @@ $enable_gateway || ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LO
 
 ## make sure following rules are the first in chain TPROXY_PRE to mark new incoming connection or gateway proxy connection
 ## so must put at last to insert first
-iptables  -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $make_newin
-ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $make_newin
+iptables  -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $mark_newin
+ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $mark_newin
 
-## message for user
+# message for user
 cat << DOC
-noproxy cgroup: $cgroup_noproxy
-proxied cgroup: $cgroup_proxy
+iptables: noproxy cgroup: ${cgroup_noproxy[@]}
+iptables: proxied cgroup: ${cgroup_proxy[@]}
 DOC
 
 
@@ -200,5 +258,5 @@ if $enable_gateway; then
     ip6tables -t nat -A POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE # only masquerade ipv6 private address
     sysctl -w net.ipv4.ip_forward=1
     sysctl -w net.ipv6.conf.all.forwarding=1
-    echo "gateway enabled"
+    echo "ipatbles: gateway enabled"
 fi

config.json

@@ -0,0 +1,18 @@
+{
+    "comment":"For usgae, see https://github.com/springzfx/cgproxy",
+
+    "port": 12345,
+    "program_noproxy": ["v2ray", "qv2ray"],
+    "program_proxy": [],
+    "cgroup_noproxy": ["/system.slice/v2ray.service"],
+    "cgroup_proxy": [],
+    "enable_gateway": false,
+    "enable_dns": true,
+    "enable_udp": true,
+    "enable_tcp": true,
+    "enable_ipv4": true,
+    "enable_ipv6": true,
+    "table": 10007,
+    "fwmark": 39283,
+    "mark_newin": 39271
+}

execsnoop-bcc/CMakeLists.txt

@@ -0,0 +1,6 @@
+include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+add_library(execsnoop MODULE execsnoop.cpp ../src/common.cpp)
+target_link_libraries(execsnoop bcc)
+install(TARGETS execsnoop DESTINATION /usr/lib/cgproxy/ PERMISSIONS ${basic_permission})

execsnoop-bcc/execsnoop.cpp

@@ -0,0 +1,104 @@
+#include "execsnoop.h"
+#include "bcc/BPF.h"
+#include "common.h"
+#include <bcc/libbpf.h>
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <string>
+#include <unistd.h>
+using namespace std;
+
+namespace CGPROXY::EXECSNOOP {
+
+const string BPF_PROGRAM = R"(
+#include <linux/fs.h>
+#include <linux/sched.h>
+#include <uapi/linux/ptrace.h>
+
+struct data_t {
+    int pid;
+};
+
+BPF_PERF_OUTPUT(events);
+
+int syscall_execve(struct pt_regs *ctx,
+    const char __user *filename,
+    const char __user *const __user *__argv,
+    const char __user *const __user *__envp)
+{
+    struct data_t data = {};
+    data.pid = bpf_get_current_pid_tgid();
+    events.perf_submit(ctx, &data, sizeof(struct data_t));
+    return 0;
+}
+
+int ret_syscall_execve(struct pt_regs *ctx){
+    struct data_t data = {};
+    data.pid = bpf_get_current_pid_tgid();
+    int retval = PT_REGS_RC(ctx);
+    if (retval==0)
+      events.perf_submit(ctx, &data, sizeof(struct data_t));
+    return 0;
+}
+)";
+
+struct data_t {
+  int pid;
+};
+
+function<int(int)> callback = NULL;
+promise<void> status;
+
+void handle_events(void *cb_cookie, void *data, int data_size) {
+  auto event = static_cast<data_t *>(data);
+  int pid = event->pid;
+
+  if (callback) callback(pid);
+}
+
+int execsnoop() {
+  debug("starting execsnoop");
+  ebpf::BPF bpf;
+
+  auto init_res = bpf.init(BPF_PROGRAM);
+  if (init_res.code() != 0) {
+    error("bpf init failed, maybe linux-headers not installed");
+    std::cerr << init_res.msg() << std::endl;
+    return 1;
+  }
+
+  string execve_fnname = bpf.get_syscall_fnname("execve");
+  // auto attach_res = bpf.attach_kprobe(execve_fnname, "syscall_execve");
+  auto attach_res =
+      bpf.attach_kprobe(execve_fnname, "ret_syscall_execve", 0, BPF_PROBE_RETURN);
+  if (attach_res.code() != 0) {
+    std::cerr << attach_res.msg() << std::endl;
+    return 1;
+  }
+
+  auto open_res = bpf.open_perf_buffer("events", &handle_events);
+  if (open_res.code() != 0) {
+    std::cerr << open_res.msg() << std::endl;
+    return 1;
+  }
+
+  if (bpf.free_bcc_memory()) {
+    std::cerr << "Failed to free llvm/clang memory" << std::endl;
+    return 1;
+  }
+
+  status.set_value();
+
+  while (true) bpf.poll_perf_buffer("events");
+
+  return 0;
+}
+
+void startThread(function<int(int)> c, promise<void> _status) {
+  status = move(_status);
+  callback = c;
+  execsnoop();
+}
+
+} // namespace CGPROXY::EXECSNOOP

execsnoop-bcc/execsnoop.h

@@ -0,0 +1,23 @@
+#ifndef EXECSNOOP_HPP
+#define EXECSNOOP_HPP 1
+
+#include <functional>
+#include <future>
+#include <string>
+using namespace std;
+
+namespace CGPROXY::EXECSNOOP {
+
+extern const string BPF_PROGRAM;
+struct data_t;
+extern function<int(int)> callback;
+void handle_events(void *cb_cookie, void *data, int data_size);
+int execsnoop();
+
+extern "C" void startThread(function<int(int)> c, promise<void> _status);
+// typedef void startThread_t(function<int(int)>, promise<void>);
+using startThread_t=decltype(startThread);
+startThread_t *_startThread; // only for dlsym()
+
+} // namespace CGPROXY::EXECSNOOP
+#endif

execsnoop-bcc/readme.md

@@ -0,0 +1,3 @@
+- depend [bcc](https://github.com/iovisor/bcc) 
+
+- huge memory usage, at least 50M

execsnoop-kernel/CMakeLists.txt

@@ -0,0 +1,19 @@
+# find libbpf
+if (build_static)
+    find_library(LIBBPF libbpf.a)
+else()
+    find_library(LIBBPF bpf)
+endif()
+
+if (LIBBPF-NOTFOUND)
+    message(FATAL_ERROR "libbpf not found")
+endif()
+
+if (build_execsnoop_dl)
+    add_library(execsnoop MODULE execsnoop_share.cpp)
+    install(TARGETS execsnoop DESTINATION ${CMAKE_INSTALL_LIBDIR}/cgproxy/)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF} -lelf -lz)
+else()
+    add_library(execsnoop STATIC execsnoop_share.cpp)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF} -l:libelf.a -l:libz.a)
+endif()

execsnoop-kernel/execsnoop_kern.c

@@ -0,0 +1,93 @@
+#include "linux/sched.h"
+#include <linux/version.h>
+#include <linux/ptrace.h>
+#include <uapi/linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+/* /sys/kernel/debug/tracing/events/syscalls/sys_enter_execve/format */
+struct syscalls_enter_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	const char filename_ptr;
+    const char *const * argv;
+    const char *const * envp;
+};
+
+/* /sys/kernel/debug/tracing/events/syscalls/sys_exit_execve/format */
+struct syscalls_exit_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	long ret;
+};
+
+struct bpf_map_def SEC("maps") records = {
+    .type = BPF_MAP_TYPE_HASH,
+    .key_size = sizeof(pid_t),
+    .value_size = sizeof(struct event),
+    .max_entries = 1024,
+};
+struct bpf_map_def SEC("maps") perf_events = {
+    .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
+    .key_size = sizeof(u32),
+    .value_size = sizeof(u32),
+    .max_entries = 128,
+};
+
+SEC("tracepoint/syscalls/sys_enter_execve")
+int syscall_enter_execve(struct syscalls_enter_execve_args *ctx){
+    pid_t pid, tgid; uid_t uid;
+	struct event *event;
+	struct task_struct *task, *task_p;
+
+    u64 id = bpf_get_current_pid_tgid();
+	pid = (pid_t)id;
+	tgid = id >> 32;
+	uid = (u32)bpf_get_current_uid_gid();
+
+	struct event empty_event={};
+	if (bpf_map_update_elem(&records, &pid, &empty_event, BPF_NOEXIST)!=0) return 0;
+	event = bpf_map_lookup_elem(&records, &pid);
+	if (!event) return 0;
+
+	event->pid = pid;
+	event->tgid = tgid;
+	event->uid = uid;
+
+	/* ppid is not reliable here, normal in arch, but become 0 in ubuntu 20.04 */
+	task = (struct task_struct*)bpf_get_current_task();
+	bpf_probe_read(&task_p, sizeof(struct task_struct*),&task->real_parent);
+	bpf_probe_read(&event->ppid,sizeof(pid_t),&task_p->tgid);
+
+    return 0;
+}
+
+SEC("tracepoint/syscalls/sys_exit_execve")
+int syscall_exit_execve(struct syscalls_exit_execve_args *ctx){
+	pid_t pid;
+	struct event *event;
+
+	pid = (pid_t)bpf_get_current_pid_tgid();
+	event = bpf_map_lookup_elem(&records,&pid);
+	if (!event) return 0;
+	if (ctx->ret < 0) goto cleanup;
+
+	/* get comm */
+	bpf_get_current_comm(&event->comm,sizeof(event->comm));
+	bpf_perf_event_output(ctx, &perf_events, BPF_F_CURRENT_CPU, event, sizeof(struct event));
+cleanup:
+	bpf_map_delete_elem(&records, &pid);
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
+u32 _version SEC("version") = LINUX_VERSION_CODE;

execsnoop-kernel/execsnoop_kern_skel.h

@@ -0,0 +1,225 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+
+/* THIS FILE IS AUTOGENERATED! */
+#ifndef __EXECSNOOP_KERN_SKEL_H__
+#define __EXECSNOOP_KERN_SKEL_H__
+
+#include <stdlib.h>
+#include <bpf/libbpf.h>
+
+struct execsnoop_kern {
+	struct bpf_object_skeleton *skeleton;
+	struct bpf_object *obj;
+	struct {
+		struct bpf_map *perf_events;
+		struct bpf_map *records;
+	} maps;
+	struct {
+		struct bpf_program *syscall_enter_execve;
+		struct bpf_program *syscall_exit_execve;
+	} progs;
+	struct {
+		struct bpf_link *syscall_enter_execve;
+		struct bpf_link *syscall_exit_execve;
+	} links;
+};
+
+static void
+execsnoop_kern__destroy(struct execsnoop_kern *obj)
+{
+	if (!obj)
+		return;
+	if (obj->skeleton)
+		bpf_object__destroy_skeleton(obj->skeleton);
+	free(obj);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj);
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_opts(const struct bpf_object_open_opts *opts)
+{
+	struct execsnoop_kern *obj;
+
+	obj = (typeof(obj))calloc(1, sizeof(*obj));
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__create_skeleton(obj))
+		goto err;
+	if (bpf_object__open_skeleton(obj->skeleton, opts))
+		goto err;
+
+	return obj;
+err:
+	execsnoop_kern__destroy(obj);
+	return NULL;
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open(void)
+{
+	return execsnoop_kern__open_opts(NULL);
+}
+
+static inline int
+execsnoop_kern__load(struct execsnoop_kern *obj)
+{
+	return bpf_object__load_skeleton(obj->skeleton);
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_and_load(void)
+{
+	struct execsnoop_kern *obj;
+
+	obj = execsnoop_kern__open();
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__load(obj)) {
+		execsnoop_kern__destroy(obj);
+		return NULL;
+	}
+	return obj;
+}
+
+static inline int
+execsnoop_kern__attach(struct execsnoop_kern *obj)
+{
+	return bpf_object__attach_skeleton(obj->skeleton);
+}
+
+static inline void
+execsnoop_kern__detach(struct execsnoop_kern *obj)
+{
+	return bpf_object__detach_skeleton(obj->skeleton);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj)
+{
+	struct bpf_object_skeleton *s;
+
+	s = (typeof(s))calloc(1, sizeof(*s));
+	if (!s)
+		return -1;
+	obj->skeleton = s;
+
+	s->sz = sizeof(*s);
+	s->name = "execsnoop_kern";
+	s->obj = &obj->obj;
+
+	/* maps */
+	s->map_cnt = 2;
+	s->map_skel_sz = sizeof(*s->maps);
+	s->maps = (typeof(s->maps))calloc(s->map_cnt, s->map_skel_sz);
+	if (!s->maps)
+		goto err;
+
+	s->maps[0].name = "perf_events";
+	s->maps[0].map = &obj->maps.perf_events;
+
+	s->maps[1].name = "records";
+	s->maps[1].map = &obj->maps.records;
+
+	/* programs */
+	s->prog_cnt = 2;
+	s->prog_skel_sz = sizeof(*s->progs);
+	s->progs = (typeof(s->progs))calloc(s->prog_cnt, s->prog_skel_sz);
+	if (!s->progs)
+		goto err;
+
+	s->progs[0].name = "syscall_enter_execve";
+	s->progs[0].prog = &obj->progs.syscall_enter_execve;
+	s->progs[0].link = &obj->links.syscall_enter_execve;
+
+	s->progs[1].name = "syscall_exit_execve";
+	s->progs[1].prog = &obj->progs.syscall_exit_execve;
+	s->progs[1].link = &obj->links.syscall_exit_execve;
+
+	s->data_sz = 2320;
+	s->data = (void *)"\
+\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\xd0\x05\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0d\0\
+\x01\0\x85\0\0\0\x0e\0\0\0\xbf\x06\0\0\0\0\0\0\x63\x6a\xfc\xff\0\0\0\0\x85\0\0\
+\0\x0f\0\0\0\xbf\x07\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\
+\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\
+\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
+\xd0\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb7\x04\0\0\x01\0\0\0\x85\
+\0\0\0\x02\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x19\0\0\0\0\0\
+\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\0\x15\x08\x12\0\0\0\0\0\x77\x06\0\
+\0\x20\0\0\0\x61\xa1\xfc\xff\0\0\0\0\x63\x78\x1c\0\0\0\0\0\x63\x68\x14\0\0\0\0\
+\0\x63\x18\x10\0\0\0\0\0\x85\0\0\0\x23\0\0\0\x07\0\0\0\x18\x05\0\0\xbf\xa1\0\0\
+\0\0\0\0\x07\x01\0\0\xf0\xff\xff\xff\xb7\x02\0\0\x08\0\0\0\xbf\x03\0\0\0\0\0\0\
+\x85\0\0\0\x04\0\0\0\x07\x08\0\0\x18\0\0\0\x79\xa3\xf0\xff\0\0\0\0\x07\x03\0\0\
+\x0c\x05\0\0\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\
+\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\xbf\x16\0\0\0\0\0\0\x85\0\0\0\x0e\0\0\0\x63\
+\x0a\xfc\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x15\x07\x13\
+\0\0\0\0\0\x79\x61\x10\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\x6d\x12\x0b\0\0\0\0\0\xbf\
+\x71\0\0\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x85\0\0\0\x10\0\0\0\xbf\x61\0\0\0\0\0\0\
+\x18\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\x03\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\
+\0\xbf\x74\0\0\0\0\0\0\xb7\x05\0\0\x20\0\0\0\x85\0\0\0\x19\0\0\0\xbf\xa2\0\0\0\
+\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\
+\0\x03\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x01\0\0\0\x04\0\0\0\x20\0\0\0\
+\0\x04\0\0\0\0\0\0\x04\0\0\0\x04\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\
+\0\x06\x07\x05\0\x10\0\0\0\0\0\0\0\x01\x7a\x52\0\x08\x7c\x0b\x01\x0c\0\0\0\x18\
+\0\0\0\x18\0\0\0\0\0\0\0\0\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\x34\0\0\
+\0\0\0\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\xba\0\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xe9\0\0\0\0\0\x03\0\x70\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe2\0\0\0\0\0\x05\0\
+\xc0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdb\0\0\0\0\0\x05\0\xe8\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\
+\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa3\0\0\0\x11\0\x08\0\0\0\0\0\0\0\0\0\
+\x04\0\0\0\0\0\0\0\x20\0\0\0\x11\0\x09\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\x07\
+\0\0\0\x11\0\x07\0\x14\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x18\0\0\0\x11\0\x07\0\0\
+\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x8e\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\0\x80\x01\
+\0\0\0\0\0\0\x51\0\0\0\x12\0\x05\0\0\0\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\x70\0\0\0\
+\0\0\0\0\x01\0\0\0\x0a\0\0\0\xb8\0\0\0\0\0\0\0\x01\0\0\0\x0a\0\0\0\x28\0\0\0\0\
+\0\0\0\x01\0\0\0\x0a\0\0\0\x88\0\0\0\0\0\0\0\x01\0\0\0\x09\0\0\0\xd0\0\0\0\0\0\
+\0\0\x01\0\0\0\x0a\0\0\0\x1c\0\0\0\0\0\0\0\x01\0\0\0\x05\0\0\0\x38\0\0\0\0\0\0\
+\0\x01\0\0\0\x06\0\0\0\0\x2e\x74\x65\x78\x74\0\x70\x65\x72\x66\x5f\x65\x76\x65\
+\x6e\x74\x73\0\x6d\x61\x70\x73\0\x72\x65\x63\x6f\x72\x64\x73\0\x5f\x76\x65\x72\
+\x73\x69\x6f\x6e\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\x6f\x69\x6e\x74\x2f\
+\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\x78\x69\x74\x5f\x65\
+\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\x65\x78\x69\x74\x5f\x65\
+\x78\x65\x63\x76\x65\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\x6f\x69\x6e\x74\
+\x2f\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\x6e\x74\x65\x72\
+\x5f\x65\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\x65\x6e\x74\x65\
+\x72\x5f\x65\x78\x65\x63\x76\x65\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x72\
+\x65\x6c\x2e\x65\x68\x5f\x66\x72\x61\x6d\x65\0\x65\x78\x65\x63\x73\x6e\x6f\x6f\
+\x70\x5f\x6b\x65\x72\x6e\x2e\x63\0\x2e\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\
+\x6d\x74\x61\x62\0\x4c\x42\x42\x31\x5f\x34\0\x4c\x42\x42\x31\x5f\x33\0\x4c\x42\
+\x42\x30\x5f\x33\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xcb\0\0\0\
+\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe0\x04\0\0\0\0\0\0\xf0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x69\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\x40\0\0\0\0\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x65\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x70\
+\x04\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x0c\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\
+\0\0\0\0\0\0\x2d\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc0\x01\0\0\
+\0\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x29\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x04\0\0\0\0\0\0\x30\0\
+\0\0\0\0\0\0\x0c\0\0\0\x05\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x13\0\0\0\
+\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x02\0\0\0\0\0\0\x28\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa4\0\0\0\x01\0\0\0\x03\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe0\x02\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x21\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\xe4\x02\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\0\0\0\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\xe8\x02\0\0\0\0\0\0\x50\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\xac\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc0\x04\0\0\0\
+\0\0\0\x20\0\0\0\0\0\0\0\x0c\0\0\0\x0a\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\
+\0\xd3\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x38\x03\0\0\0\0\0\0\x38\
+\x01\0\0\0\0\0\0\x01\0\0\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
+
+	return 0;
+err:
+	bpf_object__destroy_skeleton(s);
+	return -1;
+}
+
+#endif /* __EXECSNOOP_KERN_SKEL_H__ */

execsnoop-kernel/execsnoop_share.cpp

@@ -0,0 +1,83 @@
+
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include <sys/resource.h>
+#include "execsnoop_kern_skel.h"
+#include "execsnoop_share.h"
+
+namespace CGPROXY::EXECSNOOP {
+
+#define PERF_BUFFER_PAGES 64
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+function<int(int)> callback = NULL;
+promise<void> status;
+
+static void handle_event(void *ctx, int cpu, void *data, __u32 size) {
+  	auto e = static_cast<event*>(data);
+  	if (callback) callback(e->pid);
+}
+
+void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) {
+	fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
+}
+
+int bump_memlock_rlimit(void) {
+	struct rlimit rlim_new = { RLIM_INFINITY, RLIM_INFINITY };
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
+int execsnoop() {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int err;
+	
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+	
+	struct execsnoop_kern *obj=execsnoop_kern__open_and_load();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	err = execsnoop_kern__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		return err;
+	}
+
+	pb_opts.sample_cb = handle_event;
+	pb_opts.lost_cb = handle_lost_events;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	err = libbpf_get_error(pb);
+	if (err) {
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
+	}
+
+	// notify
+  	status.set_value();
+
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return err;
+}
+
+void startThread(function<int(int)> c, promise<void> _status) {
+  status = move(_status);
+  callback = c;
+  execsnoop();
+}
+
+}

execsnoop-kernel/execsnoop_share.h

@@ -0,0 +1,20 @@
+#ifndef EXECSNOOP_SHARE_HPP
+#define EXECSNOOP_SHARE_HPP 1
+
+#include <functional>
+#include <future>
+#include <string>
+using namespace std;
+
+namespace CGPROXY::EXECSNOOP {
+
+extern "C" void startThread(function<int(int)> c, promise<void> _status);
+
+#ifdef BUIlD_EXECSNOOP_DL
+// only for dlsym()
+using startThread_t=decltype(startThread);
+startThread_t *_startThread;
+#endif
+
+} // namespace CGPROXY::EXECSNOOP
+#endif

execsnoop-kernel/execsnoop_user.c

@@ -0,0 +1,40 @@
+
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include "bpf_load.h"
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+static void print_bpf_output(void *ctx, int cpu, void *data, __u32 size) {
+	struct event *e=data;
+    printf("comm: %s, pid: %d, tgid: %d, ppid: %d, uid: %d\n",e->comm,e->pid,e->tgid,e->ppid,e->uid);
+}
+
+int main(int argc, char **argv) {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int ret;
+
+	if (load_bpf_file("execsnoop_kern.o")!=0) {
+		printf("%s", bpf_log_buf);
+		return 1;
+	}
+
+	pb_opts.sample_cb = print_bpf_output;
+	pb = perf_buffer__new(map_fd[1], 8, &pb_opts);
+	ret = libbpf_get_error(pb);
+	if (ret) {
+		printf("failed to setup perf_buffer: %d\n", ret);
+		return 1;
+	}
+
+	while ((ret = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return ret;
+}

execsnoop-kernel/execsnoop_user_1.c

@@ -0,0 +1,69 @@
+
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include <sys/resource.h>
+#include "execsnoop_kern_skel.h"
+// #include "bpf_load.h"
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+#define PERF_BUFFER_PAGES   64
+
+static void print_bpf_output(void *ctx, int cpu, void *data, __u32 size) {
+	struct event *e=data;
+    printf("comm: %s, pid: %d, tgid: %d, ppid: %d, uid: %d\n",e->comm,e->pid,e->tgid,e->ppid,e->uid);
+}
+
+int bump_memlock_rlimit(void)
+{
+	struct rlimit rlim_new = {
+		.rlim_cur	= RLIM_INFINITY,
+		.rlim_max	= RLIM_INFINITY,
+	};
+
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
+
+int main(int argc, char **argv) {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int err;
+	
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+	
+	struct execsnoop_kern *obj=execsnoop_kern__open_and_load();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	err = execsnoop_kern__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		return err;
+	}
+
+	pb_opts.sample_cb = print_bpf_output;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	err = libbpf_get_error(pb);
+	if (err) {
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
+	}
+
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return err;
+}

execsnoop-kernel/readme.md

@@ -0,0 +1,119 @@
+
+## Prons and cons
+
+- use stable execve tracepoint, so build once and should work everywhere
+- need to build in kernel tree
+
+## Build in kernel tree
+
+- ready and config kernel tree
+
+```bash
+# kernel config
+gunzip -c /proc/config.gz > .config
+make oldconfig && make prepare
+# install headers to ./usr/include
+make headers_install -j8
+# build bpf
+make M=samples/bpf -j8
+```
+
+- put or link `execsnoop_kern.c` and `execsnoop_user.c` to *samples/bpf/*
+- edit  *samples/bpf/makefile*
+
+```makefile
+# in samples/bpf/makefile
+tprogs-y += execsnoop
+execsnoop-objs := bpf_load.o execsnoop_user.o $(TRACE_HELPERS)
+always-y += execsnoop_kern.o
+```
+
+## Run
+
+```bash
+cd samples/bpf
+sudo bash -c "ulimit -l unlimited && ./execsnoop"
+```
+
+## Detail build command
+
+using `make V=1 M=samples/bpf | tee -a log.txt` to get and filter following command
+
+- build `execsnoop_kern.o`
+
+  note `-g` is needed if with BPF CO-RE
+
+```bash
+clang -nostdinc \
+	-isystem /usr/lib/gcc/x86_64-pc-linux-gnu/10.1.0/include \
+	-I./arch/x86/include \
+	-I./arch/x86/include/generated  \
+	-I./include \
+	-I./arch/x86/include/uapi \
+	-I./arch/x86/include/generated/uapi \
+	-I./include/uapi \
+	-I./include/generated/uapi \
+	-include ./include/linux/kconfig.h \
+	-I./samples/bpf \
+	-I./tools/testing/selftests/bpf/ \
+	-I./tools/lib/ \
+	-include asm_goto_workaround.h \
+	-D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \
+	-D__TARGET_ARCH_x86 -Wno-compare-distinct-pointer-types \
+	-Wno-gnu-variable-sized-type-not-at-end \
+	-Wno-address-of-packed-member -Wno-tautological-compare \
+	-Wno-unknown-warning-option  \
+	-fno-stack-protector \
+	-O2 -emit-llvm -c samples/bpf/execsnoop_kern.c \
+	-o - | llc -march=bpf  -filetype=obj -o samples/bpf/execsnoop_kern.o
+```
+
+- build `execsnoop_user.o`
+
+```bash
+  gcc -Wall -O2 -Wmissing-prototypes -Wstrict-prototypes \
+  	-I./usr/include \
+  	-I./tools/testing/selftests/bpf/ \
+  	-I./tools/lib/ \
+  	-I./tools/include \
+  	-I./tools/perf \
+  	-DHAVE_ATTR_TEST=0  \
+  	-c -o samples/bpf/execsnoop_user.o samples/bpf/execsnoop_user.c
+```
+
+- build `execsnoop`
+
+```bash
+  gcc -Wall -O2 -Wmissing-prototypes -Wstrict-prototypes \
+	  -I./usr/include \
+	  -I./tools/testing/selftests/bpf/ \
+	  -I./tools/lib/ \
+	  -I./tools/include \
+	  -I./tools/perf \
+	  -DHAVE_ATTR_TEST=0 \
+	  -o samples/bpf/execsnoop \
+	  samples/bpf/bpf_load.o samples/bpf/execsnoop_user.o \
+	  tools/testing/selftests/bpf/trace_helpers.o tools/lib/bpf/libbpf.a \
+	  -lelf -lz
+```
+
+
+
+## With bpftool
+
+- gen
+
+```
+bpftool gen skeleton execsnoop_kern.o > execsnoop_kern_skel.h
+```
+
+- build
+
+```
+gcc -Wall -O2 execsnoop_user_1.c -o execsnoop -lbpf
+```
+
+## Some resources
+
+- [A thorough introduction to eBPF](https://lwn.net/Articles/740157/)
+- [Write eBPF program in pure C](http://terenceli.github.io/%E6%8A%80%E6%9C%AF/2020/01/18/ebpf-in-c)

execsnoop-libbpf/Makefile

@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+OUTPUT := $(abspath build)
+CLANG ?= clang
+CFLAGS ?= -g -O2 -Wall
+LLVM_STRIP ?= llvm-strip
+BPFTOOL ?= bpftool
+LIBBPF_DIR := $(abspath libbpf/src)
+LIBBPF_OBJ := $(OUTPUT)/usr/lib64/libbpf.a
+LIBBPF_H := $(OUTPUT)/usr/include
+INCLUDES := -I$(OUTPUT) -I$(LIBBPF_H)
+ARCH := $(shell uname -m | sed 's/x86_64/x86/')
+
+APPS = execsnoop
+LIBSO = libexecsnoop.so
+
+.PHONY: all
+all: $(APPS) $(LIBSO)
+
+ifeq ($(V),1)
+Q =
+msg =
+else
+Q = @
+msg = @printf '  %-8s %s%s\n' "$(1)" "$(notdir $(2))" "$(if $(3), $(3))";
+MAKEFLAGS += --no-print-directory
+endif
+
+COMMON_OBJ = \
+	$(OUTPUT)/trace_helpers.o \
+	$(OUTPUT)/syscall_helpers.o \
+	$(OUTPUT)/errno_helpers.o \
+
+
+.PHONY: install
+install: $(LIBSO)
+	install -D $(LIBSO) -t $(DESTDIR)/usr/lib/cgproxy/
+
+.PHONY: clean
+clean:
+	$(call msg,CLEAN)
+	$(Q)rm -rf $(OUTPUT) $(APPS) $(LIBSO)
+
+$(OUTPUT) $(OUTPUT)/libbpf:
+	$(call msg,MKDIR,$@)
+	$(Q)mkdir -p $@
+
+$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) $(COMMON_OBJ) | $(OUTPUT)
+	$(call msg,BINARY,$@)
+	$(Q)$(CC) $(CFLAGS) $^ -lelf -lz -o $@
+
+$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
+
+$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
+	$(call msg,CC,$@)
+	$(Q)$(CC) $(CFLAGS) $(INCLUDES) -fPIC -c $(filter %.c,$^) -o $@
+
+$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT)
+	$(call msg,GEN-SKEL,$@)
+	$(Q)$(BPFTOOL) gen skeleton $< > $@
+
+$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) vmlinux.h | $(OUTPUT)
+	$(call msg,BPF,$@)
+	$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH)		      \
+		     $(INCLUDES) -c $(filter %.c,$^) -o $@ &&		      \
+	$(LLVM_STRIP) -g $@
+
+vmlinux.h:
+	$(Q)$(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
+
+$(LIBBPF_OBJ):
+	$(Q)cd $(LIBBPF_DIR) && make DESTDIR=${OUTPUT} prefix=$(OUTPUT) CPPFLAGS="-fPIC" install install_headers
+
+# patch: | $(LIBBPF_DIR)/Makefile
+# 	@echo "patching libbpf-fPIC.patch"
+# 	$(Q)cd libbpf/src && patch -p1 --forward -i ../../libbpf-fPIC.patch || true
+
+# build libexecsnoop.so
+$(LIBSO): execsnoop_share.cpp $(OUTPUT)/execsnoop.skel.h $(LIBBPF_OBJ) $(COMMON_OBJ) | $(OUTPUT) 
+	$(call msg,LIB,$@)
+	$(Q)$(CXX) -std=c++17 $(CFLAGS) $(INCLUDES) -fPIC $< $(COMMON_OBJ) -shared -lelf -lz -Wl,--no-whole-archive,--no-undefined $(LIBBPF_OBJ) -o $@
+
+
+# delete failed targets
+.DELETE_ON_ERROR:
+# keep intermediate (.skel.h, .bpf.o, etc) targets
+.SECONDARY:
+

execsnoop-libbpf/errno_helpers.c

@@ -0,0 +1,232 @@
+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+// Copyright (c) 2020 Anton Protopopov
+#include <stdlib.h>
+#include <limits.h>
+#include <string.h>
+#include <errno.h>
+#include <stdio.h>
+
+#define warn(...) fprintf(stderr, __VA_ARGS__)
+
+#ifdef __x86_64__
+static int errno_by_name_x86_64(const char *errno_name)
+{
+
+#define strcase(X, N) if (!strcmp(errno_name, (X))) return N
+
+	strcase("EPERM", 1);
+	strcase("ENOENT", 2);
+	strcase("ESRCH", 3);
+	strcase("EINTR", 4);
+	strcase("EIO", 5);
+	strcase("ENXIO", 6);
+	strcase("E2BIG", 7);
+	strcase("ENOEXEC", 8);
+	strcase("EBADF", 9);
+	strcase("ECHILD", 10);
+	strcase("EAGAIN", 11);
+	strcase("EWOULDBLOCK", 11);
+	strcase("ENOMEM", 12);
+	strcase("EACCES", 13);
+	strcase("EFAULT", 14);
+	strcase("ENOTBLK", 15);
+	strcase("EBUSY", 16);
+	strcase("EEXIST", 17);
+	strcase("EXDEV", 18);
+	strcase("ENODEV", 19);
+	strcase("ENOTDIR", 20);
+	strcase("EISDIR", 21);
+	strcase("EINVAL", 22);
+	strcase("ENFILE", 23);
+	strcase("EMFILE", 24);
+	strcase("ENOTTY", 25);
+	strcase("ETXTBSY", 26);
+	strcase("EFBIG", 27);
+	strcase("ENOSPC", 28);
+	strcase("ESPIPE", 29);
+	strcase("EROFS", 30);
+	strcase("EMLINK", 31);
+	strcase("EPIPE", 32);
+	strcase("EDOM", 33);
+	strcase("ERANGE", 34);
+	strcase("EDEADLK", 35);
+	strcase("EDEADLOCK", 35);
+	strcase("ENAMETOOLONG", 36);
+	strcase("ENOLCK", 37);
+	strcase("ENOSYS", 38);
+	strcase("ENOTEMPTY", 39);
+	strcase("ELOOP", 40);
+	strcase("ENOMSG", 42);
+	strcase("EIDRM", 43);
+	strcase("ECHRNG", 44);
+	strcase("EL2NSYNC", 45);
+	strcase("EL3HLT", 46);
+	strcase("EL3RST", 47);
+	strcase("ELNRNG", 48);
+	strcase("EUNATCH", 49);
+	strcase("ENOCSI", 50);
+	strcase("EL2HLT", 51);
+	strcase("EBADE", 52);
+	strcase("EBADR", 53);
+	strcase("EXFULL", 54);
+	strcase("ENOANO", 55);
+	strcase("EBADRQC", 56);
+	strcase("EBADSLT", 57);
+	strcase("EBFONT", 59);
+	strcase("ENOSTR", 60);
+	strcase("ENODATA", 61);
+	strcase("ETIME", 62);
+	strcase("ENOSR", 63);
+	strcase("ENONET", 64);
+	strcase("ENOPKG", 65);
+	strcase("EREMOTE", 66);
+	strcase("ENOLINK", 67);
+	strcase("EADV", 68);
+	strcase("ESRMNT", 69);
+	strcase("ECOMM", 70);
+	strcase("EPROTO", 71);
+	strcase("EMULTIHOP", 72);
+	strcase("EDOTDOT", 73);
+	strcase("EBADMSG", 74);
+	strcase("EOVERFLOW", 75);
+	strcase("ENOTUNIQ", 76);
+	strcase("EBADFD", 77);
+	strcase("EREMCHG", 78);
+	strcase("ELIBACC", 79);
+	strcase("ELIBBAD", 80);
+	strcase("ELIBSCN", 81);
+	strcase("ELIBMAX", 82);
+	strcase("ELIBEXEC", 83);
+	strcase("EILSEQ", 84);
+	strcase("ERESTART", 85);
+	strcase("ESTRPIPE", 86);
+	strcase("EUSERS", 87);
+	strcase("ENOTSOCK", 88);
+	strcase("EDESTADDRREQ", 89);
+	strcase("EMSGSIZE", 90);
+	strcase("EPROTOTYPE", 91);
+	strcase("ENOPROTOOPT", 92);
+	strcase("EPROTONOSUPPORT", 93);
+	strcase("ESOCKTNOSUPPORT", 94);
+	strcase("ENOTSUP", 95);
+	strcase("EOPNOTSUPP", 95);
+	strcase("EPFNOSUPPORT", 96);
+	strcase("EAFNOSUPPORT", 97);
+	strcase("EADDRINUSE", 98);
+	strcase("EADDRNOTAVAIL", 99);
+	strcase("ENETDOWN", 100);
+	strcase("ENETUNREACH", 101);
+	strcase("ENETRESET", 102);
+	strcase("ECONNABORTED", 103);
+	strcase("ECONNRESET", 104);
+	strcase("ENOBUFS", 105);
+	strcase("EISCONN", 106);
+	strcase("ENOTCONN", 107);
+	strcase("ESHUTDOWN", 108);
+	strcase("ETOOMANYREFS", 109);
+	strcase("ETIMEDOUT", 110);
+	strcase("ECONNREFUSED", 111);
+	strcase("EHOSTDOWN", 112);
+	strcase("EHOSTUNREACH", 113);
+	strcase("EALREADY", 114);
+	strcase("EINPROGRESS", 115);
+	strcase("ESTALE", 116);
+	strcase("EUCLEAN", 117);
+	strcase("ENOTNAM", 118);
+	strcase("ENAVAIL", 119);
+	strcase("EISNAM", 120);
+	strcase("EREMOTEIO", 121);
+	strcase("EDQUOT", 122);
+	strcase("ENOMEDIUM", 123);
+	strcase("EMEDIUMTYPE", 124);
+	strcase("ECANCELED", 125);
+	strcase("ENOKEY", 126);
+	strcase("EKEYEXPIRED", 127);
+	strcase("EKEYREVOKED", 128);
+	strcase("EKEYREJECTED", 129);
+	strcase("EOWNERDEAD", 130);
+	strcase("ENOTRECOVERABLE", 131);
+	strcase("ERFKILL", 132);
+	strcase("EHWPOISON", 133);
+
+#undef strcase
+
+	return -1;
+
+}
+#endif
+
+/* Try to find the errno number using the errno(1) program */
+static int errno_by_name_dynamic(const char *errno_name)
+{
+	int len = strlen(errno_name);
+	int err, number = -1;
+	char buf[128];
+	char cmd[64];
+	char *end;
+	long val;
+	FILE *f;
+
+	/* sanity check to not call popen with random input */
+	for (int i = 0; i < len; i++) {
+		if (errno_name[i] < 'A' || errno_name[i] > 'Z') {
+			warn("errno_name contains invalid char 0x%02x: %s\n",
+					errno_name[i], errno_name);
+			return -1;
+		}
+	}
+
+	snprintf(cmd, sizeof(cmd), "errno %s", errno_name);
+	f = popen(cmd, "r");
+	if (!f) {
+		warn("popen: %s: %s\n", cmd, strerror(errno));
+		return -1;
+	}
+
+	if (!fgets(buf, sizeof(buf), f)) {
+		goto close;
+	} else if (ferror(f)) {
+		warn("fgets: %s\n", strerror(errno));
+		goto close;
+	}
+
+	// expecting "<name> <number> <description>"
+	if (strncmp(errno_name, buf, len) || strlen(buf) < len+2) {
+		warn("expected '%s': %s\n", errno_name, buf);
+		goto close;
+	}
+	errno = 0;
+	val = strtol(buf+len+2, &end, 10);
+	if (errno || end == (buf+len+2) || number < 0 || number > INT_MAX) {
+		warn("can't parse the second column, expected int: %s\n", buf);
+		goto close;
+	}
+	number = val;
+
+close:
+	err = pclose(f);
+	if (err < 0)
+		warn("pclose: %s\n", strerror(errno));
+#ifndef __x86_64__
+	/* Ignore the error for x86_64 where we have a table compiled in */
+	else if (err && WEXITSTATUS(err) == 127) {
+		warn("errno(1) required for errno name/number mapping\n");
+	} else if (err) {
+		warn("errno(1) exit status (see wait(2)): 0x%x\n", err);
+	}
+#endif
+	return number;
+}
+
+int errno_by_name(const char *errno_name)
+{
+#ifdef __x86_64__
+	int err;
+
+	err = errno_by_name_x86_64(errno_name);
+	if (err >= 0)
+		return err;
+#endif
+
+	return errno_by_name_dynamic(errno_name);
+}

execsnoop-libbpf/errno_helpers.h

@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __ERRNO_HELPERS_H
+#define __ERRNO_HELPERS_H
+
+int errno_by_name(const char *errno_name);
+
+#endif /* __ERRNO_HELPERS_H */

execsnoop-libbpf/execsnoop.bpf.c

@@ -0,0 +1,128 @@
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_core_read.h>
+#include "execsnoop.h"
+
+const volatile bool ignore_failed = true;
+const volatile uid_t targ_uid = INVALID_UID;
+const volatile int max_args = DEFAULT_MAXARGS;
+
+static const struct event empty_event = {};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 10240);
+	__type(key, pid_t);
+	__type(value, struct event);
+} execs SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
+	__uint(key_size, sizeof(u32));
+	__uint(value_size, sizeof(u32));
+} events SEC(".maps");
+
+static __always_inline bool valid_uid(uid_t uid) {
+	return uid != INVALID_UID;
+}
+
+SEC("tracepoint/syscalls/sys_enter_execve")
+int tracepoint__syscalls__sys_enter_execve(struct trace_event_raw_sys_enter* ctx)
+{
+	u64 id;
+	pid_t pid, tgid;
+	unsigned int ret;
+	struct event *event;
+	struct task_struct *task;
+	const char **args = (const char **)(ctx->args[1]);
+	const char *argp;
+	uid_t uid = (u32)bpf_get_current_uid_gid();
+
+	if (valid_uid(targ_uid) && targ_uid != uid)
+		return 0;
+
+	id = bpf_get_current_pid_tgid();
+	pid = (pid_t)id;
+	tgid = id >> 32;
+	if (bpf_map_update_elem(&execs, &pid, &empty_event, BPF_NOEXIST))
+		return 0;
+
+	event = bpf_map_lookup_elem(&execs, &pid);
+	if (!event)
+		return 0;
+
+	event->pid = pid;
+	event->tgid = tgid;
+	event->uid = uid;
+	task = (struct task_struct*)bpf_get_current_task();
+	event->ppid = (pid_t)BPF_CORE_READ(task, real_parent, tgid);
+	event->args_count = 0;
+	event->args_size = 0;
+
+	ret = bpf_probe_read_str(event->args, ARGSIZE, (const char*)ctx->args[0]);
+	if (ret <= ARGSIZE) {
+		event->args_size += ret;
+	} else {
+		/* write an empty string */
+		event->args[0] = '\0';
+		event->args_size++;
+	}
+
+	event->args_count++;
+	#pragma unroll
+	for (int i = 1; i < TOTAL_MAX_ARGS && i < max_args; i++) {
+		bpf_probe_read(&argp, sizeof(argp), &args[i]);
+		if (!argp)
+			return 0;
+
+		if (event->args_size > LAST_ARG)
+			return 0;
+
+		ret = bpf_probe_read_str(&event->args[event->args_size], ARGSIZE, argp);
+		if (ret > ARGSIZE)
+			return 0;
+
+		event->args_count++;
+		event->args_size += ret;
+	}
+	/* try to read one more argument to check if there is one */
+	bpf_probe_read(&argp, sizeof(argp), &args[max_args]);
+	if (!argp)
+		return 0;
+
+	/* pointer to max_args+1 isn't null, asume we have more arguments */
+	event->args_count++;
+	return 0;
+}
+
+SEC("tracepoint/syscalls/sys_exit_execve")
+int tracepoint__syscalls__sys_exit_execve(struct trace_event_raw_sys_exit* ctx)
+{
+	u64 id;
+	pid_t pid;
+	int ret;
+	struct event *event;
+	u32 uid = (u32)bpf_get_current_uid_gid();
+
+	if (valid_uid(targ_uid) && targ_uid != uid)
+		return 0;
+	id = bpf_get_current_pid_tgid();
+	pid = (pid_t)id;
+	event = bpf_map_lookup_elem(&execs, &pid);
+	if (!event)
+		return 0;
+	ret = ctx->ret;
+	if (ignore_failed && ret < 0)
+		goto cleanup;
+
+	event->retval = ret;
+	bpf_get_current_comm(&event->comm, sizeof(event->comm));
+	size_t len = EVENT_SIZE(event);
+	if (len <= sizeof(*event))
+		bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, event, len);
+cleanup:
+	bpf_map_delete_elem(&execs, &pid);
+	return 0;
+}
+
+char LICENSE[] SEC("license") = "GPL";

execsnoop-libbpf/execsnoop.c

@@ -0,0 +1,329 @@
+// Based on execsnoop(8) from BCC by Brendan Gregg and others.
+//
+#include <argp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <time.h>
+#include <unistd.h>
+#include <bpf/libbpf.h>
+#include <bpf/bpf.h>
+#include "execsnoop.h"
+#include "execsnoop.skel.h"
+#include "trace_helpers.h"
+
+#define PERF_BUFFER_PAGES   64
+#define NSEC_PRECISION (NSEC_PER_SEC / 1000)
+#define MAX_ARGS_KEY 259
+
+static struct env {
+	bool time;
+	bool timestamp;
+	bool fails;
+	uid_t uid;
+	bool quote;
+	const char *name;
+	const char *line;
+	bool print_uid;
+	bool verbose;
+	int max_args;
+} env = {
+	.max_args = DEFAULT_MAXARGS,
+	.uid = INVALID_UID
+};
+
+static struct timespec start_time;
+
+const char *argp_program_version = "execsnoop 0.1";
+const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
+const char argp_program_doc[] =
+"Trace open family syscalls\n"
+"\n"
+"USAGE: execsnoop [-h] [-T] [-t] [-x] [-u UID] [-q] [-n NAME] [-l LINE] [-U]\n"
+"                 [--max-args MAX_ARGS]\n"
+"\n"
+"EXAMPLES:\n"
+"   ./execsnoop           # trace all exec() syscalls\n"
+"   ./execsnoop -x        # include failed exec()s\n"
+"   ./execsnoop -T        # include time (HH:MM:SS)\n"
+"   ./execsnoop -U        # include UID\n"
+"   ./execsnoop -u 1000   # only trace UID 1000\n"
+"   ./execsnoop -t        # include timestamps\n"
+"   ./execsnoop -q        # add \"quotemarks\" around arguments\n"
+"   ./execsnoop -n main   # only print command lines containing \"main\"\n"
+"   ./execsnoop -l tpkg   # only print command where arguments contains \"tpkg\"";
+
+static const struct argp_option opts[] = {
+	{ "time", 'T', NULL, 0, "include time column on output (HH:MM:SS)"},
+	{ "timestamp", 't', NULL, 0, "include timestamp on output"},
+	{ "fails", 'x', NULL, 0, "include failed exec()s"},
+	{ "uid", 'u', "UID", 0, "trace this UID only"},
+	{ "quote", 'q', NULL, 0, "Add quotemarks (\") around arguments"},
+	{ "name", 'n', "NAME", 0, "only print commands matching this name, any arg"},
+	{ "line", 'l', "LINE", 0, "only print commands where arg contains this line"},
+	{ "print-uid", 'U', NULL, 0, "print UID column"},
+	{ "max-args", MAX_ARGS_KEY, "MAX_ARGS", 0,
+		"maximum number of arguments parsed and displayed, defaults to 20"},
+	{ "verbose", 'v', NULL, 0, "Verbose debug output" },
+	{},
+};
+
+static error_t parse_arg(int key, char *arg, struct argp_state *state)
+{
+	long int uid, max_args;
+
+	switch (key) {
+	case 'h':
+		argp_usage(state);
+		break;
+	case 'T':
+		env.time = true;
+		break;
+	case 't':
+		env.timestamp = true;
+		break;
+	case 'x':
+		env.fails = true;
+		break;
+	case 'u':
+		errno = 0;
+		uid = strtol(arg, NULL, 10);
+		if (errno || uid < 0 || uid >= INVALID_UID) {
+			fprintf(stderr, "Invalid UID %s\n", arg);
+			argp_usage(state);
+		}
+		env.uid = uid;
+		break;
+	case 'q':
+		env.quote = true;
+		break;
+	case 'n':
+		env.name = arg;
+		break;
+	case 'l':
+		env.line = arg;
+		break;
+	case 'U':
+		env.print_uid = true;
+		break;
+	case 'v':
+		env.verbose = true;
+		break;
+	case MAX_ARGS_KEY:
+		errno = 0;
+		max_args = strtol(arg, NULL, 10);
+		if (errno || max_args < 1 || max_args > TOTAL_MAX_ARGS) {
+			fprintf(stderr, "Invalid MAX_ARGS %s, should be in [1, %d] range\n",
+					arg, TOTAL_MAX_ARGS);
+
+			argp_usage(state);
+		}
+		env.max_args = max_args;
+		break;
+	default:
+		return ARGP_ERR_UNKNOWN;
+	}
+	return 0;
+}
+
+int libbpf_print_fn(enum libbpf_print_level level,
+		    const char *format, va_list args)
+{
+	if (level == LIBBPF_DEBUG && !env.verbose)
+		return 0;
+	return vfprintf(stderr, format, args);
+}
+
+static void time_since_start()
+{
+	long nsec, sec;
+	static struct timespec cur_time;
+	double time_diff;
+
+	clock_gettime(CLOCK_MONOTONIC, &cur_time);
+	nsec = cur_time.tv_nsec - start_time.tv_nsec;
+	sec = cur_time.tv_sec - start_time.tv_sec;
+	if (nsec < 0) {
+		nsec += NSEC_PER_SEC;
+		sec--;
+	}
+	time_diff = sec + (double)nsec / NSEC_PER_SEC;
+	printf("%-8.3f", time_diff);
+}
+
+static void inline quoted_symbol(char c) {
+	switch(c) {
+		case '"':
+			putchar('\\');
+			putchar('"');
+			break;
+		case '\t':
+			putchar('\\');
+			putchar('t');
+			break;
+		case '\n':
+			putchar('\\');
+			putchar('n');
+			break;
+		default:
+			putchar(c);
+			break;
+	}
+}
+
+static void print_args(const struct event *e, bool quote)
+{
+	int args_counter = 0;
+
+	if (env.quote)
+		putchar('"');
+
+	for (int i = 0; i < e->args_size && args_counter < e->args_count; i++) {
+		char c = e->args[i];
+		if (env.quote) {
+			if (c == '\0') {
+				args_counter++;
+				putchar('"');
+				putchar(' ');
+				if (args_counter < e->args_count) {
+					putchar('"');
+				}
+			} else {
+				quoted_symbol(c);
+			}
+		} else {
+			if (c == '\0') {
+				args_counter++;
+				putchar(' ');
+			} else {
+				putchar(c);
+			}
+		}
+	}
+	if (e->args_count == env.max_args + 1) {
+		fputs(" ...", stdout);
+	}
+}
+
+void handle_event(void *ctx, int cpu, void *data, __u32 data_sz)
+{
+	const struct event *e = data;
+	time_t t;
+	struct tm *tm;
+	char ts[32];
+
+	/* TODO: use pcre lib */
+	if (env.name && strstr(e->comm, env.name) == NULL)
+		return;
+
+	/* TODO: use pcre lib */
+	if (env.line && strstr(e->comm, env.line) == NULL)
+		return;
+
+	time(&t);
+	tm = localtime(&t);
+	strftime(ts, sizeof(ts), "%H:%M:%S", tm);
+
+	if (env.time) {
+		printf("%-8s ", ts);
+	}
+	if (env.timestamp) {
+		time_since_start();
+	}
+
+	if (env.print_uid)
+		printf("%-6d", e->uid);
+
+	printf("%-16s %-6d %-6d %3d ", e->comm, e->pid, e->ppid, e->retval);
+	print_args(e, env.quote);
+	putchar('\n');
+}
+
+void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt)
+{
+	fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
+}
+
+int main(int argc, char **argv)
+{
+	static const struct argp argp = {
+		.options = opts,
+		.parser = parse_arg,
+		.doc = argp_program_doc,
+	};
+	struct perf_buffer_opts pb_opts;
+	struct perf_buffer *pb = NULL;
+	struct execsnoop_bpf *obj;
+	int err;
+
+	err = argp_parse(&argp, argc, argv, 0, NULL, NULL);
+	if (err)
+		return err;
+
+	libbpf_set_print(libbpf_print_fn);
+
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+
+	obj = execsnoop_bpf__open();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	/* initialize global data (filtering options) */
+	obj->rodata->ignore_failed = !env.fails;
+	obj->rodata->targ_uid = env.uid;
+	obj->rodata->max_args = env.max_args;
+
+	err = execsnoop_bpf__load(obj);
+	if (err) {
+		fprintf(stderr, "failed to load BPF object: %d\n", err);
+		goto cleanup;
+	}
+
+	clock_gettime(CLOCK_MONOTONIC, &start_time);
+	err = execsnoop_bpf__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		goto cleanup;
+	}
+	/* print headers */
+	if (env.time) {
+		printf("%-9s", "TIME");
+	}
+	if (env.timestamp) {
+		printf("%-8s ", "TIME(s)");
+	}
+	if (env.print_uid) {
+		printf("%-6s ", "UID");
+	}
+
+	printf("%-16s %-6s %-6s %3s %s\n", "PCOMM", "PID", "PPID", "RET", "ARGS");
+
+	/* setup event callbacks */
+	pb_opts.sample_cb = handle_event;
+	pb_opts.lost_cb = handle_lost_events;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.events), PERF_BUFFER_PAGES, &pb_opts);
+	err = libbpf_get_error(pb);
+	if (err) {
+		pb = NULL;
+		fprintf(stderr, "failed to open perf buffer: %d\n", err);
+		goto cleanup;
+	}
+
+	/* main: poll */
+	while ((err = perf_buffer__poll(pb, 100)) >= 0)
+		;
+	printf("Error polling perf buffer: %d\n", err);
+
+cleanup:
+	perf_buffer__free(pb);
+	execsnoop_bpf__destroy(obj);
+
+	return err != 0;
+}

execsnoop-libbpf/execsnoop.h

@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __EXECSNOOP_H
+#define __EXECSNOOP_H
+
+#define ARGSIZE  128
+#define TASK_COMM_LEN 16
+#define TOTAL_MAX_ARGS 60
+#define DEFAULT_MAXARGS 20
+#define FULL_MAX_ARGS_ARR (TOTAL_MAX_ARGS * ARGSIZE)
+#define INVALID_UID ((uid_t)-1)
+#define BASE_EVENT_SIZE (size_t)(&((struct event*)0)->args)
+#define EVENT_SIZE(e) (BASE_EVENT_SIZE + e->args_size)
+#define LAST_ARG (FULL_MAX_ARGS_ARR - ARGSIZE)
+
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+	int retval;
+	int args_count;
+	unsigned int args_size;
+	char args[FULL_MAX_ARGS_ARR];
+};
+
+#endif /* __EXECSNOOP_H */

execsnoop-libbpf/execsnoop_share.cpp

@@ -0,0 +1,101 @@
+// Based on execsnoop(8) from BCC by Brendan Gregg and others.
+//
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+extern "C" {
+#define typeof(x) decltype(x)
+#include "execsnoop.h"
+#include "execsnoop.skel.h"
+#include "trace_helpers.h"
+}
+#include "execsnoop_share.h"
+
+#define PERF_BUFFER_PAGES   64
+#define NSEC_PRECISION (NSEC_PER_SEC / 1000)
+#define MAX_ARGS_KEY 259
+
+namespace CGPROXY::EXECSNOOP {
+	
+function<int(int)> callback = NULL;
+promise<void> status;
+
+void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) {
+
+  	auto e = static_cast<event*>(data);
+  	int pid = e->pid;
+  	if (callback) callback(pid);
+
+}
+
+void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) {
+	fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
+}
+
+int execsnoop() {
+	struct perf_buffer_opts pb_opts;
+	struct perf_buffer *pb = NULL;
+	struct execsnoop_bpf *obj;
+	int err;
+
+	// libbpf_set_print(libbpf_print_fn);
+
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+
+	obj = execsnoop_bpf__open();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	/* initialize global data (filtering options) */
+	obj->rodata->ignore_failed = true;
+
+	err = execsnoop_bpf__load(obj);
+	if (err) {
+		fprintf(stderr, "failed to load BPF object: %d\n", err);
+		goto cleanup;
+	}
+
+	err = execsnoop_bpf__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		goto cleanup;
+	}
+
+	/* setup event callbacks */
+	pb_opts.sample_cb = handle_event;
+	pb_opts.lost_cb = handle_lost_events;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.events), PERF_BUFFER_PAGES, &pb_opts);
+	err = libbpf_get_error(pb);
+	if (err) {
+		pb = NULL;
+		fprintf(stderr, "failed to open perf buffer: %d\n", err);
+		goto cleanup;
+	}
+
+  status.set_value();
+
+	/* main: poll */
+	while ((err = perf_buffer__poll(pb, 100)) >= 0);
+	printf("Error polling perf buffer: %d\n", err);
+
+cleanup:
+	perf_buffer__free(pb);
+	execsnoop_bpf__destroy(obj);
+
+	return err != 0;
+}
+
+
+void startThread(function<int(int)> c, promise<void> _status) {
+  status = move(_status);
+  callback = c;
+  execsnoop();
+}
+}

execsnoop-libbpf/execsnoop_share.h

@@ -0,0 +1,16 @@
+#ifndef EXECSNOOP_SHARE_HPP
+#define EXECSNOOP_SHARE_HPP 1
+
+#include <functional>
+#include <future>
+#include <string>
+using namespace std;
+
+namespace CGPROXY::EXECSNOOP {
+extern "C" void startThread(function<int(int)> c, promise<void> _status);
+// typedef void startThread_t(function<int(int)>, promise<void>);
+using startThread_t=decltype(startThread);
+startThread_t *_startThread; // only for dlsym()
+
+} // namespace CGPROXY::EXECSNOOP
+#endif

execsnoop-libbpf/libbpf-fPIC.patch

@@ -0,0 +1,21 @@
+diff --git src/Makefile src/Makefile
+index d0308c3..fcc3b6f 100644
+--- src/Makefile
++++ src/Makefile
+@@ -15,6 +15,7 @@ ifneq ($(FEATURE_REALLOCARRAY),)
+ 	ALL_CFLAGS += -DCOMPAT_NEED_REALLOCARRAY
+ endif
+ 
++STATIC_CFLAGS += -fPIC
+ SHARED_CFLAGS += -fPIC -fvisibility=hidden -DSHARED
+ 
+ CFLAGS ?= -g -O2 -Werror -Wall
+@@ -99,7 +100,7 @@ $(SHARED_OBJDIR):
+ 	mkdir -p $(SHARED_OBJDIR)
+ 
+ $(STATIC_OBJDIR)/%.o: %.c | $(STATIC_OBJDIR)
+-	$(CC) $(ALL_CFLAGS) $(CPPFLAGS) -c $< -o $@
++	$(CC) $(ALL_CFLAGS) $(STATIC_CFLAGS) $(CPPFLAGS) -c $< -o $@
+ 
+ $(SHARED_OBJDIR)/%.o: %.c | $(SHARED_OBJDIR)
+ 	$(CC) $(ALL_CFLAGS) $(SHARED_CFLAGS) $(CPPFLAGS) -c $< -o $@

execsnoop-libbpf/readme.md

@@ -0,0 +1,21 @@
+
+## Resource
+
+- [libbpf](https://github.com/libbpf/libbpf)
+- [libbpf-tools](https://github.com/iovisor/bcc/tree/master/libbpf-tools) examples use libbpf
+- [bpftool](https://www.archlinux.org/packages/community/x86_64/bpf/) to generate skeleton and dump btf
+
+## Prons
+
+- BPF CO-RE (Compile Once – Run Everywhere)
+- small memory usage
+
+## Cons
+
+- `vmlinux.h`does not contain `#define` etc. And often causes confilct with other headers to cause redifinition error
+- comment in code for different types is gone
+- need kernel built with `CONFIG_DEBUG_INFO_BTF=y`, while ubuntu 20.04 still not
+
+## Build
+
+- see *makefile*

execsnoop-libbpf/syscall_helpers.c

@@ -0,0 +1,526 @@
+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+// Copyright (c) 2020 Anton Protopopov
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+#include <stdio.h>
+#include <errno.h>
+
+static const char **syscall_names;
+static size_t syscall_names_size;
+
+#define warn(...) fprintf(stderr, __VA_ARGS__)
+#define MAX(x, y) (((x) > (y)) ? (x) : (y))
+
+static const char *parse_syscall(const char *buf, int *number)
+{
+	char *end;
+	long x;
+
+	errno = 0;
+	x = strtol(buf, &end, 10);
+	if (errno) {
+		warn("strtol(%s): %s\n", buf, strerror(errno));
+		return NULL;
+	} else if (end == buf) {
+		warn("strtol(%s): no digits found\n", buf);
+		return NULL;
+	} else if (x < 0 || x > INT_MAX) {
+		warn("strtol(%s): bad syscall number: %ld\n", buf, x);
+		return NULL;
+	}
+	if (*end != '\t') {
+		warn("bad input: %s (expected <num>\t<name>)\n", buf);
+		return NULL;
+	}
+
+	*number = x;
+	return ++end;
+}
+
+void init_syscall_names(void)
+{
+	size_t old_size, size = 1024;
+	const char *name;
+	char buf[64];
+	int number;
+	int err;
+	FILE *f;
+
+	f = popen("ausyscall --dump 2>/dev/null", "r");
+	if (!f) {
+		warn("popen: ausyscall --dump: %s\n", strerror(errno));
+		return;
+	}
+
+	syscall_names = calloc(size, sizeof(char *));
+	if (!syscall_names) {
+		warn("calloc: %s\n", strerror(errno));
+		goto close;
+	}
+
+	/* skip the header */
+	fgets(buf, sizeof(buf), f);
+
+	while (fgets(buf, sizeof(buf), f)) {
+		if (buf[strlen(buf) - 1] == '\n')
+			buf[strlen(buf) - 1] = '\0';
+
+		name = parse_syscall(buf, &number);
+		if (!name || !name[0])
+			goto close;
+
+		/* In a rare case when syscall number is > than initial 1024 */
+		if (number >= size) {
+			old_size = size;
+			size = 1024 * (1 + number / 1024);
+			syscall_names = realloc(syscall_names,
+						size * sizeof(char *));
+			if (!syscall_names) {
+				warn("realloc: %s\n", strerror(errno));
+				goto close;
+			}
+			memset(syscall_names+old_size, 0,
+			       (size - old_size) * sizeof(char *));
+		}
+
+		if (syscall_names[number]) {
+			warn("duplicate number: %d (stored: %s)",
+				number, syscall_names[number]);
+			goto close;
+		}
+
+		syscall_names[number] = strdup(name);
+		if (!syscall_names[number]) {
+			warn("strdup: %s\n", strerror(errno));
+			goto close;
+		}
+		syscall_names_size = MAX(number+1, syscall_names_size);
+	}
+
+	if (ferror(f))
+		warn("fgets: %s\n", strerror(errno));
+close:
+	err = pclose(f);
+	if (err < 0)
+		warn("pclose: %s\n", strerror(errno));
+#ifndef __x86_64__
+	/* Ignore the error for x86_64 where we have a table compiled in */
+	else if (err && WEXITSTATUS(err) == 127) {
+		warn("ausyscall required for syscalls number/name mapping\n");
+	} else if (err) {
+		warn("ausyscall exit status (see wait(2)): 0x%x\n", err);
+	}
+#endif
+}
+
+void free_syscall_names(void)
+{
+	for (size_t i = 0; i < syscall_names_size; i++)
+		free((void *) syscall_names[i]);
+	free(syscall_names);
+}
+
+/*
+ * Syscall table for Linux x86_64.
+ *
+ * Semi-automatically generated from strace/linux/x86_64/syscallent.h and
+ * linux/syscallent-common.h using the following commands:
+ *
+ *  awk -F\" '/SEN/{printf("%d %s\n", substr($0,2,3), $(NF-1));}' syscallent.h
+ *  awk '/SEN/ { printf("%d %s\n", $3, $9); }' syscallent-common.h
+ *
+ * (The idea is taken from src/python/bcc/syscall.py.)
+ */
+#ifdef __x86_64__
+static const char *syscall_names_x86_64[] = {
+	[0] = "read",
+	[1] = "write",
+	[2] = "open",
+	[3] = "close",
+	[4] = "stat",
+	[5] = "fstat",
+	[6] = "lstat",
+	[7] = "poll",
+	[8] = "lseek",
+	[9] = "mmap",
+	[10] = "mprotect",
+	[11] = "munmap",
+	[12] = "brk",
+	[13] = "rt_sigaction",
+	[14] = "rt_sigprocmask",
+	[15] = "rt_sigreturn",
+	[16] = "ioctl",
+	[17] = "pread64",
+	[18] = "pwrite64",
+	[19] = "readv",
+	[20] = "writev",
+	[21] = "access",
+	[22] = "pipe",
+	[23] = "select",
+	[24] = "sched_yield",
+	[25] = "mremap",
+	[26] = "msync",
+	[27] = "mincore",
+	[28] = "madvise",
+	[29] = "shmget",
+	[30] = "shmat",
+	[31] = "shmctl",
+	[32] = "dup",
+	[33] = "dup2",
+	[34] = "pause",
+	[35] = "nanosleep",
+	[36] = "getitimer",
+	[37] = "alarm",
+	[38] = "setitimer",
+	[39] = "getpid",
+	[40] = "sendfile",
+	[41] = "socket",
+	[42] = "connect",
+	[43] = "accept",
+	[44] = "sendto",
+	[45] = "recvfrom",
+	[46] = "sendmsg",
+	[47] = "recvmsg",
+	[48] = "shutdown",
+	[49] = "bind",
+	[50] = "listen",
+	[51] = "getsockname",
+	[52] = "getpeername",
+	[53] = "socketpair",
+	[54] = "setsockopt",
+	[55] = "getsockopt",
+	[56] = "clone",
+	[57] = "fork",
+	[58] = "vfork",
+	[59] = "execve",
+	[60] = "exit",
+	[61] = "wait4",
+	[62] = "kill",
+	[63] = "uname",
+	[64] = "semget",
+	[65] = "semop",
+	[66] = "semctl",
+	[67] = "shmdt",
+	[68] = "msgget",
+	[69] = "msgsnd",
+	[70] = "msgrcv",
+	[71] = "msgctl",
+	[72] = "fcntl",
+	[73] = "flock",
+	[74] = "fsync",
+	[75] = "fdatasync",
+	[76] = "truncate",
+	[77] = "ftruncate",
+	[78] = "getdents",
+	[79] = "getcwd",
+	[80] = "chdir",
+	[81] = "fchdir",
+	[82] = "rename",
+	[83] = "mkdir",
+	[84] = "rmdir",
+	[85] = "creat",
+	[86] = "link",
+	[87] = "unlink",
+	[88] = "symlink",
+	[89] = "readlink",
+	[90] = "chmod",
+	[91] = "fchmod",
+	[92] = "chown",
+	[93] = "fchown",
+	[94] = "lchown",
+	[95] = "umask",
+	[96] = "gettimeofday",
+	[97] = "getrlimit",
+	[98] = "getrusage",
+	[99] = "sysinfo",
+	[100] = "times",
+	[101] = "ptrace",
+	[102] = "getuid",
+	[103] = "syslog",
+	[104] = "getgid",
+	[105] = "setuid",
+	[106] = "setgid",
+	[107] = "geteuid",
+	[108] = "getegid",
+	[109] = "setpgid",
+	[110] = "getppid",
+	[111] = "getpgrp",
+	[112] = "setsid",
+	[113] = "setreuid",
+	[114] = "setregid",
+	[115] = "getgroups",
+	[116] = "setgroups",
+	[117] = "setresuid",
+	[118] = "getresuid",
+	[119] = "setresgid",
+	[120] = "getresgid",
+	[121] = "getpgid",
+	[122] = "setfsuid",
+	[123] = "setfsgid",
+	[124] = "getsid",
+	[125] = "capget",
+	[126] = "capset",
+	[127] = "rt_sigpending",
+	[128] = "rt_sigtimedwait",
+	[129] = "rt_sigqueueinfo",
+	[130] = "rt_sigsuspend",
+	[131] = "sigaltstack",
+	[132] = "utime",
+	[133] = "mknod",
+	[134] = "uselib",
+	[135] = "personality",
+	[136] = "ustat",
+	[137] = "statfs",
+	[138] = "fstatfs",
+	[139] = "sysfs",
+	[140] = "getpriority",
+	[141] = "setpriority",
+	[142] = "sched_setparam",
+	[143] = "sched_getparam",
+	[144] = "sched_setscheduler",
+	[145] = "sched_getscheduler",
+	[146] = "sched_get_priority_max",
+	[147] = "sched_get_priority_min",
+	[148] = "sched_rr_get_interval",
+	[149] = "mlock",
+	[150] = "munlock",
+	[151] = "mlockall",
+	[152] = "munlockall",
+	[153] = "vhangup",
+	[154] = "modify_ldt",
+	[155] = "pivot_root",
+	[156] = "_sysctl",
+	[157] = "prctl",
+	[158] = "arch_prctl",
+	[159] = "adjtimex",
+	[160] = "setrlimit",
+	[161] = "chroot",
+	[162] = "sync",
+	[163] = "acct",
+	[164] = "settimeofday",
+	[165] = "mount",
+	[166] = "umount2",
+	[167] = "swapon",
+	[168] = "swapoff",
+	[169] = "reboot",
+	[170] = "sethostname",
+	[171] = "setdomainname",
+	[172] = "iopl",
+	[173] = "ioperm",
+	[174] = "create_module",
+	[175] = "init_module",
+	[176] = "delete_module",
+	[177] = "get_kernel_syms",
+	[178] = "query_module",
+	[179] = "quotactl",
+	[180] = "nfsservctl",
+	[181] = "getpmsg",
+	[182] = "putpmsg",
+	[183] = "afs_syscall",
+	[184] = "tuxcall",
+	[185] = "security",
+	[186] = "gettid",
+	[187] = "readahead",
+	[188] = "setxattr",
+	[189] = "lsetxattr",
+	[190] = "fsetxattr",
+	[191] = "getxattr",
+	[192] = "lgetxattr",
+	[193] = "fgetxattr",
+	[194] = "listxattr",
+	[195] = "llistxattr",
+	[196] = "flistxattr",
+	[197] = "removexattr",
+	[198] = "lremovexattr",
+	[199] = "fremovexattr",
+	[200] = "tkill",
+	[201] = "time",
+	[202] = "futex",
+	[203] = "sched_setaffinity",
+	[204] = "sched_getaffinity",
+	[205] = "set_thread_area",
+	[206] = "io_setup",
+	[207] = "io_destroy",
+	[208] = "io_getevents",
+	[209] = "io_submit",
+	[210] = "io_cancel",
+	[211] = "get_thread_area",
+	[212] = "lookup_dcookie",
+	[213] = "epoll_create",
+	[214] = "epoll_ctl_old",
+	[215] = "epoll_wait_old",
+	[216] = "remap_file_pages",
+	[217] = "getdents64",
+	[218] = "set_tid_address",
+	[219] = "restart_syscall",
+	[220] = "semtimedop",
+	[221] = "fadvise64",
+	[222] = "timer_create",
+	[223] = "timer_settime",
+	[224] = "timer_gettime",
+	[225] = "timer_getoverrun",
+	[226] = "timer_delete",
+	[227] = "clock_settime",
+	[228] = "clock_gettime",
+	[229] = "clock_getres",
+	[230] = "clock_nanosleep",
+	[231] = "exit_group",
+	[232] = "epoll_wait",
+	[233] = "epoll_ctl",
+	[234] = "tgkill",
+	[235] = "utimes",
+	[236] = "vserver",
+	[237] = "mbind",
+	[238] = "set_mempolicy",
+	[239] = "get_mempolicy",
+	[240] = "mq_open",
+	[241] = "mq_unlink",
+	[242] = "mq_timedsend",
+	[243] = "mq_timedreceive",
+	[244] = "mq_notify",
+	[245] = "mq_getsetattr",
+	[246] = "kexec_load",
+	[247] = "waitid",
+	[248] = "add_key",
+	[249] = "request_key",
+	[250] = "keyctl",
+	[251] = "ioprio_set",
+	[252] = "ioprio_get",
+	[253] = "inotify_init",
+	[254] = "inotify_add_watch",
+	[255] = "inotify_rm_watch",
+	[256] = "migrate_pages",
+	[257] = "openat",
+	[258] = "mkdirat",
+	[259] = "mknodat",
+	[260] = "fchownat",
+	[261] = "futimesat",
+	[262] = "newfstatat",
+	[263] = "unlinkat",
+	[264] = "renameat",
+	[265] = "linkat",
+	[266] = "symlinkat",
+	[267] = "readlinkat",
+	[268] = "fchmodat",
+	[269] = "faccessat",
+	[270] = "pselect6",
+	[271] = "ppoll",
+	[272] = "unshare",
+	[273] = "set_robust_list",
+	[274] = "get_robust_list",
+	[275] = "splice",
+	[276] = "tee",
+	[277] = "sync_file_range",
+	[278] = "vmsplice",
+	[279] = "move_pages",
+	[280] = "utimensat",
+	[281] = "epoll_pwait",
+	[282] = "signalfd",
+	[283] = "timerfd_create",
+	[284] = "eventfd",
+	[285] = "fallocate",
+	[286] = "timerfd_settime",
+	[287] = "timerfd_gettime",
+	[288] = "accept4",
+	[289] = "signalfd4",
+	[290] = "eventfd2",
+	[291] = "epoll_create1",
+	[292] = "dup3",
+	[293] = "pipe2",
+	[294] = "inotify_init1",
+	[295] = "preadv",
+	[296] = "pwritev",
+	[297] = "rt_tgsigqueueinfo",
+	[298] = "perf_event_open",
+	[299] = "recvmmsg",
+	[300] = "fanotify_init",
+	[301] = "fanotify_mark",
+	[302] = "prlimit64",
+	[303] = "name_to_handle_at",
+	[304] = "open_by_handle_at",
+	[305] = "clock_adjtime",
+	[306] = "syncfs",
+	[307] = "sendmmsg",
+	[308] = "setns",
+	[309] = "getcpu",
+	[310] = "process_vm_readv",
+	[311] = "process_vm_writev",
+	[312] = "kcmp",
+	[313] = "finit_module",
+	[314] = "sched_setattr",
+	[315] = "sched_getattr",
+	[316] = "renameat2",
+	[317] = "seccomp",
+	[318] = "getrandom",
+	[319] = "memfd_create",
+	[320] = "kexec_file_load",
+	[321] = "bpf",
+	[322] = "execveat",
+	[323] = "userfaultfd",
+	[324] = "membarrier",
+	[325] = "mlock2",
+	[326] = "copy_file_range",
+	[327] = "preadv2",
+	[328] = "pwritev2",
+	[329] = "pkey_mprotect",
+	[330] = "pkey_alloc",
+	[331] = "pkey_free",
+	[332] = "statx",
+	[333] = "io_pgetevents",
+	[334] = "rseq",
+	[424] = "pidfd_send_signal",
+	[425] = "io_uring_setup",
+	[426] = "io_uring_enter",
+	[427] = "io_uring_register",
+	[428] = "open_tree",
+	[429] = "move_mount",
+	[430] = "fsopen",
+	[431] = "fsconfig",
+	[432] = "fsmount",
+	[433] = "fspick",
+	[434] = "pidfd_open",
+	[435] = "clone3",
+	[437] = "openat2",
+	[438] = "pidfd_getfd",
+};
+size_t syscall_names_x86_64_size = sizeof(syscall_names_x86_64)/sizeof(char*);
+#endif
+
+void syscall_name(unsigned n, char *buf, size_t size)
+{
+	const char *name = NULL;
+
+	if (n < syscall_names_size)
+		name = syscall_names[n];
+#ifdef __x86_64__
+	else if (n < syscall_names_x86_64_size)
+		name = syscall_names_x86_64[n];
+#endif
+
+	if (name)
+		strncpy(buf, name, size-1);
+	else
+		snprintf(buf, size, "[unknown: %u]", n);
+}
+
+int list_syscalls(void)
+{
+	const char **list = syscall_names;
+	size_t size = syscall_names_size;
+
+#ifdef __x86_64__
+	if (!size) {
+		size = syscall_names_x86_64_size;
+		list = syscall_names_x86_64;
+	}
+#endif
+
+	for (size_t i = 0; i < size; i++) {
+		if (list[i])
+			printf("%3zd: %s\n", i, list[i]);
+	}
+
+	return (!list || !size);
+}
+

execsnoop-libbpf/syscall_helpers.h

@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __SYSCALL_HELPERS_H
+#define __SYSCALL_HELPERS_H
+
+#include <stddef.h>
+
+void init_syscall_names(void);
+void free_syscall_names(void);
+void list_syscalls(void);
+void syscall_name(unsigned n, char *buf, size_t size);
+
+#endif /* __SYSCALL_HELPERS_H */

execsnoop-libbpf/trace_helpers.c

@@ -0,0 +1,234 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+#include <sys/resource.h>
+#include <time.h>
+#include "trace_helpers.h"
+
+#define min(x, y) ({				 \
+	typeof(x) _min1 = (x);			 \
+	typeof(y) _min2 = (y);			 \
+	(void) (&_min1 == &_min2);		 \
+	_min1 < _min2 ? _min1 : _min2; })
+
+struct ksyms {
+	struct ksym *syms;
+	int syms_sz;
+	int syms_cap;
+	char *strs;
+	int strs_sz;
+	int strs_cap;
+};
+
+static int ksyms__add_symbol(struct ksyms *ksyms, const char *name, unsigned long addr)
+{
+	size_t new_cap, name_len = strlen(name) + 1;
+	struct ksym *ksym;
+	void *tmp;
+
+	if (ksyms->strs_sz + name_len > ksyms->strs_cap) {
+		new_cap = ksyms->strs_cap * 4 / 3;
+		if (new_cap < ksyms->strs_sz + name_len)
+			new_cap = ksyms->strs_sz + name_len;
+		if (new_cap < 1024)
+			new_cap = 1024;
+		tmp = realloc(ksyms->strs, new_cap);
+		if (!tmp)
+			return -1;
+		ksyms->strs = tmp;
+		ksyms->strs_cap = new_cap;
+	}
+	if (ksyms->syms_sz + 1 > ksyms->syms_cap) {
+		new_cap = ksyms->syms_cap * 4 / 3;
+		if (new_cap < 1024)
+			new_cap = 1024;
+		tmp = realloc(ksyms->syms, sizeof(*ksyms->syms) * new_cap);
+		if (!tmp)
+			return -1;
+		ksyms->syms = tmp;
+		ksyms->syms_cap = new_cap;
+	}
+
+	ksym = &ksyms->syms[ksyms->syms_sz];
+	/* while constructing, re-use pointer as just a plain offset */
+	ksym->name = (void *)(unsigned long)ksyms->strs_sz;
+	ksym->addr = addr;
+
+	memcpy(ksyms->strs + ksyms->strs_sz, name, name_len);
+	ksyms->strs_sz += name_len;
+	ksyms->syms_sz++;
+
+	return 0;
+}
+
+static int ksym_cmp(const void *p1, const void *p2)
+{
+	const struct ksym *s1 = p1, *s2 = p2;
+
+	if (s1->addr == s2->addr)
+		return strcmp(s1->name, s2->name);
+	return s1->addr < s2->addr ? -1 : 1;
+}
+
+struct ksyms *ksyms__load(void)
+{
+	char sym_type, sym_name[256];
+	struct ksyms *ksyms;
+	unsigned long sym_addr;
+	int i, ret;
+	FILE *f;
+
+	f = fopen("/proc/kallsyms", "r");
+	if (!f)
+		return NULL;
+
+	ksyms = calloc(1, sizeof(*ksyms));
+	if (!ksyms)
+		goto err_out;
+
+	while (true) {
+		ret = fscanf(f, "%lx %c %s%*[^\n]\n",
+			     &sym_addr, &sym_type, sym_name);
+		if (ret == EOF && feof(f))
+			break;
+		if (ret != 3)
+			goto err_out;
+		if (ksyms__add_symbol(ksyms, sym_name, sym_addr))
+			goto err_out;
+	}
+
+	/* now when strings are finalized, adjust pointers properly */
+	for (i = 0; i < ksyms->syms_sz; i++)
+		ksyms->syms[i].name += (unsigned long)ksyms->strs;
+
+	qsort(ksyms->syms, ksyms->syms_sz, sizeof(*ksyms->syms), ksym_cmp);
+
+	fclose(f);
+	return ksyms;
+
+err_out:
+	ksyms__free(ksyms);
+	fclose(f);
+	return NULL;
+}
+
+void ksyms__free(struct ksyms *ksyms)
+{
+	if (!ksyms)
+		return;
+
+	free(ksyms->syms);
+	free(ksyms->strs);
+	free(ksyms);
+}
+
+const struct ksym *ksyms__map_addr(const struct ksyms *ksyms,
+				   unsigned long addr)
+{
+	int start = 0, end = ksyms->syms_sz - 1, mid;
+	unsigned long sym_addr;
+
+	/* find largest sym_addr <= addr using binary search */
+	while (start < end) {
+		mid = start + (end - start + 1) / 2;
+		sym_addr = ksyms->syms[mid].addr;
+
+		if (sym_addr <= addr)
+			start = mid;
+		else
+			end = mid - 1;
+	}
+
+	if (start == end && ksyms->syms[start].addr <= addr)
+		return &ksyms->syms[start];
+	return NULL;
+}
+
+const struct ksym *ksyms__get_symbol(const struct ksyms *ksyms,
+				     const char *name)
+{
+	int i;
+
+	for (i = 0; i < ksyms->syms_sz; i++) {
+		if (strcmp(ksyms->syms[i].name, name) == 0)
+			return &ksyms->syms[i];
+	}
+
+	return NULL;
+}
+
+static void print_stars(unsigned int val, unsigned int val_max, int width)
+{
+	int num_stars, num_spaces, i;
+	bool need_plus;
+
+	num_stars = min(val, val_max) * width / val_max;
+	num_spaces = width - num_stars;
+	need_plus = val > val_max;
+
+	for (i = 0; i < num_stars; i++)
+		printf("*");
+	for (i = 0; i < num_spaces; i++)
+		printf(" ");
+	if (need_plus)
+		printf("+");
+}
+
+void print_log2_hist(unsigned int *vals, int vals_size, char *val_type)
+{
+	int stars_max = 40, idx_max = -1;
+	unsigned int val, val_max = 0;
+	unsigned long long low, high;
+	int stars, width, i;
+
+	for (i = 0; i < vals_size; i++) {
+		val = vals[i];
+		if (val > 0)
+			idx_max = i;
+		if (val > val_max)
+			val_max = val;
+	}
+
+	if (idx_max < 0)
+		return;
+
+	printf("%*s%-*s : count    distribution\n", idx_max <= 32 ? 5 : 15, "",
+		idx_max <= 32 ? 19 : 29, val_type);
+
+	if (idx_max <= 32)
+		stars = stars_max;
+	else
+		stars = stars_max / 2;
+
+	for (i = 0; i <= idx_max; i++) {
+		low = (1ULL << (i + 1)) >> 1;
+		high = (1ULL << (i + 1)) - 1;
+		if (low == high)
+			low -= 1;
+		val = vals[i];
+		width = idx_max <= 32 ? 10 : 20;
+		printf("%*lld -> %-*lld : %-8d |", width, low, width, high, val);
+		print_stars(val, val_max, stars);
+		printf("|\n");
+	}
+}
+
+unsigned long long get_ktime_ns(void)
+{
+	struct timespec ts;
+
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+	return ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec;
+}
+
+int bump_memlock_rlimit(void)
+{
+	struct rlimit rlim_new = {
+		.rlim_cur	= RLIM_INFINITY,
+		.rlim_max	= RLIM_INFINITY,
+	};
+
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}

execsnoop-libbpf/trace_helpers.h

@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __TRACE_HELPERS_H
+#define __TRACE_HELPERS_H
+
+#define NSEC_PER_SEC		1000000000ULL
+
+struct ksym {
+	const char *name;
+	unsigned long addr;
+};
+
+struct ksyms;
+
+struct ksyms *ksyms__load(void);
+void ksyms__free(struct ksyms *ksyms);
+const struct ksym *ksyms__map_addr(const struct ksyms *ksyms,
+				   unsigned long addr);
+const struct ksym *ksyms__get_symbol(const struct ksyms *ksyms,
+				     const char *name);
+
+void print_log2_hist(unsigned int *vals, int vals_size, char *val_type);
+
+unsigned long long get_ktime_ns(void);
+int bump_memlock_rlimit(void);
+
+#endif /* __TRACE_HELPERS_H */

man/cgnoproxy.1

@@ -0,0 +1,16 @@
+.\" Manpage for cgproxyd
+.TH man 1 "19 May 2020" "1.0" "cgnoproxy man page"
+.SH NAME
+cgnoproxy \- Run program without proxy
+.SH SYNOPSIS
+cgnoproxy --help
+cgnoproxy [--debug] <CMD>
+cgnoproxy [--debug] --pid <PID>
+.SH ALIAS
+cgnoproxy = cgproxy --noproxy
+.SH DESCRIPTION
+cgnoproxy send current running process pid or specified pid to cgproxyd through unix socket, then pid is attached to non-proxied cgroup 
+.SH EXAMPLES
+cgnoproxy sudo v2ray -config config_file
+.SH SEE ALSO
+cgproxyd(1), cgproxy(1), cgnoproxy(1)

man/cgproxy.1

@@ -0,0 +1,14 @@
+.\" Manpage for cgproxyd
+.TH man 1 "19 May 2020" "1.0" "cgproxy man page"
+.SH NAME
+cgproxy \- Run program with proxy
+.SH SYNOPSIS
+cgproxy --help
+cgproxy [--debug] <CMD>
+cgproxy [--debug] --pid <PID>
+.SH DESCRIPTION
+cgproxy send current running process pid or specified pid to cgproxyd through unix socket, then pid is attached to proxied cgroup 
+.SH EXAMPLES
+cgproxy curl -vI https://www.google.com
+.SH SEE ALSO
+cgproxyd(1), cgproxy(1), cgnoproxy(1)

man/cgproxyd.1

@@ -0,0 +1,54 @@
+.\" Manpage for cgproxyd
+.TH man 1 "19 May 2020" "1.0" "cgproxyd man page"
+.SH NAME
+cgproxyd \- Start a daemon with unix socket to accept control from cgproxy/cgnoproxy
+.SH SYNOPSIS
+cgproxyd [--help] [--debug] [--execsnoop]
+.SH ALIAS
+cgproxyd = cgproxy --daemon
+.SH OPTIONS
+.B  --execsnoop
+enable execsnoop to support program level proxy, need bcc installed to actually work
+.SH CONFIGURATION
+.I /etc/cgproxy/config.json
+.br
+.B port 
+tproxy listenning port
+.br
+program level proxy controll, need `bcc` installed to work:
+.br
+.RS
+.B program_proxy
+program need to be proxied
+.br
+.B program_noproxy
+program that won't be proxied
+.RE
+.br
+cgroup level proxy control:
+.br
+.RS
+.B cgroup_noproxy
+cgroup array that no need to proxy, /noproxy.slice is preserved.
+.br
+.B cgroup_proxy
+cgroup array that need to proxy, /proxy.slice is preserved.
+.RE
+.br
+.B enable_gateway
+enable gateway proxy for local devices.
+.br
+.B enable_dns
+enable dns to go to proxy.
+.br
+.B enable_tcp
+.br
+.B enable_udp
+.br
+.B enable_ipv4 
+.br
+.B enable_ipv6
+.br
+.SH SEE ALSO
+cgproxyd(1), cgproxy(1), cgnoproxy(1)
+

pack/CMakeLists.txt

@@ -0,0 +1,24 @@
+## package for deb and rpm
+set(CPACK_GENERATOR "DEB;RPM")
+set(CPACK_PACKAGE_NAME "cgproxy")
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "cgproxy will transparent proxy anything running in specific cgroup.It aslo supports global transparent proxy and gateway proxy")
+
+## deb pack
+set(CPACK_DEBIAN_PACKAGE_NAME "cgproxy")
+set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "amd64")
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "systemd")
+set(CPACK_DEBIAN_PACKAGE_SECTION "network")
+set(CPACK_DEBIAN_PACKAGE_PRIORITY "Optional")
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/springzfx/cgproxy")
+set(CPACK_DEBIAN_PACKAGE_MAINTAINER "springzfx@gmail.com")
+set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/postinst;${CMAKE_CURRENT_SOURCE_DIR}/prerm")
+
+## rpm pack
+set(CPACK_RPM_PACKAGE_ARCHITECTURE, "x86_64")
+set(CPACK_RPM_PACKAGE_REQUIRES "systemd")
+set(CPACK_RPM_PACKAGE_GROUP "network")
+set(CPACK_RPM_PACKAGE_URL "https://github.com/springzfx/cgproxy")
+set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/postinst")
+set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/prerm")
+
+include(CPack)

readme.md

@@ -1,30 +1,37 @@
 
 
-# Transparent Proxy with cgroup v2
+# Transparent Proxy powered by cgroup v2
 
 
 
 ## Introduction
 
-cgproxy will transparent proxy anything running in specific cgroup. It resembles with *proxychains* and *tsock*, but without their disadvantages, and more powerfull.
+cgproxy will transparent proxy anything running in specific cgroup. It resembles with *proxychains* and *tsock*s in default setting. 
 
-It aslo supports global transparent proxy and gateway proxy. See [Global transparent proxy](#global-transparent-proxy) and  [Gateway proxy](#gateway-proxy).
+Main feature:
+
+- supports cgroup/program level proxy control. 
+- supports global transparent proxy and gateway proxy. 
+
+## Contents
 
 <!--ts-->
-
-   * [Transparent Proxy with cgroup v2](#transparent-proxy-with-cgroup-v2)
+   * [Transparent Proxy powered by cgroup v2](#transparent-proxy-powered-by-cgroup-v2)
       * [Introduction](#introduction)
+      * [Contents](#contents)
       * [Prerequest](#prerequest)
       * [How to install](#how-to-install)
-      * [How to use](#how-to-use)
+      * [Default usage](#default-usage)
+      * [Configuration](#configuration)
       * [Global transparent proxy](#global-transparent-proxy)
       * [Gateway proxy](#gateway-proxy)
       * [Other useful tools provided in this project](#other-useful-tools-provided-in-this-project)
       * [NOTES](#notes)
       * [TIPS](#tips)
       * [Licences](#licences)
+      * [Known Issus](#known-issus)
 
-<!-- Added by: fancy, at: Thu 23 Apr 2020 01:23:57 PM HKT -->
+<!-- Added by: fancy, at: Sat 04 Jul 2020 03:52:07 PM CST -->
 
 <!--te-->
 
@@ -40,17 +47,42 @@ It aslo supports global transparent proxy and gateway proxy. See [Global transpa
 
   A process listening on port (e.g.  12345)  to accept iptables TPROXY, for example v2ray's dokodemo-door in tproxy mode.
 
-## How to install
+- Iptables
+
+  Iptables version should be at least 1.6.0, run `iptables --version` to check.
+
+  ubuntu 16.04, debian 9, fedora 27 and later are desired
+
+## How to build and install
+
+### distro install
+
+- For debian and redhat series, download from [Release page](https://github.com/springzfx/cgproxy/releases)
+
+- For archlinux series, see[archlinux AUR](https://aur.archlinux.org/packages/?K=cgproxy)
+
+- **Tested on  archlinux, fedora 32, ubuntu 18.04, ubuntu 20.04,  deepin 15.11, deepin v20 beta**
+
+### build
+
+- before build, install depencies: clang, nlohmann-json, libbpf
+- then cmake standard build
 
 ```bash
-mkdir build && cd build && cmake .. && make && sudo make install
+# ready build dir
+mkdir build
+cd build
+# generate
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=/usr \
+      -Dbuild_execsnoop_dl=ON \
+      -Dbuild_static=OFF \
+      ..
+# compile
+make
 ```
 
-- It is alreay in [archlinux AUR](https://aur.archlinux.org/packages/cgproxy-git/). 
-
-- DEB and RPM are packaged in [release page](https://github.com/springzfx/cgproxy/releases).
-
-## How to use
+## Default usage
 
 - First enable and start service
 
@@ -60,57 +92,81 @@ mkdir build && cd build && cmake .. && make && sudo make install
   
 - Then prefix with cgproxy with your command, just like proxychains
 
-  ```
-  cgproxy <CMD>
+  ```bash
+  cgproxy [--debug] <CMD>
   ```
 
 - For example, test proxy
 
   ```bash
-  cgproxy curl -vIs https://www.google.com
+  cgproxy curl -vI https://www.google.com
   ```
 
 - To completely stop
   ```
   sudo systemctl disable --now cgproxy.service
   ```
-----
-<details>
-  <summary>More config in <i>/etc/cgproxy.conf</i>  (click to expand)</summary>
 
-```bash
-########################################################################
-## cgroup transparent proxy
-## any process in cgroup_proxy will be proxied, and cgroup_noproxy the opposite
-## cgroup must start with slash '/'
-# cgroup_proxy="/"
-# cgroup_noproxy="/system.slice/v2ray.service"
-cgroup_proxy="/proxy.slice"
-cgroup_noproxy="/noproxy.slice"
+## Configuration
 
-########################################################################
-## allow as gateway for local network
-enable_gateway=false
+Config file: **/etc/cgproxy/config.json**
 
-########################################################################
-## listening port of another proxy process, for example v2ray 
-port=12345
+```json
+{
+    "port": 12345,
+    "program_noproxy": ["v2ray", "qv2ray"],
+    "program_proxy": [],
+    "cgroup_noproxy": ["/system.slice/v2ray.service"],
+    "cgroup_proxy": [],
+    "enable_gateway": false,
+    "enable_dns": true,
+    "enable_udp": true,
+    "enable_tcp": true,
+    "enable_ipv4": true,
+    "enable_ipv6": true,
+    "table": 10007,
+    "fwmark": 39283,
+    "mark_newin": 39271
+}
 
-########################################################################
-## if you set to false, it's traffic won't go through proxy, but still can go direct to internet 
-enable_dns=true
-enable_tcp=true
-enable_udp=true
-enable_ipv4=true
-enable_ipv6=true
-
-########################################################################
-## do not modify this if you don't known what you are doing
-table=100
-fwmark=0x01
-mark_newin=0x02
 ```
-</details>
+
+- **port** tproxy listenning port
+
+- program level proxy control:
+
+  - **program_proxy**  program need to be proxied
+  - **program_noproxy** program that won't be proxied
+
+- cgroup level proxy control:
+
+  - **cgroup_noproxy** cgroup array that no need to proxy, `/noproxy.slice` is preserved
+  - **cgroup_proxy** cgroup array that need to proxy, `/proxy.slice` is preserved
+
+- **enable_gateway** enable gateway proxy for local devices
+
+- **enable_dns** enable dns to go to proxy
+
+- **enable_tcp**
+
+- **enable_udp**
+
+- **enable_ipv4**
+
+- **enable_ipv6**
+
+- **table**, **fwmark**, **mark_newin**  you can specify iptables and route table related parameter in case conflict.
+
+- options priority
+
+  ```
+  program_noproxy > program_proxy > cgroup_noproxy > cgroup_proxy
+  enable_ipv6 > enable_ipv4 > enable_tcp > enable_udp > enable_dns
+  ommand cgproxy and cgnoproxy always have highest priority
+  ```
+
+**Note**: cgroup in configuration need to be exist, otherwise ignored
+
 If you changed config, remember to restart service
 
 ```bash
@@ -119,58 +175,44 @@ sudo systemctl restart cgproxy.service
 
 ## Global transparent proxy
 
-- Set `cgroup_proxy="/"`  in */etc/cgproxy.conf*, this will proxy all connection
+- Set `"cgroup_proxy":["/"]`  in configuration, this will proxy all connection
 
-- And allow your proxy program (v2ray) direct to internet, two ways:
-  - active way
-
-      run `cgnoproxy <PROXY PROGRAM>`
-      
-      example: `cgnoproxy sudo v2ray -config config_file`
-      
-  - passive way
+- Allow your proxy program (v2ray) direct to internet to avoid loop. Two ways:
   
-      set `cgroup_noproxy="<PROXY PROGRAM's CGROUP>"`
+  - active way, run command
+    
+    example: `cgnoproxy sudo v2ray -config config_file`
+    
+    example: `cgnoproxy qv2ray`
+    
+  - passive way,  persistent config
   
-      example:  `cgroup_noproxy="/system.slice/v2ray.service"`
+      example:  `"program_noproxy":["v2ray" ,"qv2ray"]`
+      
+      example:  `"cgroup_noproxy":["/system.slice/v2ray.service"]`
   
 - Finally, restart cgproxy service, that's all
 
 ## Gateway proxy
 
-- Set `enable_gateway=true` in */etc/cgproxy.conf*
-- And allow your proxy software (v2ray) direct to internet, described above
-- Other device set this host as gateway, and set public dns if necessary
+- Set `"enable_gateway":true` in configuration
+- And allow your proxy software (v2ray) direct to internet if necessary, described above
+- Other device set this host as gateway, and set public dns if need
 
 ## Other useful tools provided in this project
 
 - `cgnoproxy` run program wihout proxy, very useful in global transparent proxy
 
   ```bash
-  cgnoproxy <CMD> 
+  cgnoproxy [--debug] <CMD>
+  cgnoproxy [--debug] --pid <PID>
   ```
   
-- `run_in_cgroup` run command in specific cgroup which will create if not exist , cgroup can be only one level down exist cgroup, otherwise created fail.
-
-  ```bash
-  run_in_cgroup --cgroup=CGROUP <COMMAND>
-  # example
-  run_in_cgroup --cgroup=/mycgroup.slice ping 127.0.0.1
-  ```
-  
-- `cgattach` attach specific process pid to specific cgroup which will create if not exist , cgroup can be only one level down exist cgroup, otherwise created fail.
-
-  ```bash
-  cgattch <pid> <cgroup>
-  # example
-  cgattch 9999 /proxy.slice
-  ```
+- For more detail command usage, see `man cgproxyd`  `man cgproxy`  `man cgnoproxy` 
 
 ## NOTES
 
-- `cgattach` has *suid* bit set by default, be careful to use on multi-user server for securiry. To avoid this situation,  you can remove the *suid* bit , then it will fallback to use *sudo*, with *sudoer* you can restrict permission or set NOPASSWD for youself.
-
-- v2ray TPROXY need root or special permission
+- v2ray TPROXY need root or special permission, use [service](https://github.com/springzfx/cgproxy/blob/v3.x/v2ray_config/v2ray.service) or
   
   ```bash
   sudo setcap "cap_net_admin,cap_net_bind_service=ep" /usr/lib/v2ray/v2ray
@@ -186,8 +228,12 @@ sudo systemctl restart cgproxy.service
 - Offer you qv2ray config example
   
 
-![Qv2ray config example](https://i.loli.net/2020/04/28/bdQBzUD37FOgfvt.png)
+![Qv2ray config example](https://i.loli.net/2020/06/12/6ltax93bv7NCTHU.png)
 
 ## Licences
 
 cgproxy is licenced under [![License: GPL v3](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/gpl-2.0) 
+
+## Known Issus
+
+- docker breaks cgroup path match, add kernel parameter `cgroup_no_v1=net_cls,net_prio` to resolve, see [issue #3](https://github.com/springzfx/cgproxy/issues/3) for detail

run_in_cgroup.sh

@@ -1,50 +0,0 @@
-#!/bin/bash
-
-print_help(){
-cat << 'DOC'
-usage:
-    run_in_cgroup --cgroup=CGROUP <COMMAND>
-    run_in_cgroup --help
-note:    
-    CGROUP must start will slash '/' , and no special character
-example:
-    run_in_cgroup --cggroup=/mycgroup.slice ping 127.0.0.1
-DOC
-}
-
-## parse parameter
-for i in "$@"
-do
-case $i in
-    --cgroup=*)
-        cgroup=${i#*=}
-        shift
-        ;;
-    --help)
-        print_help
-        exit 0
-        shift
-        ;;
-    -*)
-        shift
-        ;;
-    *)
-        break
-        ;;
-esac
-done
-
-[[ -z "$cgroup" ]] && print_help && exit 1
-[[ -z "$@" ]] && print_help && exit 1
-
-# test suid bit
-if [ -u "$(which cgattach)" ]; then 
-    cgattach $$ $cgroup && attached=1
-else
-    sudo cgattach $$ $cgroup && attached=1
-fi
-
-# test attach success or not
-[[ -z "$attached" ]] && print_help && exit 1
-
-exec "$@"

src/CMakeLists.txt

@@ -0,0 +1,24 @@
+find_package(nlohmann_json REQUIRED)
+include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${PROJECT_SOURCE_DIR}/execsnoop-kernel/)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+
+if (build_execsnoop_dl)
+    add_definitions(-DBUIlD_EXECSNOOP_DL)
+    set(DL_LIB "-ldl")
+    set(EXECSNOOP_LIB "")
+else()
+    set(EXECSNOOP_LIB "execsnoop")
+endif()
+
+add_executable(main main.cpp common.cpp config.cpp cgroup_attach.cpp socket_client.cpp socket_server.cpp)
+target_link_libraries(main PRIVATE nlohmann_json::nlohmann_json ${DL_LIB} ${EXECSNOOP_LIB})
+set_target_properties(main PROPERTIES OUTPUT_NAME cgproxy)
+install(TARGETS main RUNTIME)
+
+if (build_static)
+    target_link_libraries(main PRIVATE -static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive)
+else()
+    target_link_libraries(main PRIVATE -lpthread)
+endif()

src/cgproxy.hpp

@@ -0,0 +1,84 @@
+#include "common.h"
+#include "config.h"
+#include "socket_client.h"
+#include <cstdlib>
+#include <nlohmann/json.hpp>
+#include <unistd.h>
+using json = nlohmann::json;
+using namespace ::CGPROXY;
+using namespace ::CGPROXY::CONFIG;
+
+namespace CGPROXY::CGPROXY {
+
+bool print_help = false, proxy = true;
+bool attach_pid = false;
+string arg_pid;
+inline void print_usage() {
+  if (proxy) {
+    cout << "Run program with proxy" << endl;
+    cout << "Usage: cgproxy [--help] [--debug] <CMD>" << endl;
+  } else {
+    cout << "Run program without proxy" << endl;
+    cout << "Usage: cgpnoroxy [--help] [--debug] <CMD>" << endl;
+    cout << "Alias: cgnoproxy = cgproxy --noproxy" << endl;
+  }
+}
+
+bool processArgs(const int argc, char *argv[], int &shift) {
+  int i;
+  for (i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "--pid") == 0) {
+      attach_pid = true;
+      i++;
+      if (i == argc) return false;
+      arg_pid = argv[i];
+      if (!validPid(arg_pid)) return false;
+      continue;
+    }
+    if (strcmp(argv[i], "--noproxy") == 0) { proxy = false; }
+    if (strcmp(argv[i], "--debug") == 0) { enable_debug = true; }
+    if (strcmp(argv[i], "--help") == 0) { print_help = true; }
+    if (argv[i][0] != '-') { break; }
+  }
+  shift = i;
+  return true;
+}
+
+void send_pid(const pid_t pid, bool proxy, int &status) {
+  json j;
+  j["type"] = proxy ? MSG_TYPE_PROXY_PID : MSG_TYPE_NOPROXY_PID;
+  j["data"] = pid;
+  SOCKET::send(j.dump(), status);
+}
+
+int main(int argc, char *argv[]) {
+  int shift = -1;
+  if (!processArgs(argc, argv, shift)) {
+    error("parameter error");
+    exit(EXIT_FAILURE);
+  }
+
+  if (print_help) {
+    print_usage();
+    exit(0);
+  }
+
+  if (!attach_pid && argc == shift) {
+    error("no program specified");
+    exit(EXIT_FAILURE);
+  }
+
+  int status = -1;
+  send_pid(attach_pid ? stoi(arg_pid) : getpid(), proxy, status);
+  if (status != 0) {
+    error("attach process failed");
+    if (status == 1) error("maybe cgproxy.service not running");
+    exit(EXIT_FAILURE);
+  }
+  // if just attach pid, return here
+  if (attach_pid) return 0;
+
+  string s = join2str(argc - shift, argv + shift, ' ');
+  return system(s.c_str());
+}
+} // namespace CGPROXY::CGPROXY

src/cgproxyd.hpp

@@ -0,0 +1,378 @@
+#ifndef CGPROXYD_HPP
+#define CGPROXYD_HPP
+
+#include "cgroup_attach.h"
+#include "common.h"
+#include "config.h"
+#include "execsnoop_share.h"
+#include "socket_server.h"
+#include <algorithm>
+#include <csignal>
+#include <cstdlib>
+#include <dlfcn.h>
+#include <exception>
+#include <fstream>
+#include <functional>
+#include <future>
+#include <nlohmann/json.hpp>
+#include <sched.h>
+#include <sys/file.h>
+#include <unistd.h>
+
+using namespace std;
+using json = nlohmann::json;
+using namespace ::CGPROXY::SOCKET;
+using namespace ::CGPROXY::CONFIG;
+using namespace ::CGPROXY::CGROUP;
+// using namespace ::CGPROXY::EXECSNOOP;
+
+#ifdef BUIlD_EXECSNOOP_DL
+namespace CGPROXY::EXECSNOOP {
+bool loadExecsnoopLib() {
+  try {
+    info("loading %s", LIBEXECSNOOP_SO);
+    void *handle_dl = dlopen(LIBEXECSNOOP_SO, RTLD_NOW);
+    if (handle_dl == NULL) {
+      error("dlopen %s failed: %s", LIBEXECSNOOP_SO, dlerror());
+      return false;
+    }
+    _startThread = reinterpret_cast<startThread_t *>(dlsym(handle_dl, "startThread"));
+    if (_startThread == NULL) {
+      error("dlsym startThread func failed: %s", dlerror());
+      return false;
+    }
+    info("dlsym startThread func success");
+    return true;
+  } catch (exception &e) {
+    debug("exception: %s", e.what());
+    return false;
+  }
+}
+} // namespace CGPROXY::EXECSNOOP
+#endif
+
+namespace CGPROXY::CGPROXYD {
+
+bool print_help = false;
+bool enable_socketserver = true;
+bool enable_execsnoop = false;
+
+class cgproxyd {
+  thread socketserver_thread;
+  thread execsnoop_thread;
+
+  Config config;
+
+  static cgproxyd *instance;
+  static int handle_msg_static(char *msg) {
+    if (!instance) {
+      error("no cgproxyd instance assigned");
+      return ERROR;
+    }
+    return instance->handle_msg(msg);
+  }
+
+  static int handle_pid_static(int pid) {
+    if (!instance) {
+      error("no cgproxyd instance assigned");
+      return ERROR;
+    }
+    return instance->handle_pid(pid);
+  }
+
+  int handle_pid(int pid) {
+    unique_ptr<char[], decltype(&free)> path(
+        realpath(to_str("/proc/", pid, "/exe").c_str(), NULL), &free);
+    if (path == NULL) {
+      debug("execsnoop: pid %d live life too short", pid);
+      return 0;
+    }
+    debug("execsnoop: %d %s", pid, path.get());
+
+    vector<string> v;
+
+    v = config.program_noproxy;
+    if (find(v.begin(), v.end(), path.get()) != v.end()) {
+      string cg = getCgroup(pid);
+      if (cg.empty()) {
+        debug("execsnoop: cgroup get failed, ignore: %d %s", pid, path.get());
+        return 0;
+      }
+      if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
+          belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
+        info("execsnoop: already in preserverd cgroup, leave alone: %d %s", pid,
+             path.get());
+        return 0;
+      }
+      if (!belongToCgroup(cg, config.cgroup_noproxy)) {
+        int res = attach(pid, config.cgroup_noproxy_preserved);
+        if (res == 0) {
+          info("execsnoop; noproxy: %d %s", pid, path.get());
+        } else {
+          info("execsnoop; noproxy failed: %d %s", pid, path.get());
+        }
+        return res;
+      }
+    }
+
+    v = config.program_proxy;
+    if (find(v.begin(), v.end(), path.get()) != v.end()) {
+      string cg = getCgroup(pid);
+      if (cg.empty()) {
+        debug("execsnoop: cgroup get failed, ignore: %d %s", pid, path.get());
+        return 0;
+      }
+      if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
+          belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
+        info("execsnoop: already in preserverd cgroup, leave alone: %d %s", pid,
+             path.get());
+        return 0;
+      }
+      if (!belongToCgroup(cg, config.cgroup_proxy)) {
+        int res = attach(pid, config.cgroup_proxy_preserved);
+        if (res == 0) {
+          info("execsnoop: proxied: %d %s", pid, path.get());
+        } else {
+          info("execsnoop: proxied failed: %d %s", pid, path.get());
+        }
+        return res;
+      }
+    }
+    return 0;
+  }
+
+  static void signalHandler(int signum) {
+    debug("Signal %d received.", signum);
+    if (!instance) {
+      error("no cgproxyd instance assigned");
+    } else {
+      instance->stop();
+    }
+    exit(0);
+  }
+
+  // single process instance
+  int lock_fd;
+  void lock() {
+    lock_fd = open(PID_LOCK_FILE, O_CREAT | O_RDWR, 0666);
+    int rc = flock(lock_fd, LOCK_EX | LOCK_NB);
+    if (rc == -1) {
+      perror(PID_LOCK_FILE);
+      error("maybe another cgproxyd is running");
+      exit(EXIT_FAILURE);
+    } else {
+      ofstream ofs(PID_LOCK_FILE);
+      ofs << getpid() << endl;
+      ofs.close();
+    }
+  }
+  void unlock() {
+    close(lock_fd);
+    unlink(PID_LOCK_FILE);
+  }
+
+  int handle_msg(char *msg) {
+    debug("received msg: %s", msg);
+    json j;
+    try {
+      j = json::parse(msg);
+    } catch (exception &e) {
+      debug("msg paser error");
+      return MSG_ERROR;
+    }
+
+    int type, status, pid;
+    try {
+      type = j.at("type").get<int>();
+      switch (type) {
+      case MSG_TYPE_CONFIG_JSON:
+        status = config.loadFromJsonStr(j.at("data").dump());
+        info("process received config json msg");
+        if (status == SUCCESS) status = applyConfig();
+        return status;
+        break;
+      case MSG_TYPE_CONFIG_PATH:
+        status = config.loadFromFile(j.at("data").get<string>());
+        info("process received config path msg");
+        if (status == SUCCESS) status = applyConfig();
+        return status;
+        break;
+      case MSG_TYPE_PROXY_PID:
+        pid = j.at("data").get<int>();
+        info("process proxy pid msg: %d", pid);
+        status = attach(pid, config.cgroup_proxy_preserved);
+        return status;
+        break;
+      case MSG_TYPE_NOPROXY_PID:
+        pid = j.at("data").get<int>();
+        info("process noproxy pid msg: %d", pid);
+        status = attach(pid, config.cgroup_noproxy_preserved);
+        return status;
+        break;
+      default:
+        error("unknown msg");
+        return MSG_ERROR;
+        break;
+      };
+    } catch (out_of_range &e) { return MSG_ERROR; } catch (exception &e) {
+      return ERROR;
+    }
+  }
+
+  void startSocketListeningThread() {
+    promise<void> status;
+    future<void> status_f = status.get_future();
+    thread th(SOCKET::startThread, handle_msg_static, move(status));
+    socketserver_thread = move(th);
+
+    future_status fstatus = status_f.wait_for(chrono::seconds(THREAD_TIMEOUT));
+    if (fstatus == std::future_status::ready) {
+      info("socketserver thread started");
+    } else {
+      error("socketserver thread timeout, maybe failed");
+    }
+  }
+
+  void startExecsnoopThread() {
+    #ifdef BUIlD_EXECSNOOP_DL
+    if (!EXECSNOOP::loadExecsnoopLib() || EXECSNOOP::_startThread == NULL) {
+      error("execsnoop not ready to start, maybe missing libbpf");
+      return;
+    }
+    #endif
+
+    promise<void> status;
+    future<void> status_f = status.get_future();
+    #ifdef BUIlD_EXECSNOOP_DL
+    thread th(EXECSNOOP::_startThread, handle_pid_static, move(status));
+    #else
+    thread th(EXECSNOOP::startThread, handle_pid_static, move(status));
+    #endif
+
+    execsnoop_thread = move(th);
+
+    future_status fstatus = status_f.wait_for(chrono::seconds(THREAD_TIMEOUT));
+    if (fstatus == std::future_status::ready) {
+      info("execsnoop thread started");
+      processRunningProgram();
+    } else {
+      error("execsnoop thread timeout, maybe failed");
+    }
+  }
+
+  void processRunningProgram() {
+    debug("process running program");
+    for (auto &path : config.program_noproxy)
+      for (auto &pid : bash_pidof(path)) {
+        string cg = getCgroup(pid);
+        if (cg.empty()) {
+          debug("cgroup get failed, ignore: %d %s", pid, path.c_str());
+          continue;
+        }
+        if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
+            belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
+          debug("already in preserverd cgroup, leave alone: %d %s", pid, path.c_str());
+          continue;
+        }
+        if (!belongToCgroup(cg, config.cgroup_noproxy)) {
+          int status = attach(pid, config.cgroup_noproxy_preserved);
+          if (status == 0) info("noproxy running process %d %s", pid, path.c_str());
+        }
+      }
+    for (auto &path : config.program_proxy)
+      for (auto &pid : bash_pidof(path)) {
+        string cg = getCgroup(pid);
+        if (cg.empty()) {
+          debug("cgroup get failed, ignore: %d %s", pid, path.c_str());
+          continue;
+        }
+        if (belongToCgroup(cg, config.cgroup_proxy_preserved) ||
+            belongToCgroup(cg, config.cgroup_noproxy_preserved)) {
+          debug("already in preserverd cgroup, leave alone: %d %s", pid, path.c_str());
+          continue;
+        }
+        if (!belongToCgroup(cg, config.cgroup_proxy)) {
+          int status = attach(pid, config.cgroup_proxy_preserved);
+          if (status == 0) info("proxied running process %d %s", pid, path.c_str());
+        }
+      }
+  }
+
+  void assignStaticInstance() { instance = this; }
+
+public:
+  int start() {
+    lock();
+    signal(SIGINT, &signalHandler);
+    signal(SIGTERM, &signalHandler);
+    signal(SIGHUP, &signalHandler);
+
+    assignStaticInstance();
+
+    if (config.loadFromFile(DEFAULT_CONFIG_FILE)!=SUCCESS) {
+      error("load config file failed");
+      return -1;
+    }
+    applyConfig();
+
+    if (enable_socketserver) startSocketListeningThread();
+    if (enable_execsnoop) startExecsnoopThread();
+
+    if (socketserver_thread.joinable()) socketserver_thread.join();
+    if (execsnoop_thread.joinable()) execsnoop_thread.join();
+
+    return 0;
+  }
+
+  int applyConfig() {
+    system(TPROXY_IPTABLS_CLEAN);
+    config.print_summary();
+    config.toEnv();
+    system(TPROXY_IPTABLS_START);
+    // no need to track running status
+    return 0;
+  }
+
+  void stop() {
+    debug("stopping");
+    system(TPROXY_IPTABLS_CLEAN);
+    unlock();
+  }
+
+  ~cgproxyd() { stop(); }
+};
+
+cgproxyd *cgproxyd::instance = NULL;
+
+void print_usage() {
+  cout << "Start a daemon with unix socket to accept control" << endl;
+  cout << "Usage: cgproxyd [--help] [--debug]" << endl;
+  cout << "Alias: cgproxyd = cgproxy --daemon" << endl;
+}
+
+void processArgs(const int argc, char *argv[]) {
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "--debug") == 0) { enable_debug = true; }
+    if (strcmp(argv[i], "--help") == 0) { print_help = true; }
+    if (strcmp(argv[i], "--execsnoop") == 0) { enable_execsnoop = true; }
+    if (argv[i][0] != '-') { break; }
+  }
+}
+
+int main(int argc, char *argv[]) {
+  processArgs(argc, argv);
+  if (print_help) {
+    print_usage();
+    exit(0);
+  }
+
+  if (getuid() != 0) {
+    error("permission denied, need root");
+    exit(EXIT_FAILURE);
+  }
+
+  cgproxyd d;
+  return d.start();
+}
+} // namespace CGPROXY::CGPROXYD
+#endif

src/cgroup_attach.cpp

@@ -0,0 +1,92 @@
+#include "cgroup_attach.h"
+#include "common.h"
+#include <errno.h>
+#include <fstream>
+#include <iostream>
+#include <regex>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <thread>
+#include <unistd.h>
+
+namespace CGPROXY::CGROUP {
+
+string cgroup2_mount_point = CGROUP2_MOUNT_POINT;
+
+
+bool validate(string pid, string cgroup) {
+  bool pid_v = validPid(pid);
+  bool cg_v = validCgroup(cgroup);
+  if (pid_v && cg_v) return true;
+
+  error("attach paramater validate error");
+  return_error;
+}
+
+int attach(const string pid, const string cgroup_target) {
+  if (getuid() != 0) {
+    error("need root to attach cgroup");
+    return_error;
+  }
+
+  debug("attaching %s to %s", pid.c_str(), cgroup_target.c_str());
+
+  if (!validate(pid, cgroup_target)) return_error;
+  if (cgroup2_mount_point.empty()) return_error;
+  string cgroup_target_path = cgroup2_mount_point + cgroup_target;
+  string cgroup_target_procs = cgroup_target_path + "/cgroup.procs";
+
+  // check if exist, we will create it if not exist
+  if (!dirExist(cgroup_target_path)) {
+    if (mkdir(cgroup_target_path.c_str(),
+              S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH) == 0) {
+      debug("created cgroup %s success", cgroup_target.c_str());
+    } else {
+      error("created cgroup %s failed, errno %d", cgroup_target.c_str(), errno);
+      return_error;
+    }
+    // error("cgroup %s not exist",cgroup_target.c_str());
+    // return_error
+  }
+
+  if (getCgroup(pid) == cgroup_target) {
+    debug("%s already in %s", pid.c_str(), cgroup_target.c_str());
+    return_success;
+  }
+
+  // put pid to target cgroup
+  if (write2procs(pid, cgroup_target_procs) != 0) return_error;
+
+  // wait for small period and check again
+  this_thread::sleep_for(std::chrono::milliseconds(100));
+  if (getCgroup(pid) != cgroup_target && write2procs(pid, cgroup_target_procs) != 0)
+    return_error;
+  return_success;
+}
+
+int write2procs(string pid, string procspath) {
+  ofstream procs(procspath, ofstream::app);
+  if (!procs.is_open()) {
+    error("open file %s failed", procspath.c_str());
+    return_error;
+  }
+  procs << pid.c_str() << endl;
+  procs.close();
+
+  // maybe there some write error, for example process pid may not exist
+  if (!procs) {
+    error("write %s to %s failed, maybe process %s not exist", pid.c_str(),
+          procspath.c_str(), pid.c_str());
+    return_error;
+  }
+  return_success;
+}
+
+int attach(const int pid, const string cgroup_target) {
+  return attach(to_str(pid), cgroup_target);
+}
+
+} // namespace CGPROXY::CGROUP

src/cgroup_attach.h

@@ -0,0 +1,17 @@
+#ifndef CGPROUP_ATTACH_H
+#define CGPROUP_ATTACH_H
+
+#include <stdlib.h>
+#include <string>
+using namespace std;
+
+namespace CGPROXY::CGROUP {
+extern string cgroup2_mount_point;
+bool validate(string pid, string cgroup);
+int attach(const string pid, const string cgroup_target);
+int attach(const int pid, const string cgroup_target);
+int write2procs(string pid, string procspath);
+
+} // namespace CGPROXY::CGROUP
+
+#endif

src/common.cpp

@@ -0,0 +1,124 @@
+#include "common.h"
+#include <fstream>
+#include <regex>
+#include <sys/stat.h>
+#include <unistd.h>
+
+bool enable_debug = false;
+bool enable_info = true;
+
+string join2str(const vector<string> t, const char delm) {
+  string s;
+  for (const auto &e : t) e != *(t.end() - 1) ? s += e + delm : s += e;
+  return s;
+}
+
+string join2str(const int argc, char **argv, const char delm) {
+  string s;
+  for (int i = 0; i < argc; i++) {
+    s += argv[i];
+    if (i != argc - 1) s += delm;
+  }
+  return s;
+}
+
+bool startWith(string s, string prefix) { return s.rfind(prefix, 0) == 0; }
+
+bool validCgroup(const string cgroup) {
+  return regex_match(cgroup, regex("^/[a-zA-Z0-9\\-_./@]*$"));
+}
+
+bool validCgroup(const vector<string> cgroup) {
+  for (auto &e : cgroup) {
+    if (!regex_match(e, regex("^/[a-zA-Z0-9\\-_./@]*$"))) { return false; }
+  }
+  return true;
+}
+
+bool validPid(const string pid) { return regex_match(pid, regex("^[0-9]+$")); }
+
+bool validPort(const int port) { return port > 0; }
+
+bool fileExist(const string &path) {
+  struct stat st;
+  return (stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode));
+}
+
+bool dirExist(const string &path) {
+  struct stat st;
+  return (stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode));
+}
+
+vector<int> bash_pidof(const string &path) {
+  vector<int> pids;
+  unique_ptr<FILE, decltype(&pclose)> fp(popen(to_str("pidof ", path).c_str(), "r"),
+                                         &pclose);
+  if (!fp) return pids;
+  int pid;
+  while (fscanf(fp.get(), "%d", &pid) != EOF) { pids.push_back(pid); }
+  return pids;
+}
+
+string bash_which(const string &name) {
+  stringstream buffer;
+  unique_ptr<FILE, decltype(&pclose)> fp(popen(to_str("which ", name).c_str(), "r"),
+                                         &pclose);
+  if (!fp) return "";
+  char buf[READ_SIZE_MAX];
+  while (fgets(buf, READ_SIZE_MAX, fp.get()) != NULL) { buffer << buf; }
+  string s = buffer.str();
+  if (!s.empty()) s.pop_back(); // remove newline character
+  return s;
+}
+
+string bash_readlink(const string &path) {
+  stringstream buffer;
+  unique_ptr<FILE, decltype(&pclose)> fp(popen(to_str("readlink -e ", path).c_str(), "r"),
+                                         &pclose);
+  if (!fp) return "";
+  char buf[READ_SIZE_MAX];
+  while (fgets(buf, READ_SIZE_MAX, fp.get()) != NULL) { buffer << buf; }
+  string s = buffer.str();
+  if (!s.empty()) s.pop_back(); // remove newline character
+  return s;
+}
+
+string getRealExistPath(const string &name) {
+  if (name[0] == '/' && fileExist(name)) return name;
+  string path;
+  path = bash_which(name);
+  if (path.empty()) return "";
+  path = bash_readlink(path);
+  if (!fileExist(path)) return "";
+  return path;
+}
+
+bool belongToCgroup(string cg1, string cg2) { return startWith(cg1 + '/', cg2 + '/'); }
+
+bool belongToCgroup(string cg1, vector<string> cg2) {
+  for (const auto &s : cg2) {
+    if (startWith(cg1 + '/', s + '/')) return true;
+  }
+  return false;
+}
+
+string getCgroup(const pid_t &pid) { return getCgroup(to_str(pid)); }
+
+string getCgroup(const string &pid) {
+  string cgroup_f = to_str("/proc/", pid, "/cgroup");
+  if (!fileExist(cgroup_f)) return "";
+
+  string cgroup, line;
+  ifstream ifs(cgroup_f);
+  debug("prcessing file %s", cgroup_f.c_str());
+  while (ifs.good() && getline(ifs, line)) {
+    // debug("process line: %s", line.c_str());
+    if (line[0] == '0') {
+      cgroup = line.substr(3);
+      debug("get cgroup of %s: %s", pid.c_str(), cgroup.c_str());
+      break;
+    }
+  }
+  ifs.close();
+  return cgroup;
+}

src/common.h

@@ -0,0 +1,112 @@
+#ifndef COMMON_H
+#define COMMON_H 1
+
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+using namespace std;
+
+#define TPROXY_IPTABLS_START "/usr/share/cgproxy/scripts/cgroup-tproxy.sh"
+#define TPROXY_IPTABLS_CLEAN "/usr/share/cgproxy/scripts/cgroup-tproxy.sh stop"
+
+#define LIBEXECSNOOP_SO "/usr/lib/cgproxy/libexecsnoop.so"
+#define CGROUP2_MOUNT_POINT "/var/run/cgproxy/cgroup2"
+#define PID_LOCK_FILE "/var/run/cgproxyd.pid"
+#define SOCKET_PATH "/tmp/cgproxy_unix_socket"
+#define LISTEN_BACKLOG 64
+#define DEFAULT_CONFIG_FILE "/etc/cgproxy/config.json"
+#define READ_SIZE_MAX 128
+
+#define CGROUP_PROXY_PRESVERED "/proxy.slice"
+#define CGROUP_NOPROXY_PRESVERED "/noproxy.slice"
+
+#define THREAD_TIMEOUT 5
+
+#define MSG_TYPE_CONFIG_JSON 1
+#define MSG_TYPE_CONFIG_PATH 2
+#define MSG_TYPE_PROXY_PID 3
+#define MSG_TYPE_NOPROXY_PID 4
+
+#define UNKNOWN_ERROR 99
+#define ERROR -1
+#define SUCCESS 0
+#define CONN_ERROR 1
+#define MSG_ERROR 2
+#define PARSE_ERROR 3
+#define PARAM_ERROR 4
+#define APPLY_ERROR 5
+#define CGROUP_ERROR 6
+#define FILE_ERROR 7
+
+extern bool enable_debug;
+extern bool enable_info;
+
+#define error(...)                                                                       \
+  {                                                                                      \
+    fprintf(stderr, "error: ");                                                          \
+    fprintf(stderr, __VA_ARGS__);                                                        \
+    fprintf(stderr, "\n");                                                               \
+    fflush(stderr);                                                                      \
+  }
+
+#define warning(...)                                                                       \
+  {                                                                                      \
+    fprintf(stderr, "warning: ");                                                          \
+    fprintf(stderr, __VA_ARGS__);                                                        \
+    fprintf(stderr, "\n");                                                               \
+    fflush(stderr);                                                                      \
+  }
+
+#define debug(...)                                                                       \
+  if (enable_debug) {                                                                    \
+    fprintf(stdout, "debug: ");                                                          \
+    fprintf(stdout, __VA_ARGS__);                                                        \
+    fprintf(stdout, "\n");                                                               \
+    fflush(stdout);                                                                      \
+  }
+
+#define info(...)                                                                        \
+  if (enable_info) {                                                                     \
+    fprintf(stdout, "info: ");                                                           \
+    fprintf(stdout, __VA_ARGS__);                                                        \
+    fprintf(stdout, "\n");                                                               \
+    fflush(stdout);                                                                      \
+  }
+
+#define return_error return -1
+#define return_success return 0
+
+template <typename... T> string to_str(T... args) {
+  stringstream ss;
+  ss.clear();
+  ss << std::boolalpha;
+  (ss << ... << args);
+  return ss.str();
+}
+
+string join2str(const vector<string> t, const char delm = ' ');
+string join2str(const int argc, char **argv, const char delm = ' ');
+bool startWith(string prefix);
+
+bool validCgroup(const string cgroup);
+bool validCgroup(const vector<string> cgroup);
+bool validPid(const string pid);
+bool validPort(const int port);
+
+bool fileExist(const string &path);
+bool dirExist(const string &path);
+vector<int> bash_pidof(const string &path);
+string bash_which(const string &name);
+string bash_readlink(const string &path);
+string getRealExistPath(const string &name);
+
+/**
+ * whether cg1 belongs to cg2
+ */
+bool belongToCgroup(string cg1, string cg2);
+bool belongToCgroup(string cg1, vector<string> cg2);
+string getCgroup(const pid_t &pid);
+string getCgroup(const string &pid);
+
+#endif

src/config.cpp

@@ -0,0 +1,170 @@
+#include "config.h"
+#include "common.h"
+#include <fstream>
+#include <iomanip>
+#include <nlohmann/json.hpp>
+#include <set>
+#include <vector>
+using json = nlohmann::json;
+
+#define add2json(v) j[#v] = v;
+#define tryassign(v)                                                                     \
+  try {                                                                                  \
+    j.at(#v).get_to(v);                                                                  \
+  } catch (exception & e) {}
+#define merge(v)                                                                         \
+  {                                                                                      \
+    v.erase(std::remove(v.begin(), v.end(), v##_preserved), v.end());                    \
+    v.insert(v.begin(), v##_preserved);                                                  \
+  }
+
+namespace CGPROXY::CONFIG {
+
+void Config::toEnv() {
+  setenv("cgroup_mount_point", CGROUP2_MOUNT_POINT, 1);
+  setenv("program_proxy", join2str(program_proxy, ':').c_str(), 1);
+  setenv("program_noproxy", join2str(program_noproxy, ':').c_str(), 1);
+  setenv("cgroup_proxy", join2str(cgroup_proxy, ':').c_str(), 1);
+  setenv("cgroup_noproxy", join2str(cgroup_noproxy, ':').c_str(), 1);
+  setenv("enable_gateway", to_str(enable_gateway).c_str(), 1);
+  setenv("port", to_str(port).c_str(), 1);
+  setenv("enable_dns", to_str(enable_dns).c_str(), 1);
+  setenv("enable_tcp", to_str(enable_tcp).c_str(), 1);
+  setenv("enable_udp", to_str(enable_udp).c_str(), 1);
+  setenv("enable_ipv4", to_str(enable_ipv4).c_str(), 1);
+  setenv("enable_ipv6", to_str(enable_ipv6).c_str(), 1);
+  setenv("table", to_str(table).c_str(), 1);
+  setenv("fwmark", to_str(fwmark).c_str(), 1);
+  setenv("mark_newin", to_str(mark_newin).c_str(), 1);
+}
+
+int Config::saveToFile(const string f) {
+  ofstream o(f);
+  if (!o.is_open()) return FILE_ERROR;
+  string js = toJsonStr();
+  o << setw(4) << js << endl;
+  o.close();
+  return 0;
+}
+
+string Config::toJsonStr() {
+  json j;
+  add2json(program_proxy);
+  add2json(program_noproxy);
+  add2json(cgroup_proxy);
+  add2json(cgroup_noproxy);
+  add2json(enable_gateway);
+  add2json(port);
+  add2json(enable_dns);
+  add2json(enable_tcp);
+  add2json(enable_udp);
+  add2json(enable_ipv4);
+  add2json(enable_ipv6);
+  add2json(table);
+  add2json(fwmark);
+  add2json(mark_newin);
+  return j.dump();
+}
+
+int Config::loadFromFile(const string f) {
+  debug("loading config: %s", f.c_str());
+  ifstream ifs(f);
+  if (ifs.is_open()) {
+    string js = to_str(ifs.rdbuf());
+    ifs.close();
+    return loadFromJsonStr(js);
+  } else {
+    error("open failed: %s", f.c_str());
+    return FILE_ERROR;
+  }
+}
+
+int Config::loadFromJsonStr(const string js) {
+  if (!validateJsonStr(js)) {
+    error("json validate fail");
+    return PARAM_ERROR;
+  }
+  json j = json::parse(js);
+  tryassign(program_proxy);
+  tryassign(program_noproxy);
+  tryassign(cgroup_proxy);
+  tryassign(cgroup_noproxy);
+  tryassign(enable_gateway);
+  tryassign(port);
+  tryassign(enable_dns);
+  tryassign(enable_tcp);
+  tryassign(enable_udp);
+  tryassign(enable_ipv4);
+  tryassign(enable_ipv6);
+  tryassign(table);
+  tryassign(fwmark);
+  tryassign(mark_newin);
+
+  // e.g. v2ray -> /usr/bin/v2ray -> /usr/lib/v2ray/v2ray
+  toRealProgramPath(program_noproxy);
+  toRealProgramPath(program_proxy);
+
+  mergeReserved();
+
+  return 0;
+}
+
+void Config::mergeReserved() {
+  merge(cgroup_proxy);
+  merge(cgroup_noproxy);
+}
+
+bool Config::validateJsonStr(const string js) {
+  json j = json::parse(js);
+  bool status = true;
+  const set<string> boolset = {"enable_gateway", "enable_dns",  "enable_tcp",
+                               "enable_udp",     "enable_ipv4", "enable_ipv6"};
+  const set<string> allowset = {"program_proxy", "program_noproxy", "comment", "table", "fwmark", "mark_newin"};
+  for (auto &[key, value] : j.items()) {
+    if (key == "cgroup_proxy" || key == "cgroup_noproxy") {
+      if (value.is_string() && !validCgroup((string)value)) status = false;
+      // TODO what if vector<int> etc.
+      if (value.is_array() && !validCgroup((vector<string>)value)) status = false;
+      if (!value.is_string() && !value.is_array()) status = false;
+    } else if (key == "port") {
+      if (!validPort(value)) status = false;
+    } else if (boolset.find(key) != boolset.end()) {
+      if (!value.is_boolean()) status = false;
+    } else if (allowset.find(key) != allowset.end()) {
+
+    } else {
+      error("unknown key: %s", key.c_str());
+      return false;
+    }
+    if (!status) {
+      error("invalid value for key: %s", key.c_str());
+      return false;
+    }
+  }
+  return true;
+}
+
+void Config::print_summary() {
+  info("noproxy program: %s", join2str(program_noproxy).c_str());
+  info("proxied program: %s", join2str(program_proxy).c_str());
+  info("noproxy cgroup: %s", join2str(cgroup_noproxy).c_str());
+  info("proxied cgroup: %s", join2str(cgroup_proxy).c_str());
+  info("table: %d, fwmark: %d, mark_newin: %d", table, fwmark, mark_newin);
+}
+
+void Config::toRealProgramPath(vector<string> &v) {
+  vector<string> tmp;
+  for (auto &p : v) {
+    auto rpath = getRealExistPath(p);
+    if (!rpath.empty()) tmp.push_back(rpath);
+    else
+      warning("%s not exist or broken link", p.c_str());
+  }
+  v = tmp;
+}
+
+#undef tryassign
+#undef add2json
+#undef merge
+
+} // namespace CGPROXY::CONFIG

src/config.h

@@ -0,0 +1,47 @@
+#ifndef CONFIG_H
+#define CONFIG_H
+#include "common.h"
+#include <stdlib.h>
+#include <string>
+#include <vector>
+using namespace std;
+
+namespace CGPROXY::CONFIG {
+
+class Config {
+public:
+  const string cgroup_proxy_preserved = CGROUP_PROXY_PRESVERED;
+  const string cgroup_noproxy_preserved = CGROUP_NOPROXY_PRESVERED;
+
+  vector<string> program_proxy = {cgroup_proxy_preserved};
+  vector<string> program_noproxy = {cgroup_noproxy_preserved};
+  vector<string> cgroup_proxy;
+  vector<string> cgroup_noproxy;
+  bool enable_gateway = false;
+  int port = 12345;
+  bool enable_dns = true;
+  bool enable_tcp = true;
+  bool enable_udp = true;
+  bool enable_ipv4 = true;
+  bool enable_ipv6 = true;
+
+  // for iptables
+  int table=10007;
+  int fwmark=0x9973;
+  int mark_newin=0x9967;
+
+  void toEnv();
+  int saveToFile(const string f);
+  string toJsonStr();
+  int loadFromFile(const string f);
+  int loadFromJsonStr(const string js);
+  void print_summary();
+
+private:
+  void mergeReserved();
+  bool validateJsonStr(const string js);
+  void toRealProgramPath(vector<string> &v);
+};
+
+} // namespace CGPROXY::CONFIG
+#endif

src/main.cpp

@@ -0,0 +1,17 @@
+#include "cgproxy.hpp"
+#include "cgproxyd.hpp"
+
+bool as_cgproxyd = false;
+void processArgs(const int argc, char *argv[]) {
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "--daemon") == 0) { as_cgproxyd = true; }
+    if (argv[i][0] != '-') { break; }
+  }
+}
+
+int main(int argc, char *argv[]) {
+  processArgs(argc, argv);
+  if (as_cgproxyd) ::CGPROXY::CGPROXYD::main(argc, argv);
+  else
+    ::CGPROXY::CGPROXY::main(argc, argv);
+}

src/socket_client.cpp

@@ -0,0 +1,49 @@
+#include "socket_client.h"
+#include "common.h"
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#define return_if_error(flag, msg)                                                       \
+  if (flag == -1) {                                                                      \
+    perror(msg);                                                                         \
+    status = CONN_ERROR;                                                                 \
+    close(sfd);                                                                          \
+    return;                                                                              \
+  }
+
+namespace CGPROXY::SOCKET {
+
+void send(const char *msg, int &status) {
+  debug("send msg: %s", msg);
+  status = UNKNOWN_ERROR;
+
+  int flag;
+  int sfd = socket(AF_UNIX, SOCK_STREAM, 0);
+
+  struct sockaddr_un unix_socket;
+  memset(&unix_socket, '\0', sizeof(struct sockaddr_un));
+  unix_socket.sun_family = AF_UNIX;
+  strncpy(unix_socket.sun_path, SOCKET_PATH, sizeof(unix_socket.sun_path) - 1);
+
+  flag = connect(sfd, (struct sockaddr *)&unix_socket, sizeof(struct sockaddr_un));
+  return_if_error(flag, "connect");
+
+  int msg_len = strlen(msg);
+  flag = write(sfd, &msg_len, sizeof(int));
+  return_if_error(flag, "write length");
+  flag = write(sfd, msg, msg_len * sizeof(char));
+  return_if_error(flag, "write msg");
+
+  flag = read(sfd, &status, sizeof(int));
+  return_if_error(flag, "read return value");
+
+  close(sfd);
+}
+
+void send(const string msg, int &status) {
+  send(msg.c_str(), status);
+  debug("return status: %d", status);
+}
+
+} // namespace CGPROXY::SOCKET

src/socket_client.h

@@ -0,0 +1,14 @@
+#ifndef SOCKET_CLIENT_H
+#define SOCKET_CLIENT_H
+
+#include <stdlib.h>
+#include <string>
+using namespace std;
+
+namespace CGPROXY::SOCKET {
+
+void send(const char *msg, int &status);
+void send(const string msg, int &status);
+
+} // namespace CGPROXY::SOCKET
+#endif

src/socket_server.cpp

@@ -0,0 +1,67 @@
+#include "socket_server.h"
+#include "common.h"
+#include <filesystem>
+#include <stdlib.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+namespace fs = std::filesystem;
+
+namespace CGPROXY::SOCKET {
+
+void SocketServer::socketListening(function<int(char *)> callback, promise<void> status) {
+  debug("starting socket listening");
+  sfd = socket(AF_UNIX, SOCK_STREAM, 0);
+
+  if (fs::exists(SOCKET_PATH) && unlink(SOCKET_PATH) == -1) {
+    error("%s exist, and can't unlink", SOCKET_PATH);
+    return;
+  }
+  memset(&unix_socket, '\0', sizeof(struct sockaddr_un));
+  unix_socket.sun_family = AF_UNIX;
+  strncpy(unix_socket.sun_path, SOCKET_PATH, sizeof(unix_socket.sun_path) - 1);
+
+  bind(sfd, (struct sockaddr *)&unix_socket, sizeof(struct sockaddr_un));
+
+  listen(sfd, LISTEN_BACKLOG);
+  chmod(SOCKET_PATH, S_IRWXU | S_IRWXG | S_IRWXO);
+
+  status.set_value();
+
+  while (true) {
+    close(cfd);
+    cfd = accept(sfd, NULL, NULL);
+    continue_if_error(cfd, "accept");
+    debug("accept connection: %d", cfd);
+
+    // read length
+    int msg_len;
+    flag = read(cfd, &msg_len, sizeof(int));
+    continue_if_error(flag, "read length");
+    // read msg
+    auto msg = (char *)malloc(msg_len + 1);
+    flag = read(cfd, msg, msg_len * sizeof(char));
+    continue_if_error(flag, "read msg");
+    msg[msg_len] = '\0';
+    // handle msg
+    int status = callback(msg);
+    free(msg);
+    // send back flag
+    flag = write(cfd, &status, sizeof(int));
+    continue_if_error(flag, "write back");
+  }
+}
+
+SocketServer::~SocketServer() {
+  close(sfd);
+  close(cfd);
+  unlink(SOCKET_PATH);
+}
+
+void startThread(function<int(char *)> callback, promise<void> status) {
+  SocketServer server;
+  server.socketListening(callback, move(status));
+}
+
+} // namespace CGPROXY::SOCKET

src/socket_server.h

@@ -0,0 +1,31 @@
+#ifndef SOCKET_SERVER_H
+#define SOCKET_SERVER_H
+
+#include <functional>
+#include <future>
+#include <stdlib.h>
+#include <sys/un.h>
+using namespace std;
+
+namespace CGPROXY::SOCKET {
+
+#define continue_if_error(flag, msg)                                                     \
+  if (flag == -1) {                                                                      \
+    perror(msg);                                                                         \
+    continue;                                                                            \
+  }
+
+class SocketServer {
+public:
+  int sfd = -1, cfd = -1, flag = -1;
+  struct sockaddr_un unix_socket;
+
+  void socketListening(function<int(char *)> callback, promise<void> status);
+  ~SocketServer();
+};
+
+void startThread(function<int(char *)> callback, promise<void> status);
+
+} // namespace CGPROXY::SOCKET
+
+#endif

test/CMakeLists.txt

@@ -0,0 +1,7 @@
+include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${PROJECT_SOURCE_DIR}/src)
+
+find_package(nlohmann_json REQUIRED)
+add_executable(client_test socket_client_test.cpp
+        ../src/socket_client.cpp ../src/common.cpp ../src/config.cpp)
+target_link_libraries(client_test nlohmann_json::nlohmann_json)

test/socket_client_test.cpp

@@ -0,0 +1,48 @@
+#include "common.h"
+#include "config.h"
+#include "socket_client.h"
+#include <nlohmann/json.hpp>
+
+using namespace std;
+using json = nlohmann::json;
+using namespace CGPROXY;
+using namespace CGPROXY::CONFIG;
+
+void send_config(Config &config, int &status) {
+  json j;
+  j["type"] = MSG_TYPE_CONFIG_JSON;
+  j["data"] = config.toJsonStr();
+  SOCKET::send(j.dump(), status);
+}
+
+void send_config_path(const string s, int &status) {
+  json j;
+  j["type"] = MSG_TYPE_CONFIG_PATH;
+  j["data"] = s;
+  SOCKET::send(j.dump(), status);
+}
+
+void send_pid(const pid_t pid, bool proxy, int &status) {
+  json j;
+  j["type"] = proxy ? MSG_TYPE_PROXY_PID : MSG_TYPE_NOPROXY_PID;
+  j["data"] = pid;
+  SOCKET::send(j.dump(), status);
+}
+
+void test_config() {
+  Config config;
+  config.cgroup_proxy = {"/"};
+  int status;
+  send_config(config, status);
+}
+
+void test_config_path() {
+  string path = "/etc/cgproxy/config.json";
+  int status;
+  send_config_path(path, status);
+}
+
+int main() {
+  test_config();
+  return 0;
+}

tools/CMakeLists.txt

@@ -0,0 +1,5 @@
+include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${PROJECT_SOURCE_DIR}/src)
+
+add_executable(cgattach cgattach.cpp ../src/cgroup_attach.cpp ../src/common.cpp)
+install(TARGETS cgattach  DESTINATION /usr/bin PERMISSIONS ${basic_permission})

tools/cgattach.cpp

@@ -0,0 +1,31 @@
+#include "cgroup_attach.h"
+#include "common.h"
+#include <cstdlib>
+#include <unistd.h>
+using namespace std;
+
+void print_usage() { fprintf(stdout, "usage: cgattach <pid> <cgroup>\n"); }
+
+int main(int argc, char *argv[]) {
+  int flag = setuid(0);
+  if (flag != 0) {
+    perror("cgattach need root");
+    exit(EXIT_FAILURE);
+  }
+
+  if (argc != 3) {
+    error("need exact 2 paramaters");
+    print_usage();
+    exit(EXIT_FAILURE);
+  }
+
+  string pid = string(argv[1]);
+  string cgroup_target = string(argv[2]);
+
+  if (validPid(pid) && validCgroup(cgroup_target)) {
+    CGPROXY::CGROUP::attach(pid, cgroup_target);
+  } else {
+    error("param not valid");
+    exit(EXIT_FAILURE);
+  }
+}

v2ray_config/05_inbounds_04_socks5.json

@@ -5,6 +5,7 @@
       "port": 1080,
       "protocol": "socks",
       "settings": {
+        "udp": true,
         "auth": "noauth",
         "userLevel": 0
       },

v2ray_config/v2ray.service

@@ -7,7 +7,7 @@ Wants=network-online.target
 [Service]
 Type=exec
 ExecStart=/usr/lib/v2ray/v2ray -config /etc/v2ray/config.json
-User=nobody
+DynamicUser=yes
 AmbientCapabilities=CAP_NET_ADMIN CAP_NET_BIND_SERVICE
 NoNewPrivileges=yes
 Restart=on-failure
@@ -15,4 +15,4 @@ Restart=on-failure
 RestartPreventExitStatus=23
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target