bump version

.gitignore

@@ -1,6 +1,5 @@
 build
 .directory
-.vscode
 .clangd
 v2ray_config/proxy
 v2ray_config/06_outbounds_proxy.json

.vscode/gdb-root

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec sudo -E /usr/bin/gdb "$@"

.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+    "configurations": [
+    {
+        "name": "(gdb) cgproxyd",
+        "type": "cppdbg",
+        "request": "launch",
+        "program": "${workspaceFolder}/build/src/cgproxy",
+        "args": ["--daemon", "--debug", "--execsnoop"],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "miDebuggerPath": "${workspaceFolder}/.vscode/gdb-root",
+        "setupCommands": [
+            {
+                "description": "Enable pretty-printing for gdb",
+                "text": "-enable-pretty-printing",
+                "ignoreFailures": true
+            }
+        ]
+    }]
+}

CMakeLists.txt

@@ -1,20 +1,24 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.14)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-project(cgproxy VERSION 0.16)
+project(cgproxy VERSION 0.18)
+
+include(GNUInstallDirs)
+
 add_compile_options(-Wall -Wextra -Wpedantic -Wno-unused-result -Wno-unused-parameter)
 
 # for clangd
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # option(with_execsnoop "enable program level proxy control feature, need bcc installed" ON)
+option(build_execsnoop_dl "build libexecsnoop.so which will be dynamic loaded, otherwise built directly into cgproxy" ON)
+option(build_static "build with static link prefered" OFF)
 option(build_tools OFF)
-option(build_test  OFF)
+option(build_test "for develop" OFF)
-
-set(basic_permission OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
 
 add_subdirectory(src)
+add_subdirectory(execsnoop-kernel)
 add_subdirectory(pack)
 if (build_tools)
     add_subdirectory(tools)
@@ -23,23 +27,19 @@ if (build_test)
     add_subdirectory(test)
 endif()
 
-install(FILES cgproxyd DESTINATION /usr/bin PERMISSIONS ${basic_permission})
+# instal scripts and other things
-install(FILES cgnoproxy DESTINATION /usr/bin PERMISSIONS ${basic_permission})
+install(PROGRAMS cgproxyd TYPE BIN)
-install(FILES cgproxy.service DESTINATION /usr/lib/systemd/system/)
+install(PROGRAMS cgnoproxy TYPE BIN)
-install(FILES config.json DESTINATION /etc/cgproxy/)
+install(PROGRAMS cgroup-tproxy.sh DESTINATION ${CMAKE_INSTALL_DATADIR}/cgproxy/scripts)
-install(FILES cgroup-tproxy.sh DESTINATION /usr/share/cgproxy/scripts/ PERMISSIONS ${basic_permission})
+install(FILES cgproxy.service DESTINATION ${CMAKE_INSTALL_LIBDIR}/systemd/system)
-install(FILES readme.md DESTINATION /usr/share/doc/cgproxy/)
+install(FILES config.json DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/cgproxy)
+install(FILES readme.md DESTINATION ${CMAKE_INSTALL_DOCDIR})
 
 # man pages
-set(man_gz
+set(man_gz ${PROJECT_SOURCE_DIR}/man/cgproxyd.1.gz ${PROJECT_SOURCE_DIR}/man/cgproxy.1.gz ${PROJECT_SOURCE_DIR}/man/cgnoproxy.1.gz)
-${PROJECT_BINARY_DIR}/cgproxyd.1.gz
+add_custom_command(OUTPUT ${man_gz}
-${PROJECT_BINARY_DIR}/cgproxy.1.gz
-${PROJECT_BINARY_DIR}/cgnoproxy.1.gz
-)
-add_custom_target(man
     COMMAND gzip -fk cgproxyd.1 cgproxy.1 cgnoproxy.1
-    COMMAND mv *.gz ${PROJECT_BINARY_DIR}
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/man
 )
-add_dependencies(main man)
+add_custom_target(man ALL DEPENDS ${man_gz})
-install(FILES ${man_gz} DESTINATION /usr/share/man/man1/)
+install(FILES ${man_gz} DESTINATION ${CMAKE_INSTALL_MANDIR}/man1/)

cgroup-tproxy.sh

@@ -60,16 +60,28 @@ fi
 [ -z ${enable_ipv4+x} ] && enable_ipv4=true
 [ -z ${enable_ipv6+x} ] && enable_ipv6=true
 
-## do not modify this if you don't known what you are doing
+##
-table=100
+get_available_route_table(){
-fwmark=0x01
+    table=10007 
-make_newin=0x02
+    while true; do 
+        ip route show table $table &> /dev/null && ((table++)) || { echo $table && break; }
+    done
+}
+
+## mark/route things
+[ -z ${table+x} ]       && table=10007 # just a prime number
+[ -z ${fwmark+x} ]      && fwmark=0x9973
+[ -z ${mark_newin+x} ]  && mark_newin=0x9967
+
+
+# echo "table: $table fwmark: $fwmark, mark_newin: $mark_newin"
 
 ## cgroup things
-cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET)
+[ -z ${cgroup_mount_point+x} ] && cgroup_mount_point=$(findmnt -t cgroup2 -n -o TARGET | head -n 1)
-cgroup_type="cgroup2"
+[ -z $cgroup_mount_point ] && { >&2 echo "iptables: no cgroup2 mount point available"; exit -1; }
-cgroup_procs_file="cgroup.procs"
+[ ! -d $cgroup_mount_point ] && mkdir -p $cgroup_mount_point
-
+[ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" != "cgroup2" ] && mount -t cgroup2 none $cgroup_mount_point
+[ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" != "cgroup2" ] && { >&2 echo "iptables: mount $cgroup_mount_point failed"; exit -1; }
 
 stop(){
     iptables -t mangle -L TPROXY_PRE &> /dev/null || return
@@ -97,6 +109,8 @@ stop(){
     ## may not exist, just ignore, and tracking their existence is not reliable
     iptables -t nat -D POSTROUTING -m owner ! --socket-exists -j MASQUERADE &> /dev/null
     ip6tables -t nat -D POSTROUTING -m owner ! --socket-exists -s fc00::/7 -j MASQUERADE &> /dev/null
+    ## unmount cgroup2
+    [ "$(findmnt -M $cgroup_mount_point -n -o FSTYPE)" = "cgroup2" ] && umount $cgroup_mount_point
 }
 
 ## parse parameter
@@ -123,6 +137,20 @@ done
 test -d $cgroup_mount_point$cgroup_proxy    || mkdir $cgroup_mount_point$cgroup_proxy   || exit -1; 
 test -d $cgroup_mount_point$cgroup_noproxy  || mkdir $cgroup_mount_point$cgroup_noproxy || exit -1; 
 
+## filter cgroup that not exist
+_cgroup_noproxy=()
+for cg in ${cgroup_noproxy[@]}; do
+    test -d $cgroup_mount_point$cg && _cgroup_noproxy+=($cg) || { >&2 echo "iptables: $cg not exist, ignore";}
+done
+unset cgroup_noproxy && cgroup_noproxy=${_cgroup_noproxy[@]}
+
+## filter cgroup that not exist
+_cgroup_proxy=()
+for cg in ${cgroup_proxy[@]}; do
+    test -d $cgroup_mount_point$cg && _cgroup_proxy+=($cg) || { >&2 echo "iptables: $cg not exist, ignore";}
+done
+unset cgroup_proxy && cgroup_proxy=${_cgroup_proxy[@]}
+
 
 echo "iptables: applying tproxy iptables"
 ## use TPROXY
@@ -130,8 +158,8 @@ echo "iptables: applying tproxy iptables"
 ip rule add fwmark $fwmark table $table
 ip route add local default dev lo table $table
 iptables -t mangle -N TPROXY_ENT
-iptables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
+iptables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark
-iptables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
+iptables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip 127.0.0.1 --on-port $port --tproxy-mark $fwmark
 
 iptables -t mangle -N TPROXY_PRE
 iptables -t mangle -A TPROXY_PRE -m socket --transparent -j MARK --set-mark $fwmark
@@ -146,14 +174,14 @@ iptables -t mangle -A PREROUTING -j TPROXY_PRE
 
 iptables -t mangle -N TPROXY_OUT
 iptables -t mangle -A TPROXY_OUT -p icmp -j RETURN
-iptables -t mangle -A TPROXY_OUT -m connmark --mark  $make_newin -j RETURN
+iptables -t mangle -A TPROXY_OUT -m connmark --mark  $mark_newin -j RETURN
 iptables -t mangle -A TPROXY_OUT -m addrtype --dst-type LOCAL -j RETURN
 iptables -t mangle -A TPROXY_OUT -m addrtype ! --dst-type UNICAST -j RETURN
 for cg in ${cgroup_noproxy[@]}; do
-iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN || { >&2 echo "iptables: $cg not exist, won't apply"; }
+iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
 done
 for cg in ${cgroup_proxy[@]}; do
-iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark || { >&2 echo "iptables: $cg not exist, won't apply"; }
+iptables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark
 done
 iptables -t mangle -A OUTPUT -j TPROXY_OUT
 
@@ -161,8 +189,8 @@ iptables -t mangle -A OUTPUT -j TPROXY_OUT
 ip -6 rule add fwmark $fwmark table $table
 ip -6 route add local default dev lo table $table
 ip6tables -t mangle -N TPROXY_ENT
-ip6tables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
+ip6tables -t mangle -A TPROXY_ENT -p tcp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark
-ip6tables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip localhost --on-port $port --tproxy-mark $fwmark
+ip6tables -t mangle -A TPROXY_ENT -p udp -j TPROXY --on-ip ::1 --on-port $port --tproxy-mark $fwmark
 
 ip6tables -t mangle -N TPROXY_PRE
 ip6tables -t mangle -A TPROXY_PRE -m socket --transparent -j MARK --set-mark $fwmark
@@ -177,14 +205,14 @@ ip6tables -t mangle -A PREROUTING -j TPROXY_PRE
 
 ip6tables -t mangle -N TPROXY_OUT
 ip6tables -t mangle -A TPROXY_OUT -p icmpv6 -j RETURN
-ip6tables -t mangle -A TPROXY_OUT -m connmark --mark  $make_newin -j RETURN
+ip6tables -t mangle -A TPROXY_OUT -m connmark --mark  $mark_newin -j RETURN
 ip6tables -t mangle -A TPROXY_OUT -m addrtype --dst-type LOCAL -j RETURN
 ip6tables -t mangle -A TPROXY_OUT -m addrtype ! --dst-type UNICAST -j RETURN
 for cg in ${cgroup_noproxy[@]}; do
-ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN || { >&2 echo "iptables: $cg not exist, won't apply"; }
+ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j RETURN
 done
 for cg in ${cgroup_proxy[@]}; do
-ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark || { >&2 echo "iptables: $cg not exist, won't apply"; }
+ip6tables -t mangle -A TPROXY_OUT -m cgroup --path $cg -j MARK --set-mark $fwmark
 done
 ip6tables -t mangle -A OUTPUT -j TPROXY_OUT
 
@@ -215,10 +243,10 @@ $enable_gateway || ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LO
 
 ## make sure following rules are the first in chain TPROXY_PRE to mark new incoming connection or gateway proxy connection
 ## so must put at last to insert first
-iptables  -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $make_newin
+iptables  -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $mark_newin
-ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $make_newin
+ip6tables -t mangle -I TPROXY_PRE -m addrtype ! --src-type LOCAL -m conntrack --ctstate NEW -j CONNMARK --set-mark $mark_newin
 
-## message for user
+# message for user
 cat << DOC
 iptables: noproxy cgroup: ${cgroup_noproxy[@]}
 iptables: proxied cgroup: ${cgroup_proxy[@]}

config.json

@@ -1,4 +1,6 @@
 {
+    "comment":"For usgae, see https://github.com/springzfx/cgproxy",
+
     "port": 12345,
     "program_noproxy": ["v2ray", "qv2ray"],
     "program_proxy": [],
@@ -9,5 +11,8 @@
     "enable_udp": true,
     "enable_tcp": true,
     "enable_ipv4": true,
-    "enable_ipv6": true
+    "enable_ipv6": true,
+    "table": 10007,
+    "fwmark": 39283,
+    "mark_newin": 39271
 }

execsnoop-bcc/readme.md

@@ -0,0 +1,3 @@
+- depend [bcc](https://github.com/iovisor/bcc) 
+
+- huge memory usage, at least 50M

execsnoop-kernel/CMakeLists.txt

@@ -0,0 +1,19 @@
+# find libbpf
+if (build_static)
+    find_library(LIBBPF libbpf.a)
+else()
+    find_library(LIBBPF bpf)
+endif()
+
+if (LIBBPF-NOTFOUND)
+    message(FATAL_ERROR "libbpf not found")
+endif()
+
+if (build_execsnoop_dl)
+    add_library(execsnoop MODULE execsnoop_share.cpp)
+    install(TARGETS execsnoop DESTINATION ${CMAKE_INSTALL_LIBDIR}/cgproxy/)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF} -lelf -lz)
+else()
+    add_library(execsnoop STATIC execsnoop_share.cpp)
+    target_link_libraries(execsnoop PRIVATE ${LIBBPF} -l:libelf.a -l:libz.a)
+endif()

execsnoop-kernel/execsnoop_kern.c

@@ -0,0 +1,93 @@
+#include "linux/sched.h"
+#include <linux/version.h>
+#include <linux/ptrace.h>
+#include <uapi/linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+/* /sys/kernel/debug/tracing/events/syscalls/sys_enter_execve/format */
+struct syscalls_enter_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	const char filename_ptr;
+    const char *const * argv;
+    const char *const * envp;
+};
+
+/* /sys/kernel/debug/tracing/events/syscalls/sys_exit_execve/format */
+struct syscalls_exit_execve_args {
+	__u64 unused;
+	int syscall_nr;
+	long ret;
+};
+
+struct bpf_map_def SEC("maps") records = {
+    .type = BPF_MAP_TYPE_HASH,
+    .key_size = sizeof(pid_t),
+    .value_size = sizeof(struct event),
+    .max_entries = 1024,
+};
+struct bpf_map_def SEC("maps") perf_events = {
+    .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
+    .key_size = sizeof(u32),
+    .value_size = sizeof(u32),
+    .max_entries = 128,
+};
+
+SEC("tracepoint/syscalls/sys_enter_execve")
+int syscall_enter_execve(struct syscalls_enter_execve_args *ctx){
+    pid_t pid, tgid; uid_t uid;
+	struct event *event;
+	struct task_struct *task, *task_p;
+
+    u64 id = bpf_get_current_pid_tgid();
+	pid = (pid_t)id;
+	tgid = id >> 32;
+	uid = (u32)bpf_get_current_uid_gid();
+
+	struct event empty_event={};
+	if (bpf_map_update_elem(&records, &pid, &empty_event, BPF_NOEXIST)!=0) return 0;
+	event = bpf_map_lookup_elem(&records, &pid);
+	if (!event) return 0;
+
+	event->pid = pid;
+	event->tgid = tgid;
+	event->uid = uid;
+
+	/* ppid is not reliable here, normal in arch, but become 0 in ubuntu 20.04 */
+	task = (struct task_struct*)bpf_get_current_task();
+	bpf_probe_read(&task_p, sizeof(struct task_struct*),&task->real_parent);
+	bpf_probe_read(&event->ppid,sizeof(pid_t),&task_p->tgid);
+
+    return 0;
+}
+
+SEC("tracepoint/syscalls/sys_exit_execve")
+int syscall_exit_execve(struct syscalls_exit_execve_args *ctx){
+	pid_t pid;
+	struct event *event;
+
+	pid = (pid_t)bpf_get_current_pid_tgid();
+	event = bpf_map_lookup_elem(&records,&pid);
+	if (!event) return 0;
+	if (ctx->ret < 0) goto cleanup;
+
+	/* get comm */
+	bpf_get_current_comm(&event->comm,sizeof(event->comm));
+	bpf_perf_event_output(ctx, &perf_events, BPF_F_CURRENT_CPU, event, sizeof(struct event));
+cleanup:
+	bpf_map_delete_elem(&records, &pid);
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
+u32 _version SEC("version") = LINUX_VERSION_CODE;

execsnoop-kernel/execsnoop_kern_skel.h

@@ -0,0 +1,225 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+
+/* THIS FILE IS AUTOGENERATED! */
+#ifndef __EXECSNOOP_KERN_SKEL_H__
+#define __EXECSNOOP_KERN_SKEL_H__
+
+#include <stdlib.h>
+#include <bpf/libbpf.h>
+
+struct execsnoop_kern {
+	struct bpf_object_skeleton *skeleton;
+	struct bpf_object *obj;
+	struct {
+		struct bpf_map *perf_events;
+		struct bpf_map *records;
+	} maps;
+	struct {
+		struct bpf_program *syscall_enter_execve;
+		struct bpf_program *syscall_exit_execve;
+	} progs;
+	struct {
+		struct bpf_link *syscall_enter_execve;
+		struct bpf_link *syscall_exit_execve;
+	} links;
+};
+
+static void
+execsnoop_kern__destroy(struct execsnoop_kern *obj)
+{
+	if (!obj)
+		return;
+	if (obj->skeleton)
+		bpf_object__destroy_skeleton(obj->skeleton);
+	free(obj);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj);
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_opts(const struct bpf_object_open_opts *opts)
+{
+	struct execsnoop_kern *obj;
+
+	obj = (typeof(obj))calloc(1, sizeof(*obj));
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__create_skeleton(obj))
+		goto err;
+	if (bpf_object__open_skeleton(obj->skeleton, opts))
+		goto err;
+
+	return obj;
+err:
+	execsnoop_kern__destroy(obj);
+	return NULL;
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open(void)
+{
+	return execsnoop_kern__open_opts(NULL);
+}
+
+static inline int
+execsnoop_kern__load(struct execsnoop_kern *obj)
+{
+	return bpf_object__load_skeleton(obj->skeleton);
+}
+
+static inline struct execsnoop_kern *
+execsnoop_kern__open_and_load(void)
+{
+	struct execsnoop_kern *obj;
+
+	obj = execsnoop_kern__open();
+	if (!obj)
+		return NULL;
+	if (execsnoop_kern__load(obj)) {
+		execsnoop_kern__destroy(obj);
+		return NULL;
+	}
+	return obj;
+}
+
+static inline int
+execsnoop_kern__attach(struct execsnoop_kern *obj)
+{
+	return bpf_object__attach_skeleton(obj->skeleton);
+}
+
+static inline void
+execsnoop_kern__detach(struct execsnoop_kern *obj)
+{
+	return bpf_object__detach_skeleton(obj->skeleton);
+}
+
+static inline int
+execsnoop_kern__create_skeleton(struct execsnoop_kern *obj)
+{
+	struct bpf_object_skeleton *s;
+
+	s = (typeof(s))calloc(1, sizeof(*s));
+	if (!s)
+		return -1;
+	obj->skeleton = s;
+
+	s->sz = sizeof(*s);
+	s->name = "execsnoop_kern";
+	s->obj = &obj->obj;
+
+	/* maps */
+	s->map_cnt = 2;
+	s->map_skel_sz = sizeof(*s->maps);
+	s->maps = (typeof(s->maps))calloc(s->map_cnt, s->map_skel_sz);
+	if (!s->maps)
+		goto err;
+
+	s->maps[0].name = "perf_events";
+	s->maps[0].map = &obj->maps.perf_events;
+
+	s->maps[1].name = "records";
+	s->maps[1].map = &obj->maps.records;
+
+	/* programs */
+	s->prog_cnt = 2;
+	s->prog_skel_sz = sizeof(*s->progs);
+	s->progs = (typeof(s->progs))calloc(s->prog_cnt, s->prog_skel_sz);
+	if (!s->progs)
+		goto err;
+
+	s->progs[0].name = "syscall_enter_execve";
+	s->progs[0].prog = &obj->progs.syscall_enter_execve;
+	s->progs[0].link = &obj->links.syscall_enter_execve;
+
+	s->progs[1].name = "syscall_exit_execve";
+	s->progs[1].prog = &obj->progs.syscall_exit_execve;
+	s->progs[1].link = &obj->links.syscall_exit_execve;
+
+	s->data_sz = 2320;
+	s->data = (void *)"\
+\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\xd0\x05\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0d\0\
+\x01\0\x85\0\0\0\x0e\0\0\0\xbf\x06\0\0\0\0\0\0\x63\x6a\xfc\xff\0\0\0\0\x85\0\0\
+\0\x0f\0\0\0\xbf\x07\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\
+\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\
+\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
+\xd0\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb7\x04\0\0\x01\0\0\0\x85\
+\0\0\0\x02\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x19\0\0\0\0\0\
+\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\0\x15\x08\x12\0\0\0\0\0\x77\x06\0\
+\0\x20\0\0\0\x61\xa1\xfc\xff\0\0\0\0\x63\x78\x1c\0\0\0\0\0\x63\x68\x14\0\0\0\0\
+\0\x63\x18\x10\0\0\0\0\0\x85\0\0\0\x23\0\0\0\x07\0\0\0\x18\x05\0\0\xbf\xa1\0\0\
+\0\0\0\0\x07\x01\0\0\xf0\xff\xff\xff\xb7\x02\0\0\x08\0\0\0\xbf\x03\0\0\0\0\0\0\
+\x85\0\0\0\x04\0\0\0\x07\x08\0\0\x18\0\0\0\x79\xa3\xf0\xff\0\0\0\0\x07\x03\0\0\
+\x0c\x05\0\0\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\
+\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\xbf\x16\0\0\0\0\0\0\x85\0\0\0\x0e\0\0\0\x63\
+\x0a\xfc\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x15\x07\x13\
+\0\0\0\0\0\x79\x61\x10\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\x6d\x12\x0b\0\0\0\0\0\xbf\
+\x71\0\0\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x85\0\0\0\x10\0\0\0\xbf\x61\0\0\0\0\0\0\
+\x18\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\x03\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\
+\0\xbf\x74\0\0\0\0\0\0\xb7\x05\0\0\x20\0\0\0\x85\0\0\0\x19\0\0\0\xbf\xa2\0\0\0\
+\0\0\0\x07\x02\0\0\xfc\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\
+\0\x03\0\0\0\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\x01\0\0\0\x04\0\0\0\x20\0\0\0\
+\0\x04\0\0\0\0\0\0\x04\0\0\0\x04\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\
+\0\x06\x07\x05\0\x10\0\0\0\0\0\0\0\x01\x7a\x52\0\x08\x7c\x0b\x01\x0c\0\0\0\x18\
+\0\0\0\x18\0\0\0\0\0\0\0\0\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\x34\0\0\
+\0\0\0\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\xba\0\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xe9\0\0\0\0\0\x03\0\x70\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe2\0\0\0\0\0\x05\0\
+\xc0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdb\0\0\0\0\0\x05\0\xe8\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\
+\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa3\0\0\0\x11\0\x08\0\0\0\0\0\0\0\0\0\
+\x04\0\0\0\0\0\0\0\x20\0\0\0\x11\0\x09\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\x07\
+\0\0\0\x11\0\x07\0\x14\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x18\0\0\0\x11\0\x07\0\0\
+\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x8e\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\0\x80\x01\
+\0\0\0\0\0\0\x51\0\0\0\x12\0\x05\0\0\0\0\0\0\0\0\0\xf8\0\0\0\0\0\0\0\x70\0\0\0\
+\0\0\0\0\x01\0\0\0\x0a\0\0\0\xb8\0\0\0\0\0\0\0\x01\0\0\0\x0a\0\0\0\x28\0\0\0\0\
+\0\0\0\x01\0\0\0\x0a\0\0\0\x88\0\0\0\0\0\0\0\x01\0\0\0\x09\0\0\0\xd0\0\0\0\0\0\
+\0\0\x01\0\0\0\x0a\0\0\0\x1c\0\0\0\0\0\0\0\x01\0\0\0\x05\0\0\0\x38\0\0\0\0\0\0\
+\0\x01\0\0\0\x06\0\0\0\0\x2e\x74\x65\x78\x74\0\x70\x65\x72\x66\x5f\x65\x76\x65\
+\x6e\x74\x73\0\x6d\x61\x70\x73\0\x72\x65\x63\x6f\x72\x64\x73\0\x5f\x76\x65\x72\
+\x73\x69\x6f\x6e\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\x6f\x69\x6e\x74\x2f\
+\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\x78\x69\x74\x5f\x65\
+\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\x65\x78\x69\x74\x5f\x65\
+\x78\x65\x63\x76\x65\0\x2e\x72\x65\x6c\x74\x72\x61\x63\x65\x70\x6f\x69\x6e\x74\
+\x2f\x73\x79\x73\x63\x61\x6c\x6c\x73\x2f\x73\x79\x73\x5f\x65\x6e\x74\x65\x72\
+\x5f\x65\x78\x65\x63\x76\x65\0\x73\x79\x73\x63\x61\x6c\x6c\x5f\x65\x6e\x74\x65\
+\x72\x5f\x65\x78\x65\x63\x76\x65\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x72\
+\x65\x6c\x2e\x65\x68\x5f\x66\x72\x61\x6d\x65\0\x65\x78\x65\x63\x73\x6e\x6f\x6f\
+\x70\x5f\x6b\x65\x72\x6e\x2e\x63\0\x2e\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\
+\x6d\x74\x61\x62\0\x4c\x42\x42\x31\x5f\x34\0\x4c\x42\x42\x31\x5f\x33\0\x4c\x42\
+\x42\x30\x5f\x33\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xcb\0\0\0\
+\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe0\x04\0\0\0\0\0\0\xf0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x69\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\x40\0\0\0\0\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x65\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x70\
+\x04\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x0c\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\
+\0\0\0\0\0\0\x2d\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc0\x01\0\0\
+\0\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x29\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x04\0\0\0\0\0\0\x30\0\
+\0\0\0\0\0\0\x0c\0\0\0\x05\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x13\0\0\0\
+\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x02\0\0\0\0\0\0\x28\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa4\0\0\0\x01\0\0\0\x03\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe0\x02\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x21\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\xe4\x02\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\0\0\0\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\xe8\x02\0\0\0\0\0\0\x50\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\xac\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc0\x04\0\0\0\
+\0\0\0\x20\0\0\0\0\0\0\0\x0c\0\0\0\x0a\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\
+\0\xd3\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x38\x03\0\0\0\0\0\0\x38\
+\x01\0\0\0\0\0\0\x01\0\0\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
+
+	return 0;
+err:
+	bpf_object__destroy_skeleton(s);
+	return -1;
+}
+
+#endif /* __EXECSNOOP_KERN_SKEL_H__ */

execsnoop-kernel/execsnoop_share.cpp

@@ -0,0 +1,83 @@
+
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include <sys/resource.h>
+#include "execsnoop_kern_skel.h"
+#include "execsnoop_share.h"
+
+namespace CGPROXY::EXECSNOOP {
+
+#define PERF_BUFFER_PAGES 64
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+function<int(int)> callback = NULL;
+promise<void> status;
+
+static void handle_event(void *ctx, int cpu, void *data, __u32 size) {
+  	auto e = static_cast<event*>(data);
+  	if (callback) callback(e->pid);
+}
+
+void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) {
+	fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
+}
+
+int bump_memlock_rlimit(void) {
+	struct rlimit rlim_new = { RLIM_INFINITY, RLIM_INFINITY };
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
+int execsnoop() {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int err;
+	
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+	
+	struct execsnoop_kern *obj=execsnoop_kern__open_and_load();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	err = execsnoop_kern__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		return err;
+	}
+
+	pb_opts.sample_cb = handle_event;
+	pb_opts.lost_cb = handle_lost_events;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	err = libbpf_get_error(pb);
+	if (err) {
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
+	}
+
+	// notify
+  	status.set_value();
+
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return err;
+}
+
+void startThread(function<int(int)> c, promise<void> _status) {
+  status = move(_status);
+  callback = c;
+  execsnoop();
+}
+
+}

execsnoop-kernel/execsnoop_share.h

@@ -0,0 +1,20 @@
+#ifndef EXECSNOOP_SHARE_HPP
+#define EXECSNOOP_SHARE_HPP 1
+
+#include <functional>
+#include <future>
+#include <string>
+using namespace std;
+
+namespace CGPROXY::EXECSNOOP {
+
+extern "C" void startThread(function<int(int)> c, promise<void> _status);
+
+#ifdef BUIlD_EXECSNOOP_DL
+// only for dlsym()
+using startThread_t=decltype(startThread);
+startThread_t *_startThread;
+#endif
+
+} // namespace CGPROXY::EXECSNOOP
+#endif

execsnoop-kernel/execsnoop_user.c

@@ -0,0 +1,40 @@
+
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include "bpf_load.h"
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+static void print_bpf_output(void *ctx, int cpu, void *data, __u32 size) {
+	struct event *e=data;
+    printf("comm: %s, pid: %d, tgid: %d, ppid: %d, uid: %d\n",e->comm,e->pid,e->tgid,e->ppid,e->uid);
+}
+
+int main(int argc, char **argv) {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int ret;
+
+	if (load_bpf_file("execsnoop_kern.o")!=0) {
+		printf("%s", bpf_log_buf);
+		return 1;
+	}
+
+	pb_opts.sample_cb = print_bpf_output;
+	pb = perf_buffer__new(map_fd[1], 8, &pb_opts);
+	ret = libbpf_get_error(pb);
+	if (ret) {
+		printf("failed to setup perf_buffer: %d\n", ret);
+		return 1;
+	}
+
+	while ((ret = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return ret;
+}

execsnoop-kernel/execsnoop_user_1.c

@@ -0,0 +1,69 @@
+
+#include <signal.h>
+#include <bpf/libbpf.h>
+#include <sys/resource.h>
+#include "execsnoop_kern_skel.h"
+// #include "bpf_load.h"
+
+#define TASK_COMM_LEN 16
+struct event {
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	pid_t tgid;
+	pid_t ppid;
+	uid_t uid;
+};
+
+#define PERF_BUFFER_PAGES   64
+
+static void print_bpf_output(void *ctx, int cpu, void *data, __u32 size) {
+	struct event *e=data;
+    printf("comm: %s, pid: %d, tgid: %d, ppid: %d, uid: %d\n",e->comm,e->pid,e->tgid,e->ppid,e->uid);
+}
+
+int bump_memlock_rlimit(void)
+{
+	struct rlimit rlim_new = {
+		.rlim_cur	= RLIM_INFINITY,
+		.rlim_max	= RLIM_INFINITY,
+	};
+
+	return setrlimit(RLIMIT_MEMLOCK, &rlim_new);
+}
+
+
+int main(int argc, char **argv) {
+	struct perf_buffer_opts pb_opts = {};
+	struct perf_buffer *pb;
+	int err;
+	
+	err = bump_memlock_rlimit();
+	if (err) {
+		fprintf(stderr, "failed to increase rlimit: %d\n", err);
+		return 1;
+	}
+	
+	struct execsnoop_kern *obj=execsnoop_kern__open_and_load();
+	if (!obj) {
+		fprintf(stderr, "failed to open and/or load BPF object\n");
+		return 1;
+	}
+
+	err = execsnoop_kern__attach(obj);
+	if (err) {
+		fprintf(stderr, "failed to attach BPF programs\n");
+		return err;
+	}
+
+	pb_opts.sample_cb = print_bpf_output;
+	pb = perf_buffer__new(bpf_map__fd(obj->maps.perf_events), PERF_BUFFER_PAGES, &pb_opts);
+	err = libbpf_get_error(pb);
+	if (err) {
+		printf("failed to setup perf_buffer: %d\n", err);
+		return 1;
+	}
+
+	while ((err = perf_buffer__poll(pb, -1)) >= 0) {}
+	kill(0, SIGINT);
+	return err;
+}

execsnoop-kernel/readme.md

@@ -0,0 +1,119 @@
+
+## Prons and cons
+
+- use stable execve tracepoint, so build once and should work everywhere
+- need to build in kernel tree
+
+## Build in kernel tree
+
+- ready and config kernel tree
+
+```bash
+# kernel config
+gunzip -c /proc/config.gz > .config
+make oldconfig && make prepare
+# install headers to ./usr/include
+make headers_install -j8
+# build bpf
+make M=samples/bpf -j8
+```
+
+- put or link `execsnoop_kern.c` and `execsnoop_user.c` to *samples/bpf/*
+- edit  *samples/bpf/makefile*
+
+```makefile
+# in samples/bpf/makefile
+tprogs-y += execsnoop
+execsnoop-objs := bpf_load.o execsnoop_user.o $(TRACE_HELPERS)
+always-y += execsnoop_kern.o
+```
+
+## Run
+
+```bash
+cd samples/bpf
+sudo bash -c "ulimit -l unlimited && ./execsnoop"
+```
+
+## Detail build command
+
+using `make V=1 M=samples/bpf | tee -a log.txt` to get and filter following command
+
+- build `execsnoop_kern.o`
+
+  note `-g` is needed if with BPF CO-RE
+
+```bash
+clang -nostdinc \
+	-isystem /usr/lib/gcc/x86_64-pc-linux-gnu/10.1.0/include \
+	-I./arch/x86/include \
+	-I./arch/x86/include/generated  \
+	-I./include \
+	-I./arch/x86/include/uapi \
+	-I./arch/x86/include/generated/uapi \
+	-I./include/uapi \
+	-I./include/generated/uapi \
+	-include ./include/linux/kconfig.h \
+	-I./samples/bpf \
+	-I./tools/testing/selftests/bpf/ \
+	-I./tools/lib/ \
+	-include asm_goto_workaround.h \
+	-D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \
+	-D__TARGET_ARCH_x86 -Wno-compare-distinct-pointer-types \
+	-Wno-gnu-variable-sized-type-not-at-end \
+	-Wno-address-of-packed-member -Wno-tautological-compare \
+	-Wno-unknown-warning-option  \
+	-fno-stack-protector \
+	-O2 -emit-llvm -c samples/bpf/execsnoop_kern.c \
+	-o - | llc -march=bpf  -filetype=obj -o samples/bpf/execsnoop_kern.o
+```
+
+- build `execsnoop_user.o`
+
+```bash
+  gcc -Wall -O2 -Wmissing-prototypes -Wstrict-prototypes \
+  	-I./usr/include \
+  	-I./tools/testing/selftests/bpf/ \
+  	-I./tools/lib/ \
+  	-I./tools/include \
+  	-I./tools/perf \
+  	-DHAVE_ATTR_TEST=0  \
+  	-c -o samples/bpf/execsnoop_user.o samples/bpf/execsnoop_user.c
+```
+
+- build `execsnoop`
+
+```bash
+  gcc -Wall -O2 -Wmissing-prototypes -Wstrict-prototypes \
+	  -I./usr/include \
+	  -I./tools/testing/selftests/bpf/ \
+	  -I./tools/lib/ \
+	  -I./tools/include \
+	  -I./tools/perf \
+	  -DHAVE_ATTR_TEST=0 \
+	  -o samples/bpf/execsnoop \
+	  samples/bpf/bpf_load.o samples/bpf/execsnoop_user.o \
+	  tools/testing/selftests/bpf/trace_helpers.o tools/lib/bpf/libbpf.a \
+	  -lelf -lz
+```
+
+
+
+## With bpftool
+
+- gen
+
+```
+bpftool gen skeleton execsnoop_kern.o > execsnoop_kern_skel.h
+```
+
+- build
+
+```
+gcc -Wall -O2 execsnoop_user_1.c -o execsnoop -lbpf
+```
+
+## Some resources
+
+- [A thorough introduction to eBPF](https://lwn.net/Articles/740157/)
+- [Write eBPF program in pure C](http://terenceli.github.io/%E6%8A%80%E6%9C%AF/2020/01/18/ebpf-in-c)

execsnoop-libbpf/Makefile

@@ -67,12 +67,12 @@ $(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) vmlinux.h | $(OUTPUT)
 vmlinux.h:
 	$(Q)$(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
 
-$(LIBBPF_OBJ): patch
+$(LIBBPF_OBJ):
-	$(Q)cd $(LIBBPF_DIR) && make DESTDIR=${OUTPUT} prefix=$(OUTPUT) install install_headers
+	$(Q)cd $(LIBBPF_DIR) && make DESTDIR=${OUTPUT} prefix=$(OUTPUT) CPPFLAGS="-fPIC" install install_headers
 
-patch: | $(LIBBPF_DIR)/Makefile
+# patch: | $(LIBBPF_DIR)/Makefile
-	@echo "patching libbpf-fPIC.patch"
+# 	@echo "patching libbpf-fPIC.patch"
-	$(Q)cd libbpf/src && patch -p1 --forward -i ../../libbpf-fPIC.patch || true
+# 	$(Q)cd libbpf/src && patch -p1 --forward -i ../../libbpf-fPIC.patch || true
 
 # build libexecsnoop.so
 $(LIBSO): execsnoop_share.cpp $(OUTPUT)/execsnoop.skel.h $(LIBBPF_OBJ) $(COMMON_OBJ) | $(OUTPUT) 

execsnoop-libbpf/readme.md

@@ -1,11 +1,21 @@
 
-## Depency
+## Resource
-- libbpf
-  - /usr/lib/libbpf.a
-  - some head file
-- bpf
-  - `/usr/bin/bpftool` to generate skeleton and dump btf
 
-## Refer
+- [libbpf](https://github.com/libbpf/libbpf)
+- [libbpf-tools](https://github.com/iovisor/bcc/tree/master/libbpf-tools) examples use libbpf
+- [bpftool](https://www.archlinux.org/packages/community/x86_64/bpf/) to generate skeleton and dump btf
 
-- https://facebookmicrosites.github.io/bpf/blog/2020/02/19/bpf-portability-and-co-re.html
+## Prons
+
+- BPF CO-RE (Compile Once – Run Everywhere)
+- small memory usage
+
+## Cons
+
+- `vmlinux.h`does not contain `#define` etc. And often causes confilct with other headers to cause redifinition error
+- comment in code for different types is gone
+- need kernel built with `CONFIG_DEBUG_INFO_BTF=y`, while ubuntu 20.04 still not
+
+## Build
+
+- see *makefile*

pack/CMakeLists.txt

@@ -5,7 +5,7 @@ set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "cgproxy will transparent proxy anything r
 
 ## deb pack
 set(CPACK_DEBIAN_PACKAGE_NAME "cgproxy")
-set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "x86_64")
+set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "amd64")
 set(CPACK_DEBIAN_PACKAGE_DEPENDS "systemd")
 set(CPACK_DEBIAN_PACKAGE_SECTION "network")
 set(CPACK_DEBIAN_PACKAGE_PRIORITY "Optional")

readme.md

@@ -1,6 +1,6 @@
 
 
-# Transparent Proxy powered with cgroup v2
+# Transparent Proxy powered by cgroup v2
 
 
 
@@ -16,9 +16,9 @@ Main feature:
 ## Contents
 
 <!--ts-->
-
+   * [Transparent Proxy powered by cgroup v2](#transparent-proxy-powered-by-cgroup-v2)
-   * [Transparent Proxy with cgroup v2](#transparent-proxy-with-cgroup-v2)
       * [Introduction](#introduction)
+      * [Contents](#contents)
       * [Prerequest](#prerequest)
       * [How to install](#how-to-install)
       * [Default usage](#default-usage)
@@ -29,8 +29,9 @@ Main feature:
       * [NOTES](#notes)
       * [TIPS](#tips)
       * [Licences](#licences)
+      * [Known Issus](#known-issus)
 
-<!-- Added by: fancy, at: Sat 16 May 2020 03:12:07 PM HKT -->
+<!-- Added by: fancy, at: Sat 04 Jul 2020 03:52:07 PM CST -->
 
 <!--te-->
 
@@ -52,17 +53,35 @@ Main feature:
 
   ubuntu 16.04, debian 9, fedora 27 and later are desired
 
-## How to install
+## How to build and install
+
+### distro install
+
+- For debian and redhat series, download from [Release page](https://github.com/springzfx/cgproxy/releases)
+
+- For archlinux series, see[archlinux AUR](https://aur.archlinux.org/packages/?K=cgproxy)
+
+- **Tested on  archlinux, fedora 32, ubuntu 18.04, ubuntu 20.04,  deepin 15.11, deepin v20 beta**
+
+### build
+
+- before build, install depencies: clang, nlohmann-json, libbpf
+- then cmake standard build
 
 ```bash
-cd execsnoop-libbpf && make libexecsnoop.so
+# ready build dir
-mkdir build && cd build && cmake .. && make install
+mkdir build
+cd build
+# generate
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=/usr \
+      -Dbuild_execsnoop_dl=ON \
+      -Dbuild_static=OFF \
+      ..
+# compile
+make
 ```
 
-- It is alreay in [archlinux AUR](https://aur.archlinux.org/packages/?K=cgproxy). 
-
-- DEB and RPM are packaged in [release page](https://github.com/springzfx/cgproxy/releases).
-
 ## Default usage
 
 - First enable and start service
@@ -96,16 +115,20 @@ Config file: **/etc/cgproxy/config.json**
 {
     "port": 12345,
     "program_noproxy": ["v2ray", "qv2ray"],
-    "program_proxy": [ ],
+    "program_proxy": [],
     "cgroup_noproxy": ["/system.slice/v2ray.service"],
-    "cgroup_proxy": [ ],
+    "cgroup_proxy": [],
     "enable_gateway": false,
     "enable_dns": true,
     "enable_udp": true,
     "enable_tcp": true,
     "enable_ipv4": true,
-    "enable_ipv6": true
+    "enable_ipv6": true,
+    "table": 10007,
+    "fwmark": 39283,
+    "mark_newin": 39271
 }
+
 ```
 
 - **port** tproxy listenning port
@@ -132,11 +155,14 @@ Config file: **/etc/cgproxy/config.json**
 
 - **enable_ipv6**
 
+- **table**, **fwmark**, **mark_newin**  you can specify iptables and route table related parameter in case conflict.
+
 - options priority
 
   ```
   program_noproxy > program_proxy > cgroup_noproxy > cgroup_proxy
   enable_ipv6 > enable_ipv4 > enable_tcp > enable_udp > enable_dns
+  ommand cgproxy and cgnoproxy always have highest priority
   ```
 
 **Note**: cgroup in configuration need to be exist, otherwise ignored

src/CMakeLists.txt

@@ -1,15 +1,24 @@
-find_package(Threads REQUIRED)
 find_package(nlohmann_json REQUIRED)
 include_directories(${PROJECT_SOURCE_DIR})
-include_directories(${PROJECT_SOURCE_DIR}/execsnoop-libbpf/)
+include_directories(${PROJECT_SOURCE_DIR}/execsnoop-kernel/)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
-add_executable(main main.cpp
-                    common.cpp config.cpp cgroup_attach.cpp
-                    socket_client.cpp socket_server.cpp)
-target_link_libraries(main PRIVATE nlohmann_json::nlohmann_json Threads::Threads ${CMAKE_DL_LIBS})
-set_target_properties(main PROPERTIES LINKER_LANGUAGE CXX)
-set_target_properties(main PROPERTIES OUTPUT_NAME cgproxy)
 
-install(TARGETS main DESTINATION /usr/bin PERMISSIONS ${basic_permission})
+if (build_execsnoop_dl)
-install(FILES ../execsnoop-libbpf/libexecsnoop.so DESTINATION /usr/lib/cgproxy/ PERMISSIONS ${basic_permission})
+    add_definitions(-DBUIlD_EXECSNOOP_DL)
+    set(DL_LIB "-ldl")
+    set(EXECSNOOP_LIB "")
+else()
+    set(EXECSNOOP_LIB "execsnoop")
+endif()
+
+add_executable(main main.cpp common.cpp config.cpp cgroup_attach.cpp socket_client.cpp socket_server.cpp)
+target_link_libraries(main PRIVATE nlohmann_json::nlohmann_json ${DL_LIB} ${EXECSNOOP_LIB})
+set_target_properties(main PROPERTIES OUTPUT_NAME cgproxy)
+install(TARGETS main RUNTIME)
+
+if (build_static)
+    target_link_libraries(main PRIVATE -static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive)
+else()
+    target_link_libraries(main PRIVATE -lpthread)
+endif()

src/cgproxyd.hpp

@@ -26,6 +26,7 @@ using namespace ::CGPROXY::CONFIG;
 using namespace ::CGPROXY::CGROUP;
 // using namespace ::CGPROXY::EXECSNOOP;
 
+#ifdef BUIlD_EXECSNOOP_DL
 namespace CGPROXY::EXECSNOOP {
 bool loadExecsnoopLib() {
   try {
@@ -48,6 +49,7 @@ bool loadExecsnoopLib() {
   }
 }
 } // namespace CGPROXY::EXECSNOOP
+#endif
 
 namespace CGPROXY::CGPROXYD {
 
@@ -232,14 +234,21 @@ class cgproxyd {
   }
 
   void startExecsnoopThread() {
+    #ifdef BUIlD_EXECSNOOP_DL
     if (!EXECSNOOP::loadExecsnoopLib() || EXECSNOOP::_startThread == NULL) {
-      error("execsnoop not ready to start, maybe bcc not installed");
+      error("execsnoop not ready to start, maybe missing libbpf");
       return;
     }
+    #endif
 
     promise<void> status;
     future<void> status_f = status.get_future();
+    #ifdef BUIlD_EXECSNOOP_DL
     thread th(EXECSNOOP::_startThread, handle_pid_static, move(status));
+    #else
+    thread th(EXECSNOOP::startThread, handle_pid_static, move(status));
+    #endif
+
     execsnoop_thread = move(th);
 
     future_status fstatus = status_f.wait_for(chrono::seconds(THREAD_TIMEOUT));
@@ -300,7 +309,10 @@ public:
 
     assignStaticInstance();
 
-    config.loadFromFile(DEFAULT_CONFIG_FILE);
+    if (config.loadFromFile(DEFAULT_CONFIG_FILE)!=SUCCESS) {
+      error("load config file failed");
+      return -1;
+    }
     applyConfig();
 
     if (enable_socketserver) startSocketListeningThread();
@@ -311,6 +323,7 @@ public:
 
     return 0;
   }
+
   int applyConfig() {
     system(TPROXY_IPTABLS_CLEAN);
     config.print_summary();

src/cgroup_attach.cpp

@@ -14,19 +14,8 @@
 
 namespace CGPROXY::CGROUP {
 
-string cgroup2_mount_point = get_cgroup2_mount_point();
+string cgroup2_mount_point = CGROUP2_MOUNT_POINT;
 
-string get_cgroup2_mount_point() {
-  stringstream buffer;
-  unique_ptr<FILE, decltype(&pclose)> fp(popen("findmnt -t cgroup2 -n -o TARGET", "r"),
-                                         &pclose);
-  if (!fp) return "";
-  char buf[READ_SIZE_MAX];
-  while (fgets(buf, READ_SIZE_MAX, fp.get()) != NULL) { buffer << buf; }
-  string s = buffer.str();
-  s.pop_back(); // remove newline character
-  return s;
-}
 
 bool validate(string pid, string cgroup) {
   bool pid_v = validPid(pid);

src/cgroup_attach.h

@@ -8,7 +8,6 @@ using namespace std;
 namespace CGPROXY::CGROUP {
 extern string cgroup2_mount_point;
 bool validate(string pid, string cgroup);
-string get_cgroup2_mount_point();
 int attach(const string pid, const string cgroup_target);
 int attach(const int pid, const string cgroup_target);
 int write2procs(string pid, string procspath);

src/common.cpp

@@ -67,7 +67,7 @@ string bash_which(const string &name) {
   char buf[READ_SIZE_MAX];
   while (fgets(buf, READ_SIZE_MAX, fp.get()) != NULL) { buffer << buf; }
   string s = buffer.str();
-  s.pop_back(); // remove newline character
+  if (!s.empty()) s.pop_back(); // remove newline character
   return s;
 }
 
@@ -79,7 +79,7 @@ string bash_readlink(const string &path) {
   char buf[READ_SIZE_MAX];
   while (fgets(buf, READ_SIZE_MAX, fp.get()) != NULL) { buffer << buf; }
   string s = buffer.str();
-  s.pop_back(); // remove newline character
+  if (!s.empty()) s.pop_back(); // remove newline character
   return s;
 }
 

src/common.h

@@ -11,6 +11,7 @@ using namespace std;
 #define TPROXY_IPTABLS_CLEAN "/usr/share/cgproxy/scripts/cgroup-tproxy.sh stop"
 
 #define LIBEXECSNOOP_SO "/usr/lib/cgproxy/libexecsnoop.so"
+#define CGROUP2_MOUNT_POINT "/var/run/cgproxy/cgroup2"
 #define PID_LOCK_FILE "/var/run/cgproxyd.pid"
 #define SOCKET_PATH "/tmp/cgproxy_unix_socket"
 #define LISTEN_BACKLOG 64
@@ -49,6 +50,14 @@ extern bool enable_info;
     fflush(stderr);                                                                      \
   }
 
+#define warning(...)                                                                       \
+  {                                                                                      \
+    fprintf(stderr, "warning: ");                                                          \
+    fprintf(stderr, __VA_ARGS__);                                                        \
+    fprintf(stderr, "\n");                                                               \
+    fflush(stderr);                                                                      \
+  }
+
 #define debug(...)                                                                       \
   if (enable_debug) {                                                                    \
     fprintf(stdout, "debug: ");                                                          \

src/config.cpp

@@ -21,6 +21,7 @@ using json = nlohmann::json;
 namespace CGPROXY::CONFIG {
 
 void Config::toEnv() {
+  setenv("cgroup_mount_point", CGROUP2_MOUNT_POINT, 1);
   setenv("program_proxy", join2str(program_proxy, ':').c_str(), 1);
   setenv("program_noproxy", join2str(program_noproxy, ':').c_str(), 1);
   setenv("cgroup_proxy", join2str(cgroup_proxy, ':').c_str(), 1);
@@ -32,6 +33,9 @@ void Config::toEnv() {
   setenv("enable_udp", to_str(enable_udp).c_str(), 1);
   setenv("enable_ipv4", to_str(enable_ipv4).c_str(), 1);
   setenv("enable_ipv6", to_str(enable_ipv6).c_str(), 1);
+  setenv("table", to_str(table).c_str(), 1);
+  setenv("fwmark", to_str(fwmark).c_str(), 1);
+  setenv("mark_newin", to_str(mark_newin).c_str(), 1);
 }
 
 int Config::saveToFile(const string f) {
@@ -56,6 +60,9 @@ string Config::toJsonStr() {
   add2json(enable_udp);
   add2json(enable_ipv4);
   add2json(enable_ipv6);
+  add2json(table);
+  add2json(fwmark);
+  add2json(mark_newin);
   return j.dump();
 }
 
@@ -89,6 +96,9 @@ int Config::loadFromJsonStr(const string js) {
   tryassign(enable_udp);
   tryassign(enable_ipv4);
   tryassign(enable_ipv6);
+  tryassign(table);
+  tryassign(fwmark);
+  tryassign(mark_newin);
 
   // e.g. v2ray -> /usr/bin/v2ray -> /usr/lib/v2ray/v2ray
   toRealProgramPath(program_noproxy);
@@ -109,7 +119,7 @@ bool Config::validateJsonStr(const string js) {
   bool status = true;
   const set<string> boolset = {"enable_gateway", "enable_dns",  "enable_tcp",
                                "enable_udp",     "enable_ipv4", "enable_ipv6"};
-  const set<string> allowset = {"program_proxy", "program_noproxy"};
+  const set<string> allowset = {"program_proxy", "program_noproxy", "comment", "table", "fwmark", "mark_newin"};
   for (auto &[key, value] : j.items()) {
     if (key == "cgroup_proxy" || key == "cgroup_noproxy") {
       if (value.is_string() && !validCgroup((string)value)) status = false;
@@ -139,6 +149,7 @@ void Config::print_summary() {
   info("proxied program: %s", join2str(program_proxy).c_str());
   info("noproxy cgroup: %s", join2str(cgroup_noproxy).c_str());
   info("proxied cgroup: %s", join2str(cgroup_proxy).c_str());
+  info("table: %d, fwmark: %d, mark_newin: %d", table, fwmark, mark_newin);
 }
 
 void Config::toRealProgramPath(vector<string> &v) {
@@ -147,7 +158,7 @@ void Config::toRealProgramPath(vector<string> &v) {
     auto rpath = getRealExistPath(p);
     if (!rpath.empty()) tmp.push_back(rpath);
     else
-      error("%s not exist or broken link", p.c_str());
+      warning("%s not exist or broken link", p.c_str());
   }
   v = tmp;
 }

src/config.h

@@ -25,6 +25,11 @@ public:
   bool enable_ipv4 = true;
   bool enable_ipv6 = true;
 
+  // for iptables
+  int table=10007;
+  int fwmark=0x9973;
+  int mark_newin=0x9967;
+
   void toEnv();
   int saveToFile(const string f);
   string toJsonStr();

tools/CMakeLists.txt

@@ -2,11 +2,4 @@ include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/src)
 
 add_executable(cgattach cgattach.cpp ../src/cgroup_attach.cpp ../src/common.cpp)
-install(TARGETS cgattach  DESTINATION /usr/bin PERMISSIONS ${basic_permission})
+install(TARGETS cgattach  DESTINATION /usr/bin PERMISSIONS ${basic_permission})
-
-if (with_execsnoop)
-add_executable(execsnoop_exec execsnoop.cpp ../src/common.cpp ../src/execsnoop.cpp)
-set_target_properties(execsnoop_exec PROPERTIES OUTPUT_NAME execsnoop)
-target_link_libraries(execsnoop_exec bcc)
-install(TARGETS execsnoop_exec  DESTINATION /usr/bin PERMISSIONS ${basic_permission})
-endif()

tools/execsnoop.cpp

@@ -1,24 +0,0 @@
-#include "execsnoop.h"
-#include "common.h"
-#include <unistd.h>
-using namespace std;
-using namespace CGPROXY::EXECSNOOP;
-
-#define PATH_MAX_LEN 128
-
-int handle_pid(int pid) {
-  char path[PATH_MAX_LEN];
-  auto size = readlink(to_str("/proc/", pid, "/exe").c_str(), path, PATH_MAX_LEN);
-  if (size == -1) error("readlink: %s", to_str("/proc/", pid, "/exe").c_str());
-  path[size] = '\0';
-  info("%d %s", pid, path);
-  return 0;
-}
-
-int main() {
-  enable_debug = true;
-  enable_info = true;
-  callback = handle_pid;
-  execsnoop();
-  return 0;
-}

v2ray_config/05_inbounds_04_socks5.json

@@ -5,6 +5,7 @@
       "port": 1080,
       "protocol": "socks",
       "settings": {
+        "udp": true,
         "auth": "noauth",
         "userLevel": 0
       },