From f501c7e4764353b49af51ec0fac771030ce043fc Mon Sep 17 00:00:00 2001 From: springzfx Date: Sun, 24 May 2020 23:21:06 +0800 Subject: [PATCH] add execsnoop in c++ --- CMakeLists.txt | 1 - execsnoop.py | 130 ------------------------------------------ src/CMakeLists.txt | 2 +- src/cgproxyd.hpp | 114 +++++++++++++++++++++++++++--------- src/cgroup_attach.cpp | 46 +++++++-------- src/cgroup_attach.h | 4 +- src/common.cpp | 58 +++++++++++++++++++ src/common.h | 22 +++++-- src/config.cpp | 23 ++++++++ src/config.h | 2 + src/execsnoop.hpp | 111 ++++++++++++++++++++++++++++++++++++ src/socket_server.cpp | 2 +- src/socket_server.h | 2 +- tools/CMakeLists.txt | 9 ++- tools/execsnoop.cpp | 23 ++++++++ 15 files changed, 355 insertions(+), 194 deletions(-) delete mode 100644 execsnoop.py create mode 100644 src/execsnoop.hpp create mode 100644 tools/execsnoop.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b40b30e..270bed5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,6 @@ install(FILES cgnoproxy DESTINATION /usr/bin PERMISSIONS ${basic_permission}) install(FILES cgproxy.service DESTINATION /usr/lib/systemd/system/) install(FILES config.json DESTINATION /etc/cgproxy/) install(FILES cgroup-tproxy.sh DESTINATION /usr/share/cgproxy/scripts/ PERMISSIONS ${basic_permission}) -install(FILES execsnoop.py DESTINATION /usr/share/cgproxy/scripts/ PERMISSIONS ${basic_permission}) install(FILES readme.md DESTINATION /usr/share/doc/cgproxy/) # man pages diff --git a/execsnoop.py b/execsnoop.py deleted file mode 100644 index f1dda27..0000000 --- a/execsnoop.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/python -# This won't catch all new processes: an application may fork() but not exec(). - -from __future__ import print_function -import os, sys, signal, shutil -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - -try: - from bcc import BPF -except: - eprint("python-bcc not installed") - exit(0) - -# define BPF program -bpf_text = """ -#include -#include -#include - -struct data_t { - u32 pid; -}; - -BPF_PERF_OUTPUT(events); - -int syscall__execve(struct pt_regs *ctx, - const char __user *filename, - const char __user *const __user *__argv, - const char __user *const __user *__envp) -{ - struct data_t data = {}; - data.pid = bpf_get_current_pid_tgid() >> 32; - events.perf_submit(ctx, &data, sizeof(struct data_t)); - return 0; -} -""" - -def getRealPath(exec_path): - path=exec_path.strip() - if not path.startswith("/"): - path=shutil.which(path) - if path: - path=os.path.realpath(path) - if os.path.isfile(path): return path - eprint("'{0}' not exist or broken link".format(exec_path)) - -def getParam(): - global exec_path_proxy, exec_path_noproxy - exec_path_str=os.getenv('program_proxy') - if exec_path_str: - paths=exec_path_str.split(':') - exec_path_proxy=[getRealPath(x) for x in paths] - print("program with proxy:", end =" ") - print(*exec_path_proxy,flush=True) - - exec_path_str=os.getenv('program_noproxy') - if exec_path_str: - paths=exec_path_str.split(':') - exec_path_noproxy=[getRealPath(x) for x in paths] - print("program without proxy:", end =" ") - print(*exec_path_noproxy, flush=True) - -def exit_gracefully(signum, frame): - eprint("execsnoop receive signal: {0}".format(signum),flush=True) - sys.exit(0) - -def attach(pid, path, proxy=True): - if proxy: - print("proxy: %-6d %s" % (pid, path),flush=True) - os.system("/usr/bin/cgproxy --pid {0}".format(pid)) - else: - print("noproxy: %-6d %s" % (pid, path),flush=True) - os.system("/usr/bin/cgproxy --pid {0} --noproxy".format(pid)) - -def processAlreadyRunning(): - from subprocess import check_output - def get_pid(name): - try: - return map(int,check_output(["pidof",name]).split()) - except: - return [] - global exec_path_proxy, exec_path_noproxy - for path in exec_path_proxy: - for pid in get_pid(path): - attach(pid,path,True) - for path in exec_path_noproxy: - for pid in get_pid(path): - attach(pid,path,False) - -signal.signal(signal.SIGINT, exit_gracefully) -signal.signal(signal.SIGHUP, exit_gracefully) -signal.signal(signal.SIGTERM, exit_gracefully) - -debug=False -if (len(sys.argv)>1 and sys.argv[1]=="--debug"): - debug=True - -exec_path_proxy=[] -exec_path_noproxy=[] -getParam() -processAlreadyRunning() - -# initialize BPF -b = BPF(text=bpf_text) -execve_fnname = b.get_syscall_fnname("execve") -b.attach_kprobe(event=execve_fnname, fn_name="syscall__execve") - -# process event -def print_event(cpu, data, size): - event = b["events"].event(data) - pid=event.pid - try: - exec_path=os.readlink("/proc/{0}/exe".format(pid)) - except: # in case process exit too early - if (debug): - print("process exit too early: {0}".format(pid)) - return - if (exec_path in exec_path_noproxy): - attach(pid, exec_path, False) - elif (exec_path in exec_path_proxy): - attach(pid, exec_path, True) - elif (debug): - print("debug: %d %s" % (pid, exec_path),flush=True) - - -# loop with callback to print_event -b["events"].open_perf_buffer(print_event) -while 1: - b.perf_buffer_poll() \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 625ae58..4cc5108 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,7 @@ add_executable(main main.cpp common.cpp config.cpp cgroup_attach.cpp socket_client.cpp socket_server.cpp) -target_link_libraries(main nlohmann_json::nlohmann_json Threads::Threads) +target_link_libraries(main nlohmann_json::nlohmann_json Threads::Threads bcc) set_target_properties(main PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(main PROPERTIES OUTPUT_NAME cgproxy) diff --git a/src/cgproxyd.hpp b/src/cgproxyd.hpp index 706e7ec..f36055d 100644 --- a/src/cgproxyd.hpp +++ b/src/cgproxyd.hpp @@ -4,10 +4,14 @@ #include "cgroup_attach.h" #include "common.h" #include "config.h" +#include "execsnoop.hpp" #include "socket_server.h" +#include #include +#include #include #include +#include #include #include #include @@ -17,17 +21,22 @@ using json = nlohmann::json; using namespace ::CGPROXY::SOCKET; using namespace ::CGPROXY::CONFIG; using namespace ::CGPROXY::CGROUP; +using namespace ::CGPROXY::EXESNOOP; namespace CGPROXY::CGPROXYD { bool print_help = false; +bool enable_socketserver = true; bool enable_execsnoop = false; class cgproxyd { - thread_arg arg_t; - Config config; + SOCKET::thread_arg socketserver_thread_arg; pthread_t socket_thread_id = -1; - pid_t exec_snoop_pid = -1; + + EXESNOOP::thread_arg execsnoop_thread_arg; + pthread_t execsnoop_thread_id = -1; + + Config config; static cgproxyd *instance; static int handle_msg_static(char *msg) { @@ -38,6 +47,40 @@ class cgproxyd { return instance->handle_msg(msg); } + static int handle_pid_static(int pid) { + if (!instance) { + error("no cgproxyd instance assigned"); + return ERROR; + } + return instance->handle_pid(pid); + } + + int handle_pid(int pid) { + auto path=realpath(to_str("/proc/",pid,"/exe").c_str(), NULL); + if (path==NULL) { + debug("pid %d live life too short", pid); + return 0; + } + debug("execsnoop: %d %s", pid, path); + + vector v; + + v = config.program_noproxy; + if (find(v.begin(), v.end(), path) != v.end()) { + info("exesnoop noproxy: %d %s", pid, path); + free(path); + return attach(pid, config.cgroup_noproxy_preserved); + } + v = config.program_proxy; + if (find(v.begin(), v.end(), path) != v.end()) { + info("exesnoop proxied: %d %s", pid, path); + free(path); + return attach(pid, config.cgroup_proxy_preserved); + } + free(path); + return 0; + } + static void signalHandler(int signum) { debug("Signal %d received.", signum); if (!instance) { @@ -85,12 +128,12 @@ class cgproxyd { switch (type) { case MSG_TYPE_CONFIG_JSON: status = config.loadFromJsonStr(j.at("data").dump()); - if (status == SUCCESS) status = applyConfig(&config); + if (status == SUCCESS) status = applyConfig(); return status; break; case MSG_TYPE_CONFIG_PATH: status = config.loadFromFile(j.at("data").get()); - if (status == SUCCESS) status = applyConfig(&config); + if (status == SUCCESS) status = applyConfig(); return status; break; case MSG_TYPE_PROXY_PID: @@ -111,27 +154,35 @@ class cgproxyd { } pthread_t startSocketListeningThread() { - arg_t.handle_msg = &handle_msg_static; + socketserver_thread_arg.handle_msg = &handle_msg_static; pthread_t thread_id; - int status = pthread_create(&thread_id, NULL, &SocketServer::startThread, &arg_t); + int status = + pthread_create(&thread_id, NULL, &SOCKET::startThread, &socketserver_thread_arg); if (status != 0) error("socket thread create failed"); return thread_id; } - void startExecSnoopProc() { - if (exec_snoop_pid != -1){ - kill(exec_snoop_pid, SIGINT); - exec_snoop_pid=-1; - } - pid_t pid = fork(); - if (pid == 0) { - execl(BPF_EXEC_SNOOP_START, (char *) NULL); - exit(0); - } else if (pid<0){ - error("fork precess failed"); - }else { - exec_snoop_pid = pid; - } + pthread_t startExecSnoopThread() { + execsnoop_thread_arg.handle_pid = &handle_pid_static; + pthread_t thread_id; + int status = + pthread_create(&thread_id, NULL, &EXESNOOP::startThread, &execsnoop_thread_arg); + if (status != 0) error("execsnoop thread create failed"); + return thread_id; + } + + void processRunningProgram(){ + debug("process running program") + for (auto &path:config.program_noproxy) + for (auto &pid:bash_pidof(path)){ + int status=attach(pid, config.cgroup_noproxy_preserved); + if (status==0) info("noproxy running process %d %s",pid, path.c_str()); + } + for (auto &path:config.program_proxy) + for (auto &pid:bash_pidof(path)){ + int status=attach(pid, config.cgroup_proxy_preserved); + if (status==0) info("proxied running process %d %s",pid, path.c_str()); + } } void assignStaticInstance() { instance = this; } @@ -143,19 +194,26 @@ public: signal(SIGTERM, &signalHandler); signal(SIGHUP, &signalHandler); - config.loadFromFile(DEFAULT_CONFIG_FILE); - applyConfig(&config); - assignStaticInstance(); - socket_thread_id = startSocketListeningThread(); + + config.loadFromFile(DEFAULT_CONFIG_FILE); + applyConfig(); + processRunningProgram(); + + if (enable_socketserver) { socket_thread_id = startSocketListeningThread(); } + if (enable_execsnoop) { execsnoop_thread_id = startExecSnoopThread(); } + + cout<toEnv(); + config.print_summary(); + config.toEnv(); system(TPROXY_IPTABLS_START); - if (enable_execsnoop) startExecSnoopProc(); // no need to track running status return 0; } diff --git a/src/cgroup_attach.cpp b/src/cgroup_attach.cpp index d2b1df0..ac06585 100644 --- a/src/cgroup_attach.cpp +++ b/src/cgroup_attach.cpp @@ -13,48 +13,46 @@ namespace CGPROXY::CGROUP { +string cgroup2_mount_point = get_cgroup2_mount_point(); + bool exist(string path) { struct stat st; if (stat(path.c_str(), &st) != -1) { return S_ISDIR(st.st_mode); } return false; } +string get_cgroup2_mount_point() { + stringstream buffer; + FILE *fp = popen("findmnt -t cgroup2 -n -o TARGET", "r"); + if (!fp) return ""; + char buf[64]; while (fgets(buf,64,fp)!=NULL) { buffer< +#include #include +#include +#include +#include bool enable_debug = false; +bool enable_info = true; string join2str(const vector t, const char delm) { string s; @@ -32,3 +38,55 @@ bool validCgroup(const vector cgroup) { bool validPid(const string pid) { return regex_match(pid, regex("^[0-9]+$")); } bool validPort(const int port) { return port > 0; } + +bool fileExist(const string &path) { + struct stat st; + return (stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode)); +} + +bool dirExist(const string &path) { + struct stat st; + return (stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode)); +} + +vector bash_pidof(const string &path){ + vector pids; + FILE *fp = popen(to_str("pidof ", path).c_str(), "r"); + if (!fp) return pids; + int pid; + char buf[64]; while (fscanf(fp,"%d",&pid)!=EOF) { pids.push_back(pid); } + pclose(fp); + return pids; +} + +string bash_which(const string &name) { + stringstream buffer; + FILE *fp = popen(to_str("which ", name).c_str(), "r"); + if (!fp) return ""; + char buf[64]; while (fgets(buf,64,fp)!=NULL) { buffer< string to_str(T... args) { stringstream ss; @@ -70,4 +77,11 @@ bool validCgroup(const vector cgroup); bool validPid(const string pid); bool validPort(const int port); +bool fileExist(const string &path); +bool dirExist(const string &path); +vector bash_pidof(const string &path); +string bash_which(const string &name); +string bash_readlink(const string &path); +string getRealExistPath(const string &name); + #endif \ No newline at end of file diff --git a/src/config.cpp b/src/config.cpp index 088abb5..c545143 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -4,6 +4,7 @@ #include #include #include +#include using json = nlohmann::json; #define add2json(v) j[#v] = v; @@ -89,6 +90,11 @@ int Config::loadFromJsonStr(const string js) { tryassign(enable_udp); tryassign(enable_ipv4); tryassign(enable_ipv6); + + // e.g. v2ray -> /usr/bin/v2ray -> /usr/lib/v2ray/v2ray + toRealProgramPath(program_noproxy); + toRealProgramPath(program_proxy); + return 0; } @@ -127,4 +133,21 @@ bool Config::validateJsonStr(const string js) { return true; } +void Config::print_summary() { + info("noproxy program: %s", join2str(program_noproxy).c_str()); + info("proxied program: %s", join2str(program_proxy).c_str()); + info("noproxy cgroup: %s", join2str(cgroup_noproxy).c_str()); + info("proxied cgroup: %s", join2str(cgroup_proxy).c_str()); +} + +void Config::toRealProgramPath(vector &v) { + vector tmp; + for (auto &p : v) { + auto rpath = getRealExistPath(p); + if (!rpath.empty()) tmp.push_back(rpath); + else error("%s not exist or broken link", p.c_str()); + } + v=tmp; +} + } // namespace CGPROXY::CONFIG \ No newline at end of file diff --git a/src/config.h b/src/config.h index 399ecfc..1da4ccf 100644 --- a/src/config.h +++ b/src/config.h @@ -30,10 +30,12 @@ public: string toJsonStr(); int loadFromFile(const string f); int loadFromJsonStr(const string js); + void print_summary(); private: void mergeReserved(); bool validateJsonStr(const string js); + void toRealProgramPath(vector &v); }; } // namespace CGPROXY::CONFIG diff --git a/src/execsnoop.hpp b/src/execsnoop.hpp new file mode 100644 index 0000000..1df2714 --- /dev/null +++ b/src/execsnoop.hpp @@ -0,0 +1,111 @@ +#ifndef EXECSNOOP_HPP +#define EXECSNOOP_HPP 1 + +#include +#include "bcc/BPF.h" +#include "common.h" +#include +#include +#include +#include +#include +#include +using namespace std; + +namespace CGPROXY::EXESNOOP { + +const string BPF_PROGRAM = R"( +#include +#include +#include + +struct data_t { + int pid; +}; + +BPF_PERF_OUTPUT(events); + +int syscall_execve(struct pt_regs *ctx, + const char __user *filename, + const char __user *const __user *__argv, + const char __user *const __user *__envp) +{ + struct data_t data = {}; + data.pid = bpf_get_current_pid_tgid(); + events.perf_submit(ctx, &data, sizeof(struct data_t)); + return 0; +} + +int ret_syscall_execve(struct pt_regs *ctx){ + struct data_t data = {}; + data.pid = bpf_get_current_pid_tgid(); + int retval = PT_REGS_RC(ctx); + if (retval==0) + events.perf_submit(ctx, &data, sizeof(struct data_t)); + return 0; +} +)"; + +struct data_t { + int pid; +}; + +function callback = NULL; + +void handle_events(void *cb_cookie, void *data, int data_size) { + auto event = static_cast(data); + int pid = event->pid; + + if (callback) callback(pid); +} + +int execsnoop() { + debug("starting execsnoop"); + ebpf::BPF bpf; + auto init_res = bpf.init(BPF_PROGRAM); + if (init_res.code() != 0) { + std::cerr << init_res.msg() << std::endl; + return 1; + } + + string execve_fnname = bpf.get_syscall_fnname("execve"); + // auto attach_res = bpf.attach_kprobe(execve_fnname, "syscall_execve"); + auto attach_res = + bpf.attach_kprobe(execve_fnname, "ret_syscall_execve", 0, BPF_PROBE_RETURN); + if (attach_res.code() != 0) { + std::cerr << attach_res.msg() << std::endl; + return 1; + } + + auto open_res = bpf.open_perf_buffer("events", &handle_events); + if (open_res.code() != 0) { + std::cerr << open_res.msg() << std::endl; + return 1; + } + + if (bpf.free_bcc_memory()) { + std::cerr << "Failed to free llvm/clang memory" << std::endl; + return 1; + } + + while (true) bpf.poll_perf_buffer("events"); + + return 0; +} + +struct thread_arg { + function handle_pid; +}; + +void *startThread(void *arg) { + thread_arg *p = (thread_arg *)arg; + callback = p->handle_pid; + try { + execsnoop(); + } catch (exception &e) { + error("bcc may not be installed, %s",e.what()); + } + return (void *)0; +} +} // namespace CGPROXY::EXESNOOP +#endif \ No newline at end of file diff --git a/src/socket_server.cpp b/src/socket_server.cpp index ae15ab3..e8d0fad 100644 --- a/src/socket_server.cpp +++ b/src/socket_server.cpp @@ -49,7 +49,7 @@ void SocketServer::socketListening(function callback) { } } -void *SocketServer::startThread(void *arg) { +void *startThread(void *arg) { thread_arg *p = (thread_arg *)arg; SocketServer server; server.socketListening(p->handle_msg); diff --git a/src/socket_server.h b/src/socket_server.h index c70bbb1..b5f84a1 100644 --- a/src/socket_server.h +++ b/src/socket_server.h @@ -17,6 +17,7 @@ namespace CGPROXY::SOCKET { struct thread_arg { function handle_msg; }; +void *startThread(void *arg); class SocketServer { public: @@ -25,7 +26,6 @@ public: void socketListening(function callback); ~SocketServer(); - static void *startThread(void *arg); }; } // namespace CGPROXY::SOCKET diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 6358358..c85a2fe 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,4 +1,9 @@ include_directories(${PROJECT_SOURCE_DIR}) include_directories(${PROJECT_SOURCE_DIR}/src) -add_executable(cgattach cgattach.cpp ../src/cgroup_attach.cpp ../src/common.cpp) -install(TARGETS cgattach DESTINATION /usr/bin PERMISSIONS ${basic_permission}) \ No newline at end of file + +add_executable(cgattach cgattach.cpp ../src/cgroup_attach.cpp ../src/common.cpp) +install(TARGETS cgattach DESTINATION /usr/bin PERMISSIONS ${basic_permission}) + +add_executable(execsnoop execsnoop.cpp ../src/common.cpp) +target_link_libraries(execsnoop bcc) +install(TARGETS execsnoop DESTINATION /usr/bin PERMISSIONS ${basic_permission}) \ No newline at end of file diff --git a/tools/execsnoop.cpp b/tools/execsnoop.cpp new file mode 100644 index 0000000..d401da4 --- /dev/null +++ b/tools/execsnoop.cpp @@ -0,0 +1,23 @@ +#include "execsnoop.hpp" +#include "common.h" +using namespace std; +using namespace CGPROXY::EXESNOOP; + +#define PATH_MAX_LEN 128 + +int handle_pid(int pid) { + char path[PATH_MAX_LEN]; + auto size = readlink(to_str("/proc/", pid, "/exe").c_str(), path, PATH_MAX_LEN); + if (size == -1) error("readlink: %s", to_str("/proc/", pid, "/exe").c_str()); + path[size] = '\0'; + info("%d %s", pid, path); + return 0; +} + +int main() { + enable_debug = true; + enable_info = true; + callback = handle_pid; + execsnoop(); + return 0; +}