I know that with ps I can see the list or tree of the current processes running in the system. But what I want to achieve is to "follow" the new processes that are created when using the computer.
As analogy, when you use tail -f to follow the new contents appended to a file or to any input, then I want to keep a follow list of the process that are currently being created.
Is this even posible?
If kprobes are enabled in the kernel you can use execsnoop from perf-tools:
In first terminal:
% while true; do uptime; sleep 1; done
In another terminal:
% git clone https://github.com/brendangregg/perf-tools.git
% cd perf-tools
% sudo ./execsnoop
Tracing exec()s. Ctrl-C to end.
Instrumenting sys_execve
PID PPID ARGS
83939 83937 cat -v trace_pipe
83938 83934 gawk -v o=1 -v opt_name=0 -v name= -v opt_duration=0 [...]
83940 76640 uptime
83941 76640 sleep 1
83942 76640 uptime
83943 76640 sleep 1
83944 76640 uptime
83945 76640 sleep 1
^C
Ending tracing...
bpfcc-tools.
Some examples of bpftrace usage to achieve the goal.
The simplest one is tracing all exec calls in the system:
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_exec*{ printf("pid: %d, comm: %s, args: ", pid, comm); join(args->argv); }'
There are at least two tracepoints you need to watch sys_enter_execve and sys_enter_execveat. In the example I use the * symbol to match both syscalls (this syntax works since 2019).
One may also want to monitor all threads being created in the system as:
sudo bpftrace -e 'kprobe:_do_fork{ printf("pid = %d, comm = %s\n", pid, comm); }'
No process arguments for you in this case though, nevertheless it may be useful.
To see the list of all available events execute bpftrace -l.
You can use forkstat as stated here: https://stackoverflow.com/a/40532202/781153
Install with: apt-get install forkstat
And just run: forkstat
The easiest way is to enable system call auditing
See the following link for details，
Does anyone know a simple way to monitor root process spawn | Server Fault
If you're monitoring all processes, just remove the -F uid=0 part
Logs are written to /var/log/audit/audit.log
CONFIG_PROC_EVENTS=y
Sample session:
$ su
# ./proc_events.out &
set mcast listen ok
# sleep 2 & sleep 1 &
fork: parent tid=48 pid=48 -> child tid=56 pid=56
fork: parent tid=48 pid=48 -> child tid=57 pid=57
exec: tid=57 pid=57
exec: tid=56 pid=56
exit: tid=57 pid=57 exit_code=0
exit: tid=56 pid=56 exit_code=0
CONFIG_PROC_EVENTS exposes the events to userland via a netlink socket.
proc_events.c
#define _XOPEN_SOURCE 700
#include <sys/socket.h>
#include <linux/netlink.h>
#include <linux/connector.h>
#include <linux/cn_proc.h>
#include <signal.h>
#include <errno.h>
#include <stdbool.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
static volatile bool need_exit = false;
static int nl_connect()
{
int rc;
int nl_sock;
struct sockaddr_nl sa_nl;
nl_sock = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_CONNECTOR);
if (nl_sock == -1) {
perror("socket");
return -1;
}
sa_nl.nl_family = AF_NETLINK;
sa_nl.nl_groups = CN_IDX_PROC;
sa_nl.nl_pid = getpid();
rc = bind(nl_sock, (struct sockaddr *)&sa_nl, sizeof(sa_nl));
if (rc == -1) {
perror("bind");
close(nl_sock);
return -1;
}
return nl_sock;
}
static int set_proc_ev_listen(int nl_sock, bool enable)
{
int rc;
struct __attribute__ ((aligned(NLMSG_ALIGNTO))) {
struct nlmsghdr nl_hdr;
struct __attribute__ ((__packed__)) {
struct cn_msg cn_msg;
enum proc_cn_mcast_op cn_mcast;
};
} nlcn_msg;
memset(&nlcn_msg, 0, sizeof(nlcn_msg));
nlcn_msg.nl_hdr.nlmsg_len = sizeof(nlcn_msg);
nlcn_msg.nl_hdr.nlmsg_pid = getpid();
nlcn_msg.nl_hdr.nlmsg_type = NLMSG_DONE;
nlcn_msg.cn_msg.id.idx = CN_IDX_PROC;
nlcn_msg.cn_msg.id.val = CN_VAL_PROC;
nlcn_msg.cn_msg.len = sizeof(enum proc_cn_mcast_op);
nlcn_msg.cn_mcast = enable ? PROC_CN_MCAST_LISTEN : PROC_CN_MCAST_IGNORE;
rc = send(nl_sock, &nlcn_msg, sizeof(nlcn_msg), 0);
if (rc == -1) {
perror("netlink send");
return -1;
}
return 0;
}
static int handle_proc_ev(int nl_sock)
{
int rc;
struct __attribute__ ((aligned(NLMSG_ALIGNTO))) {
struct nlmsghdr nl_hdr;
struct __attribute__ ((__packed__)) {
struct cn_msg cn_msg;
struct proc_event proc_ev;
};
} nlcn_msg;
while (!need_exit) {
rc = recv(nl_sock, &nlcn_msg, sizeof(nlcn_msg), 0);
if (rc == 0) {
/* shutdown? */
return 0;
} else if (rc == -1) {
if (errno == EINTR) continue;
perror("netlink recv");
return -1;
}
switch (nlcn_msg.proc_ev.what) {
case PROC_EVENT_NONE:
printf("set mcast listen ok\n");
break;
case PROC_EVENT_FORK:
printf("fork: parent tid=%d pid=%d -> child tid=%d pid=%d\n",
nlcn_msg.proc_ev.event_data.fork.parent_pid,
nlcn_msg.proc_ev.event_data.fork.parent_tgid,
nlcn_msg.proc_ev.event_data.fork.child_pid,
nlcn_msg.proc_ev.event_data.fork.child_tgid);
break;
case PROC_EVENT_EXEC:
printf("exec: tid=%d pid=%d\n",
nlcn_msg.proc_ev.event_data.exec.process_pid,
nlcn_msg.proc_ev.event_data.exec.process_tgid);
break;
case PROC_EVENT_UID:
printf("uid change: tid=%d pid=%d from %d to %d\n",
nlcn_msg.proc_ev.event_data.id.process_pid,
nlcn_msg.proc_ev.event_data.id.process_tgid,
nlcn_msg.proc_ev.event_data.id.r.ruid,
nlcn_msg.proc_ev.event_data.id.e.euid);
break;
case PROC_EVENT_GID:
printf("gid change: tid=%d pid=%d from %d to %d\n",
nlcn_msg.proc_ev.event_data.id.process_pid,
nlcn_msg.proc_ev.event_data.id.process_tgid,
nlcn_msg.proc_ev.event_data.id.r.rgid,
nlcn_msg.proc_ev.event_data.id.e.egid);
break;
case PROC_EVENT_EXIT:
printf("exit: tid=%d pid=%d exit_code=%d\n",
nlcn_msg.proc_ev.event_data.exit.process_pid,
nlcn_msg.proc_ev.event_data.exit.process_tgid,
nlcn_msg.proc_ev.event_data.exit.exit_code);
break;
default:
printf("unhandled proc event\n");
break;
}
}
return 0;
}
static void on_sigint(__attribute__ ((unused)) int unused)
{
need_exit = true;
}
int main()
{
int nl_sock;
int rc = EXIT_SUCCESS;
signal(SIGINT, &on_sigint);
siginterrupt(SIGINT, true);
nl_sock = nl_connect();
if (nl_sock == -1)
exit(EXIT_FAILURE);
rc = set_proc_ev_listen(nl_sock, true);
if (rc == -1) {
rc = EXIT_FAILURE;
goto out;
}
rc = handle_proc_ev(nl_sock);
if (rc == -1) {
rc = EXIT_FAILURE;
goto out;
}
set_proc_ev_listen(nl_sock, false);
out:
close(nl_sock);
exit(rc);
}
GitHub upsatream, code adapted from: https://bewareofgeek.livejournal.com/2945.html
I don't think however that you can you get process data such as UID and process arguments because exec_proc_event contains so little data: https://github.com/torvalds/linux/blob/v4.16/include/uapi/linux/cn_proc.h#L80 We could try to immediately read it from /proc, but there is a risk that the process finished and another one took its PID, so it wouldn't be reliable.
Tested on Ubuntu 17.10, which has CONFIG_PROC_EVENTS=y enabled by default.
There are multiple Linux APIs that you can use to do this and multiple usermode tools that use the various APIs. Here are some of the APIs:
There are really too many details to properly compare these options in a single StackOverflow answer, but I've written about this in detail elsewhere.
You can apparently follow a process using strace. If you know the PID of the process then you can do:
strace -o strace-<pid>.out -f -p <pid>
Notice the -f switch. It will help you to follow newly created processes that are descendants of the process whose PID was used in the command, above. For information on strace see this question.
TL;DR: use execnsoop.
Check that KPROBES are enabled in the Kernel
$ zgrep CONFIG_KPROBES= /boot/config-$(uname -r) /proc/config* 2> /dev/null
/boot/config-4.15.0-64-generic:CONFIG_KPROBES=y
Install BPF Compiler Collection (BCC)
For Debian/Ubuntu the package linux-headers-* is also needed:
sudo apt-get install bpfcc-tools linux-headers-$(uname -r)
For other distributions see Installing BCC
Run execsnoop (examples and options)
root@ubuntu ~ # execsnoop-bpfcc -tx
TIME(s) PCOMM PID PPID RET ARGS
5.773 ping 13027 12847 0 /bin/ping hostname
You can use SysmonForLinux tool which is originally one of the powerful Windows tools released for Linux by the Sysinternals team for collecting most of the system logs including process creation and termination. Its default configuration logs to all process creations.
you can use https://github.com/radiopushka/pmmanager (Written by me)
and simply run "pmmanager watchdog"
it will list processes that just got launched and ran for longer that 1 second(that being the only downside). The program should compile and run pretty much anywhere and doesn't need anything. Also it doesn't require kprobes and root.
Please note that the execnsoop with ebpf may not caputure all processes newed in system. It only watches on syscall execve according to github. However, syscall clone is also a way to create new processes, which escape from execve. So you probably need to watch clone as well.
About difference between clone and fork please refer to this question in stackoverflow
