use std::{ collections::{BTreeSet, HashMap}, ffi::CString, ffi::OsString, io::Write, net::TcpStream, os::unix::prelude::OsStringExt, path::PathBuf, process::exit, time::Instant }; use nix::{ errno::Errno, libc::{pid_t, raise, tcsetpgrp, AT_EMPTY_PATH, AT_FDCWD, SIGSTOP, STDIN_FILENO}, sys::{ ptrace::{self, traceme, AddressType}, signal::Signal, wait::{waitpid, WaitPidFlag, WaitStatus}, }, unistd::{execvp, getpid, setpgid, ForkResult}, }; use serde_json::de::IoRead; use sha2::{Sha256, Digest}; use crate::filestore::{parse_format, Sha256Hash}; use super::types::*; pub fn read_generic_string( pid: Pid, address: AddressType, ctor: impl Fn(Vec) -> TString, ) -> anyhow::Result { let mut buf = Vec::new(); let mut address = address; const WORD_SIZE: usize = 8; // FIXME loop { let word = match ptrace::read(pid.into(), address) { Err(e) => { log::warn!("Cannot read tracee {pid} memory {address:?}: {e}"); return Ok(ctor(buf)); } Ok(word) => word, }; let word_bytes = word.to_ne_bytes(); for &byte in word_bytes.iter() { if byte == 0 { return Ok(ctor(buf)); } buf.push(byte); } address = unsafe { address.add(WORD_SIZE) }; } } #[allow(unused)] pub fn read_cstring(pid: Pid, address: AddressType) -> anyhow::Result { read_generic_string(pid, address, |x| CString::new(x).unwrap()) } pub fn read_pathbuf(pid: Pid, address: AddressType) -> anyhow::Result { read_generic_string(pid, address, |x| PathBuf::from(OsString::from_vec(x))) } pub fn read_string(pid: Pid, address: AddressType) -> anyhow::Result { // Waiting on https://github.com/rust-lang/libs-team/issues/116 read_generic_string(pid, address, |x| String::from_utf8_lossy(&x).to_string()) } pub fn read_null_ended_array( pid: Pid, mut address: AddressType, reader: impl Fn(Pid, AddressType) -> anyhow::Result, ) -> anyhow::Result> { let mut res = Vec::new(); const WORD_SIZE: usize = 8; // FIXME loop { let ptr = match ptrace::read(pid.into(), address) { Err(e) => { log::warn!("Cannot read tracee {pid} memory {address:?}: {e}"); return Ok(res); } Ok(ptr) => ptr, }; if ptr == 0 { return Ok(res); } else { res.push(reader(pid, ptr as AddressType)?); } address = unsafe { address.add(WORD_SIZE) }; } } #[allow(unused)] pub fn read_cstring_array(pid: Pid, address: AddressType) -> anyhow::Result> { read_null_ended_array(pid, address, read_cstring) } #[allow(unused)] pub fn read_string_array(pid: Pid, address: AddressType) -> anyhow::Result> { read_null_ended_array(pid, address, read_string) } macro_rules! syscall_no_from_regs { ($regs:ident) => { $regs.orig_rax as i64 }; } macro_rules! syscall_res_from_regs { ($regs:ident) => { $regs.rax as i64 }; } macro_rules! syscall_arg { ($regs:ident, 0) => { $regs.rdi }; ($regs:ident, 1) => { $regs.rsi }; ($regs:ident, 2) => { $regs.rdx }; ($regs:ident, 3) => { $regs.r10 }; ($regs:ident, 4) => { $regs.r8 }; ($regs:ident, 5) => { $regs.r9 }; } pub fn read_argv(pid: Pid) -> anyhow::Result> { let filename = format!("/proc/{pid}/cmdline"); let buf = std::fs::read(filename)?; Ok(buf .split(|&c| c == 0) .map(CString::new) .collect::, _>>()?) } pub fn read_comm(pid: Pid) -> anyhow::Result { let filename = format!("/proc/{pid}/comm"); let mut buf = std::fs::read(filename)?; buf.pop(); // remove trailing newline Ok(String::from_utf8(buf)?) } pub fn read_cwd(pid: Pid) -> std::io::Result { let filename = format!("/proc/{pid}/cwd"); let buf = std::fs::read_link(filename)?; Ok(buf) } pub fn read_fd(pid: Pid, fd: i32) -> std::io::Result { if fd == AT_FDCWD { return read_cwd(pid); } let filename = format!("/proc/{pid}/fd/{fd}"); std::fs::read_link(filename) } /* #[derive(Debug)] pub enum Interpreter { None, Shebang(String), ExecutableUnaccessible, Error(io::Error), } impl Display for Interpreter { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { Interpreter::None => write!(f, "none"), Interpreter::Shebang(s) => write!(f, "{:?}", s), Interpreter::ExecutableUnaccessible => { write!(f, "executable unaccessible") } Interpreter::Error(e) => write!(f, "(err: {e})"), } } } pub fn read_interpreter_recursive(exe: impl AsRef) -> Vec { let mut exe = Cow::Borrowed(exe.as_ref()); let mut interpreters = Vec::new(); loop { match read_interpreter(exe.as_ref()) { Interpreter::Shebang(shebang) => { exe = Cow::Owned(PathBuf::from( shebang.split_ascii_whitespace().next().unwrap_or(""), )); interpreters.push(Interpreter::Shebang(shebang)); } Interpreter::None => break, err => { interpreters.push(err); break; } }; } interpreters } pub fn read_interpreter(exe: &Path) -> Interpreter { fn err_to_interpreter(e: io::Error) -> Interpreter { if e.kind() == io::ErrorKind::PermissionDenied || e.kind() == io::ErrorKind::NotFound { Interpreter::ExecutableUnaccessible } else { Interpreter::Error(e) } } let file = match std::fs::File::open(exe) { Ok(file) => file, Err(e) => return err_to_interpreter(e), }; let mut reader = BufReader::new(file); // First, check if it's a shebang script let mut buf = [0u8; 2]; if let Err(e) = reader.read_exact(&mut buf) { return Interpreter::Error(e); }; if &buf != b"#!" { return Interpreter::None; } // Read the rest of the line let mut buf = Vec::new(); if let Err(e) = reader.read_until(b'\n', &mut buf) { return Interpreter::Error(e); }; // Get trimed shebang line [start, end) indices // If the shebang line is empty, we don't care let start = buf .iter() .position(|&c| !c.is_ascii_whitespace()) .unwrap_or(0); let end = buf .iter() .rposition(|&c| !c.is_ascii_whitespace()) .map(|x| x + 1) .unwrap_or(buf.len()); let shebang = String::from_utf8_lossy(&buf[start..end]); Interpreter::Shebang(shebang.into_owned()) } */ #[derive(Default)] pub struct ProcessStateStore { processes: HashMap>, } #[derive(Debug)] pub struct ProcessState { pub pid: Pid, pub ppid: Option, pub status: ProcessStatus, pub start_time: u64, pub argv: Vec, pub comm: String, pub presyscall: bool, pub is_exec_successful: bool, pub syscall: i64, pub pending_syscall_event: Vec, } #[derive(Debug, Clone, PartialEq)] pub enum ProcessStatus { SigstopReceived, PtraceForkEventReceived, Running, Exited(i32), } #[derive(Debug)] pub struct ExecData { pub filename: PathBuf, pub argv: Vec, pub envp: Vec, pub cwd: PathBuf, //pub interpreters: Vec, } impl ProcessStateStore { pub fn insert(&mut self, state: ProcessState) { self.processes.entry(state.pid).or_default().push(state); } pub fn get_current_mut(&mut self, pid: Pid) -> Option<&mut ProcessState> { // The last process in the vector is the current process // println!("Getting {pid}"); self.processes.get_mut(&pid)?.last_mut() } } impl ProcessState { pub fn new(pid: Pid, start_time: u64) -> anyhow::Result { Ok(Self { pid, ppid: None, status: ProcessStatus::Running, comm: read_comm(pid)?, argv: read_argv(pid)?, start_time, presyscall: true, is_exec_successful: false, syscall: -1, pending_syscall_event: vec![], }) } } fn ptrace_syscall(pid: Pid, sig: Option) -> Result<(), Errno> { match ptrace::syscall(pid.into(), sig) { Err(Errno::ESRCH) => { log::info!("ptrace syscall failed: {pid}, ESRCH, child probably gone!"); Ok(()) } other => other, } } pub struct TracerClient { store: ProcessStateStore, start_time: Instant, pending_events: Vec, pending_files: BTreeSet<(PathBuf, Sha256Hash)>, machine: i32, } impl TracerClient { pub fn log(&mut self, ident: Identifier, event: Event) { self.pending_events.push(LogEntry { ident, event, timestamp: Instant::now().duration_since(self.start_time), }); } pub fn log_root(&mut self, pid: Pid, event: Event) { self.log(Identifier { pid, machine: self.machine }, event); } fn ingest_file(&mut self, path: PathBuf) -> anyhow::Result<()> { let stat = std::fs::metadata(&path)?; if !stat.is_file() { return Ok(()); } let mut fp = std::fs::File::open(&path)?; let mut h = Sha256::new(); log::debug!("Hashing {} (client)", path.to_string_lossy()); std::io::copy(&mut fp, &mut h)?; let hash = h.finalize().into(); self.pending_files.insert((path, hash)); Ok(()) } fn drain_syscall_events(&mut self, pid: Pid, mut filter: Box) { let p = self.store.get_current_mut(pid).unwrap(); for mut event in p.pending_syscall_event.drain(..) { (filter)(&mut event); self.pending_events.push(LogEntry { ident: Identifier { pid, machine: self.machine }, event, timestamp: Instant::now().duration_since(self.start_time), }); } } pub fn run(machine: i32, connect: String, args: Vec) -> anyhow::Result<()> { let mut this = Self { store: ProcessStateStore::default(), start_time: Instant::now(), pending_events: vec![], pending_files: BTreeSet::new(), machine, }; let sock = TcpStream::connect(&connect).expect(format!("Could not connect to {connect}").as_str()); if let ForkResult::Parent { child } = unsafe { nix::unistd::fork()? } { this.run_internal(sock, child.into()) } else { let me = getpid(); setpgid(me, me)?; traceme()?; if 0 != unsafe { raise(SIGSTOP) } { log::error!("raise failed!"); exit(-1); } let args = args .into_iter() .map(CString::new) .collect::, _>>()?; execvp(&args[0], &args).expect(format!("Failed to execute {args:?}").as_str()); unreachable!(); } } fn run_internal(&mut self, mut sock: TcpStream, root_child: Pid) -> anyhow::Result<()> { waitpid(nix::unistd::Pid::from(root_child.into()), Some(WaitPidFlag::WSTOPPED))?; // wait for child to stop log::trace!("child stopped"); let mut root_child_state = ProcessState::new(root_child, 0)?; root_child_state.ppid = Some(getpid().into()); self.store.insert(root_child_state); // Set foreground process group of the terminal if -1 == unsafe { tcsetpgrp(STDIN_FILENO, root_child.0) } { return Err(Errno::last().into()); } // restart child log::trace!("resuming child"); let ptrace_opts = { use nix::sys::ptrace::Options; Options::PTRACE_O_TRACEEXEC | Options::PTRACE_O_TRACEEXIT | Options::PTRACE_O_EXITKILL | Options::PTRACE_O_TRACESYSGOOD | Options::PTRACE_O_TRACEFORK | Options::PTRACE_O_TRACECLONE | Options::PTRACE_O_TRACEVFORK }; ptrace::setoptions(root_child.into(), ptrace_opts)?; // restart child ptrace::syscall(nix::unistd::Pid::from(root_child.into()), None)?; let mut continuing = true; while continuing { let status = { let status = waitpid(None, Some(WaitPidFlag::__WALL)); if status.is_err_and(|e| e == nix::errno::Errno::ECHILD) { continuing = false; Ok(WaitStatus::StillAlive) } else { status } }?; // log::trace!("waitpid: {:?}", status); let signal = match status { WaitStatus::Stopped(pid, sig) => { let pid = pid.into(); log::trace!("stopped: {pid}, sig {:?}", sig); match sig { Signal::SIGSTOP => { log::trace!("sigstop event, child: {pid}"); if let Some(state) = self.store.get_current_mut(pid) { if state.status == ProcessStatus::PtraceForkEventReceived { log::trace!("sigstop event received after ptrace fork event, pid: {pid}"); state.status = ProcessStatus::Running; } else if pid != root_child { log::error!("Unexpected SIGSTOP: {state:?}") } } else { log::trace!("sigstop event received before ptrace fork event, pid: {pid}"); let mut state = ProcessState::new(pid, 0)?; state.status = ProcessStatus::SigstopReceived; self.store.insert(state); } None } Signal::SIGCHLD => { // From lurk: // // The SIGCHLD signal is sent to a process when a child process terminates, interrupted, or resumes after being interrupted // This means, that if our tracee forked and said fork exits before the parent, the parent will get stopped. // Therefor issue a PTRACE_SYSCALL request to the parent to continue execution. // This is also important if we trace without the following forks option. Some(Signal::SIGCHLD) } _ => { // Just deliver the signal to tracee Some(sig) } } } WaitStatus::Exited(pid, code) => { let pid = pid.into(); log::trace!("exited: pid {}, code {:?}", pid, code); self.log_root(pid, Event::Exit { code }); self.store.get_current_mut(pid).unwrap().status = ProcessStatus::Exited(code); None } WaitStatus::PtraceEvent(pid, sig, evt) => { log::trace!("ptrace event: {:?} {:?}", sig, evt); match evt { nix::libc::PTRACE_EVENT_FORK | nix::libc::PTRACE_EVENT_VFORK | nix::libc::PTRACE_EVENT_CLONE => { let new_child = Pid(ptrace::getevent(pid.into())? as pid_t); log::trace!( "ptrace fork event, evt {evt}, pid: {pid}, child: {new_child}" ); self.log_root(pid.into(), Event::Fork { child: new_child }); if let Some(state) = self.store.get_current_mut(new_child) { if state.status == ProcessStatus::SigstopReceived { log::trace!("ptrace fork event received after sigstop, pid: {pid}, child: {new_child}"); state.status = ProcessStatus::Running; state.ppid = Some(pid.into()); } else if new_child != root_child { log::error!("Unexpected fork event: {state:?}") } } else { log::trace!("ptrace fork event received before sigstop, pid: {pid}, child: {new_child}"); let mut state = ProcessState::new(new_child, 0)?; state.status = ProcessStatus::PtraceForkEventReceived; state.ppid = Some(pid.into()); self.store.insert(state); } // Resume parent None } nix::libc::PTRACE_EVENT_EXEC => { log::trace!("exec event"); let p = self.store.get_current_mut(pid.into()).unwrap(); assert!(!p.presyscall); // After execve or execveat, in syscall exit event, // the registers might be clobbered(e.g. aarch64). // So we need to determine whether exec is successful here. // PTRACE_EVENT_EXEC only happens for successful exec. p.is_exec_successful = true; let path = p .pending_syscall_event .iter() .find_map(|e| match e { Event::Exec { prog, .. } => Some(prog.clone()), _ => None, }) .unwrap(); self.ingest_file(path)?; self.drain_syscall_events(pid.into(), Box::new(|_| {})); // Don't use seccomp_aware_cont here because that will skip the next syscall exit stop None } nix::libc::PTRACE_EVENT_EXIT => { log::trace!("exit event"); None } nix::libc::PTRACE_EVENT_SECCOMP => { log::trace!("seccomp event"); self.on_syscall_enter(pid.into())?; None } _ => { log::trace!("other event"); None } } } WaitStatus::Signaled(pid, sig, _) => { let pid: Pid = pid.into(); log::debug!("signaled: {pid}, {:?}", sig); if pid == root_child { exit(128 + (sig as i32)) } None } WaitStatus::PtraceSyscall(pid) => { let pid = pid.into(); let presyscall = self.store.get_current_mut(pid).unwrap().presyscall; if presyscall { self.on_syscall_enter(pid)?; } else { self.on_syscall_exit(pid)?; } None } _ => None }; if !self.pending_files.is_empty() || !continuing { let mut events = vec![]; let mut files = BTreeSet::new(); std::mem::swap(&mut events, &mut self.pending_events); std::mem::swap(&mut files, &mut self.pending_files); let mut msg = TracerClientMessage::Events { events, files }; loop { serde_json::to_writer(&sock, &msg)?; sock.write_all("\n".as_bytes())?; let event: TracerServerRequest = serde_json::StreamDeserializer::new(&mut IoRead::new(&sock)).next().unwrap()?; match event { TracerServerRequest::Continue => break, TracerServerRequest::AnalyzeFiles { paths } => { let mut formats = HashMap::new(); let mut files = BTreeSet::new(); for path in paths { let mut fp = std::fs::File::open(&path)?; log::debug!("Parsing format of {} (client)", path.to_string_lossy()); let (format, mut references) = parse_format(&mut fp)?; formats.insert(path, format); files.append(&mut references); } msg = TracerClientMessage::FileFormats { formats, files } }, TracerServerRequest::AllocatedId { id } => { panic!("Receieved unsolicited AllocatedId({id})"); } } } } // https://stackoverflow.com/questions/29997244/occasionally-missing-ptrace-event-vfork-when-running-ptrace // DO NOT send PTRACE_SYSCALL until we receive the PTRACE_EVENT_FORK, etc. if let Some(pid) = status.pid() { let pid = pid.into(); let p = self.store.get_current_mut(pid).expect("No such process??"); if !matches!(p.status, ProcessStatus::SigstopReceived | ProcessStatus::Exited(_)) { ptrace_syscall(pid, signal)?; } } } Ok(()) } fn on_syscall_enter(&mut self, pid: Pid) -> anyhow::Result<()> { let p = self.store.get_current_mut(pid).unwrap(); p.presyscall = !p.presyscall; // SYSCALL ENTRY let regs = match ptrace::getregs(pid.into()) { Ok(regs) => regs, Err(Errno::ESRCH) => { log::info!("ptrace getregs failed: {pid}, ESRCH, child probably gone!"); return Ok(()); } e => e?, }; let syscallno = syscall_no_from_regs!(regs); p.syscall = syscallno; // log::trace!("pre syscall: {syscallno}"); match syscallno { nix::libc::SYS_execveat => { // int execveat(int dirfd, const char *pathname, // char *const _Nullable argv[], // char *const _Nullable envp[], // int flags); let dirfd = syscall_arg!(regs, 0) as i32; let pathname = read_string(pid, syscall_arg!(regs, 1) as AddressType)?; //let argv = read_string_array(pid, syscall_arg!(regs, 2) as AddressType)?; //let envp = read_string_array(pid, syscall_arg!(regs, 3) as AddressType)?; let flags = syscall_arg!(regs, 4) as i32; let filename = resolve_filename_at_fd(pid, pathname, dirfd, flags)?; //let interpreters = read_interpreter_recursive(&filename); p.pending_syscall_event.push(Event::Exec { prog: filename }); } nix::libc::SYS_execve => { let filename = read_pathbuf(pid, syscall_arg!(regs, 0) as AddressType)?; //let argv = read_string_array(pid, syscall_arg!(regs, 1) as AddressType)?; //let envp = read_string_array(pid, syscall_arg!(regs, 2) as AddressType)?; //let interpreters = read_interpreter_recursive(&filename); p.pending_syscall_event.push(Event::Exec { prog: filename }); } nix::libc::SYS_open => { let path = read_pathbuf(pid, syscall_arg!(regs, 0) as AddressType)?; p.pending_syscall_event.push(Event::FdOpen { source: FdSource::File { path }, fd: -1, }); } nix::libc::SYS_openat => { let dirfd = syscall_arg!(regs, 0) as i32; let pathname = read_string(pid, syscall_arg!(regs, 1) as AddressType)?; let flags = syscall_arg!(regs, 2) as i32; let path = resolve_filename_at_fd(pid, pathname, dirfd, flags)?; p.pending_syscall_event.push(Event::FdOpen { source: FdSource::File { path }, fd: 0, }); } nix::libc::SYS_read | nix::libc::SYS_readv | nix::libc::SYS_preadv | nix::libc::SYS_preadv2 => { let fd = syscall_arg!(regs, 0) as i32; p.pending_syscall_event.push(Event::FdRead { fd }); } nix::libc::SYS_write | nix::libc::SYS_writev | nix::libc::SYS_pwritev | nix::libc::SYS_pwritev2 => { let fd = syscall_arg!(regs, 0) as i32; p.pending_syscall_event.push(Event::FdWrite { fd }); } nix::libc::SYS_dup | nix::libc::SYS_dup2 | nix::libc::SYS_dup3 => { let oldfd = syscall_arg!(regs, 0) as i32; p.pending_syscall_event .push(Event::FdDup { oldfd, newfd: -1 }); } nix::libc::SYS_fcntl => { let fd = syscall_arg!(regs, 0) as i32; let cmd = syscall_arg!(regs, 1) as i32; match cmd { nix::libc::F_DUPFD => p.pending_syscall_event.push(Event::FdDup { oldfd: fd, newfd: -1, }), _ => {} } } nix::libc::SYS_close => { let fd = syscall_arg!(regs, 0) as i32; p.pending_syscall_event.push(Event::FdClose { fd }); } _ => {} } //self.syscall_enter_cont(pid)?; Ok(()) } fn on_syscall_exit(&mut self, pid: Pid) -> anyhow::Result<()> { // SYSCALL EXIT // log::trace!("post syscall {}", p.syscall); let p = self.store.get_current_mut(pid).unwrap(); p.presyscall = !p.presyscall; let regs = match ptrace::getregs(pid.into()) { Ok(regs) => regs, Err(Errno::ESRCH) => { log::info!("ptrace getregs failed: {pid}, ESRCH, child probably gone!"); return Ok(()); } e => e?, }; let result = syscall_res_from_regs!(regs); let mut pending_files = vec![]; let filter: Option> = match p.syscall { nix::libc::SYS_execve => { // SAFETY: p.preexecve is false, so p.exec_data is Some p.is_exec_successful = false; // update comm p.comm = read_comm(pid)?; None } nix::libc::SYS_execveat => { p.is_exec_successful = false; // update comm p.comm = read_comm(pid)?; None } nix::libc::SYS_open | nix::libc::SYS_openat => { if result >= 0 { for pending in p.pending_syscall_event.iter_mut() { if let Event::FdOpen { source: FdSource::File { path }, .. } = pending { pending_files.push(path.clone()); } } Some(Box::new(move |event| match event { Event::FdOpen { fd: ref mut dest, .. } => { *dest = result as i32; } _ => {} })) } else { None } } nix::libc::SYS_dup | nix::libc::SYS_dup2 | nix::libc::SYS_dup3 => { if result >= 0 { Some(Box::new(move |event| match event { Event::FdDup { newfd: ref mut dest, .. } => { *dest = result as i32; } _ => {} })) } else { None } } nix::libc::SYS_fcntl => { if result >= 0 { Some(Box::new(move |event| match event { Event::FdDup { newfd: ref mut dest, .. } => { *dest = result as i32; } _ => {} })) } else { None } } _ => { if result >= 0 { Some(Box::new(|_| {})) } else { None } } }; if let Some(filter) = filter { self.drain_syscall_events(pid, filter); } else { p.pending_syscall_event.clear(); } for path in pending_files { self.ingest_file(path)?; } Ok(()) } } fn resolve_filename_at_fd( pid: Pid, pathname: String, dirfd: i32, flags: i32, ) -> anyhow::Result { let pathname_is_empty = pathname.is_empty(); let pathname = PathBuf::from(pathname); Ok( match ( pathname.is_absolute(), pathname_is_empty && ((flags & AT_EMPTY_PATH) != 0), ) { (true, _) => { // If pathname is absolute, then dirfd is ignored. pathname } (false, true) => { // If pathname is an empty string and the AT_EMPTY_PATH flag is specified, then the file descriptor dirfd // specifies the file to be executed read_fd(pid, dirfd)? } (false, false) => { // pathname is relative to dirfd let dir = read_fd(pid, dirfd)?; dir.join(pathname) } }, ) }