implement agent runtime on racket CS

This commit is contained in:
xenia 2020-12-29 06:24:23 -05:00
parent 311ce74f82
commit fc83741b02
10 changed files with 419 additions and 34 deletions

View File

@ -23,20 +23,45 @@ RKT_NAME=$(APP_NAME).rkt
MONOCYPHER_VERSION=3.1.1
RACKET_VERSION=7.9
CC=$(CROSS_COMPILE)gcc
OBJCOPY=$(CROSS_COMPILE)objcopy
TARGET_ARCH=$(shell $(CC) -dumpmachine)
# change this if you're 32 bit. idk how to make this more automatic
TARGET_BITS=64
####################################################################################################
ARCH_DIR=arch_$(TARGET_ARCH)
APP_CFLAGS=-pipe -fPIC -O3 -DAPP_NAME='"$(APP_NAME)"' -DAPP_ARCH='"$(TARGET_ARCH)"'
APP_CFLAGS=-pipe -fPIC -O3 -DAPP_NAME='"$(APP_NAME)"' -DAPP_ARCH='"$(TARGET_ARCH)"' \
-DELF_BITS=$(TARGET_BITS)
APP_LDFLAGS=-static -ldl -lm -lpthread
# musl needs -lucontext
all: $(ARCH_DIR)/$(APP_NAME)
include dependencies.mk
# main build
ifdef ENABLE_CS
CS_BOOT1=$(RACKET_CS_BOOT_FILES_DIR)/petite.boot
CS_BOOT2=$(RACKET_CS_BOOT_FILES_DIR)/scheme.boot
CS_BOOT3=$(RACKET_CS_BOOT_FILES_DIR)/racket.boot
$(ARCH_DIR)/$(APP_NAME): main_cs.c app.zo vnd-deps $(CS_BOOT1) $(CS_BOOT2) $(CS_BOOT3)
[ -d $(ARCH_DIR) ] || mkdir -p $(ARCH_DIR)
$(CC) -o $@ $(APP_CFLAGS) $(VND_CFLAGS) main_cs.c \
$(VND_LDFLAGS) $(APP_LDFLAGS)
$(OBJCOPY) \
--add-section .csboot1=$(CS_BOOT1) \
--set-section-flags .csboot1=noload,readonly \
--add-section .csboot2=$(CS_BOOT2) \
--set-section-flags .csboot2=noload,readonly \
--add-section .csboot3=$(CS_BOOT3) \
--set-section-flags .csboot3=noload,readonly \
--add-section .boot=app.zo \
--set-section-flags .boot=noload,readonly \
$@ $@.ext
mv -f $@.ext $@
else
$(ARCH_DIR)/$(APP_NAME): main_bc.c app.o vnd-deps
[ -d $(ARCH_DIR) ] || mkdir -p $(ARCH_DIR)
$(CC) -o $@ $(APP_CFLAGS) $(VND_CFLAGS) main_bc.c app.o \
@ -46,9 +71,10 @@ $(ARCH_DIR)/$(APP_NAME): main_bc.c app.o vnd-deps
# it's potentially less portable but works on GNU ld and probably lld too
app.o: app.zo
$(LD) -r -b binary $< -o $@
endif
app.zo: $(RKT_NAME)
raco ctool --mods $@ $<
$(RACO) ctool --mods $@ ++lib racket/base $<
clean:
$(RM) $(ARCH_DIR)/$(APP_NAME) *.zo *.o

View File

@ -0,0 +1,29 @@
// crossfire: distributed brute force infrastructure
//
// Copyright (C) 2020 haskal
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#pragma once
#include <stdint.h>
#include <stdio.h>
// ffi defs
typedef struct {
const char* name;
uintptr_t ptr;
} ffi_ent;
#define STR(x) #x
#define FFI_ENT(name) {STR(name), (uintptr_t) name}

View File

@ -36,6 +36,16 @@ define extract-file =
tar xf $< -C $(VENDOR_DIR) && touch $@
endef
VND_CFLAGS=
VND_LDFLAGS=
.PHONY: vnd-deps-clean vnd-deps vnd-deps-dl
mrproper: vnd-deps-clean
vnd-deps-clean:
$(RM) -r $(VENDOR_DIR) $(CACHE_DIR)
## monocypher
VND_MONOCYPHER=$(MONOCYPHER_DIR)/lib/libmonocypher.a
@ -51,22 +61,58 @@ $(MONOCYPHER_DIR)/.extracted: $(MONOCYPHER_FILE)
$(MONOCYPHER_FILE):
$(call download-file,$(MONOCYPHER_URL))
vnd-deps-dl: $(MONOCYPHER_FILE)
VND_CFLAGS+=$(VND_MONOCYPHER_CFLAGS)
VND_LDFLAGS+=$(VND_MONOCYPHER_LDFLAGS)
vnd-deps: $(VND_MONOCYPHER)
## racket
### racket BC
VND_RACKET_BC=$(RACKET_DIR)/lib/libracket3m.a
VND_RACKET_BC_CFLAGS=-I$(RACKET_DIR)/src/include
VND_RACKET_BC_LDFLAGS=-L$(RACKET_DIR)/lib -lracket3m -lrktio -lffi
# musl also needs -lucontext
$(VND_RACKET_BC): $(RACKET_DIR)/.extracted
cd $(RACKET_DIR)/src && ./configure --enable-bconly --prefix=/usr --sysconfdir=/etc \
--disable-libffi
--disable-libffi --disable-docs
cd $(RACKET_DIR)/src && $(MAKE)
[ -d $(RACKET_DIR)/lib ] || mkdir $(RACKET_DIR)/lib
cp $(RACKET_DIR)/src/bc/rktio/librktio.a $(RACKET_DIR)/lib
cp $(RACKET_DIR)/src/bc/libracket3m.a $(RACKET_DIR)/lib
cp $(RACKET_DIR)/src/bc/foreign/libffi/.libs/libffi.a $(RACKET_DIR)/lib
### racket CS
VND_RACKET_CS=$(RACKET_DIR)/lib/libracketcs.a
VND_RACKET_CS_CFLAGS=-I$(RACKET_DIR)/include
VND_RACKET_CS_LDFLAGS=-L$(RACKET_DIR)/lib -lracketcs
$(VND_RACKET_CS): $(RACKET_DIR)/.extracted
cd $(RACKET_DIR)/src && ./configure --enable-csonly --disable-docs
cd $(RACKET_DIR)/src && $(MAKE)
cd $(RACKET_DIR)/src && $(MAKE) install
cd $(RACKET_DIR) && bin/raco pkg install -i --auto cext-lib
### racket common
ifdef ENABLE_CS
vnd-deps: $(VND_RACKET_CS)
VND_CFLAGS+=$(VND_RACKET_CS_CFLAGS)
VND_LDFLAGS+=$(VND_RACKET_CS_LDFLAGS)
RACO=$(RACKET_DIR)/bin/raco
RACKET_CS_BOOT_FILES_DIR=$(RACKET_DIR)/lib
else
vnd-deps: $(VND_RACKET_BC)
VND_CFLAGS+=$(VND_RACKET_BC_CFLAGS)
VND_LDFLAGS+=$(VND_RACKET_BC_LDFLAGS)
RACO=raco
endif
### racket download
$(RACKET_DIR)/.extracted: $(RACKET_FILE)
$(call extract-file,$(RACKET_DIR))
@ -74,15 +120,4 @@ $(RACKET_DIR)/.extracted: $(RACKET_FILE)
$(RACKET_FILE):
$(call download-file,$(RACKET_URL))
## common
.PHONY: vnd-deps-clean vnd-deps
VND_CFLAGS=$(VND_MONOCYPHER_CFLAGS) $(VND_RACKET_BC_CFLAGS)
VND_LDFLAGS=$(VND_MONOCYPHER_LDFLAGS) $(VND_RACKET_BC_LDFLAGS)
vnd-deps: $(VND_MONOCYPHER) $(VND_RACKET_BC)
mrproper: vnd-deps-clean
vnd-deps-clean:
$(RM) -r $(VENDOR_DIR) $(CACHE_DIR)
vnd-deps-dl: $(RACKET_FILE)

View File

@ -0,0 +1,131 @@
// crossfire: distributed brute force infrastructure
//
// Copyright (C) 2020 haskal
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#pragma once
#include <sys/types.h>
#include <elf.h>
#include <fcntl.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdint.h>
#include <stdbool.h>
#include <sys/mman.h>
#include <errno.h>
#define _SELF_NAME "/proc/self/exe"
#if ELF_BITS == 32
#define Elf_Ehdr Elf32_Ehdr
#define Elf_Shdr Elf32_Shdr
#elif ELF_BITS == 64
#define Elf_Ehdr Elf64_Ehdr
#define Elf_Shdr Elf64_Shdr
#endif
#define CHK_GENERIC(what, ret) if (what) { return ret; }
#define CHK_NULL_NULL(what) CHK_GENERIC(what == NULL, NULL)
#define CHK_NEG_NEG(what) CHK_GENERIC(what < 0, -1)
static inline char* readlink_exactly(const char* name) {
ssize_t len;
ssize_t blen = 256;
char* s = malloc(blen);
CHK_NULL_NULL(s);
while (true) {
len = readlink(name, s, blen - 1);
if (len == (blen - 1)) {
free(s);
blen *= 2;
s = malloc(blen);
CHK_NULL_NULL(s);
} else if (len < 0) {
return NULL;
} else {
return s;
}
}
}
static inline int readexactly(int fd, uint8_t* buf, size_t size) {
for (size_t i = 0; i < size;) {
ssize_t res = read(fd, &buf[i], size - i);
if (res < 0) {
if (errno == EINTR) { continue; }
return -1;
}
i += res;
}
return size;
}
typedef struct {
char* self_name;
size_t self_size;
void* self_mm;
Elf_Ehdr* ehdr;
char* strs;
} boot_ctx;
typedef struct {
off_t offset;
size_t size;
} boot_lookup;
static int init_boot_ctx(boot_ctx* ctx) {
// i fucking hate C
char* self_name = readlink_exactly(_SELF_NAME);
CHK_GENERIC(self_name == NULL, -1);
ctx->self_name = self_name;
int fd = open(self_name, O_RDONLY);
CHK_NEG_NEG(fd);
ssize_t len = lseek(fd, 0, SEEK_END);
CHK_NEG_NEG(len);
ctx->self_size = len;
void* map = mmap(NULL, len, PROT_READ, MAP_SHARED, fd, 0);
CHK_GENERIC(map == MAP_FAILED, -1);
CHK_NEG_NEG(close(fd));
ctx->self_mm = map;
ctx->ehdr = map;
off_t shdr_off = ctx->ehdr->e_shoff + (ctx->ehdr->e_shstrndx * ctx->ehdr->e_shentsize);
Elf_Shdr* shdr = (void*) (((uint8_t*) map) + shdr_off);
ctx->strs = ((uint8_t*) map) + shdr->sh_offset;
return 0;
}
static boot_lookup boot_ctx_lookup(boot_ctx* ctx, const char* sectname) {
boot_lookup bl = { (off_t) -1, (size_t) -1 };
for (size_t i = 0; i < ctx->ehdr->e_shnum; i++) {
Elf_Shdr* shdr = (void*) (((uint8_t*) ctx->self_mm)
+ ctx->ehdr->e_shoff + (i * ctx->ehdr->e_shentsize));
if (!strcmp(ctx->strs + shdr->sh_name, sectname)) {
bl.offset = shdr->sh_offset;
bl.size = shdr->sh_size;
return bl;
}
}
return bl;
}
static void unload_boot_ctx(boot_ctx* ctx) {
free(ctx->self_name);
if (munmap(ctx->self_mm, ctx->self_size) < 0) {
printf("NO SHONKS\n");
exit(-1);
}
}

View File

@ -16,27 +16,20 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#define MZ_PRECISE_GC
#include "crossfire-embedding.h"
#include <time.h>
#include <unistd.h>
#include <racket/scheme.h>
#include <stdint.h>
#include <stdio.h>
#include <monocypher.h>
// application embedding
#define rkt_app_start _binary_app_zo_start
#define rkt_app_end _binary_app_zo_end
extern const char rkt_app_start;
extern const char rkt_app_end;
// ffi defs
typedef struct {
const char* name;
uintptr_t ptr;
} ffi_ent;
#define STR(x) #x
#define FFI_ENT(name) {STR(name), (uintptr_t) name}
// runtime hacks
void rktio_init_cpu(void* rktio);
int rktio_processor_count(void* rktio);
@ -54,6 +47,11 @@ static const ffi_ent ffi_table[] = {
FFI_ENT(crypto_blake2b_update),
FFI_ENT(crypto_blake2b_final),
FFI_ENT(clock_gettime),
FFI_ENT(gethostname),
FFI_ENT(fsync),
// hacks
FFI_ENT(rktio_init_cpu),
FFI_ENT(rktio_processor_count)
};

163
agent-deployment/main_cs.c Normal file
View File

@ -0,0 +1,163 @@
// crossfire: distributed brute force infrastructure
//
// Copyright (C) 2020 haskal
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#include "crossfire-embedding.h"
#include "find-section.h"
#include <stdio.h>
#include <stdarg.h>
#include <time.h>
#include <unistd.h>
#include <chezscheme.h>
#include <racketcs.h>
#include <monocypher.h>
#define error() \
fprintf(stderr, "[WRAPPER] fatal boot error... corrupted binary?\n"); \
return -1;
static const ffi_ent ffi_table[] = {
FFI_ENT(crypto_sign_public_key),
FFI_ENT(crypto_sign),
FFI_ENT(crypto_check),
FFI_ENT(crypto_key_exchange_public_key),
FFI_ENT(crypto_key_exchange),
FFI_ENT(crypto_lock),
FFI_ENT(crypto_unlock),
FFI_ENT(crypto_wipe),
FFI_ENT(crypto_blake2b_init),
FFI_ENT(crypto_blake2b_update),
FFI_ENT(crypto_blake2b_final),
FFI_ENT(clock_gettime),
FFI_ENT(gethostname),
FFI_ENT(fsync)
};
static const size_t ffi_table_size = sizeof(ffi_table)/sizeof(ffi_ent);
// yeet
// chez people don't yell at me pls
static ptr scheme_list(int count, ...) {
va_list ap;
va_start(ap, count);
ptr vals[count];
for (int i = 0; i < count; i++) {
ptr arg = va_arg(ap, ptr);
vals[i] = arg;
}
va_end(ap);
ptr val = Snil;
for (int i = 0; i < count; i++) {
val = Scons(vals[count - i - 1], val);
}
return val;
}
static void cs_setup_ffi_table() {
// this is pain
// chez scheme is pain
// why is it like this
// gimme my scheme_primitive_module holy shit pls,,,,
racket_namespace_require(Sstring_to_symbol("racket/base"));
// also since this is '#%kernel level code, technically it's all UB
// which is fun
ptr module = Sstring_to_symbol("module");
ptr __module_begin = Sstring_to_symbol("#%module-begin");
ptr quote = Sstring_to_symbol("quote");
ptr __kernel = Sstring_to_symbol("#%kernel");
ptr __provide = Sstring_to_symbol("#%provide");
ptr define_values = Sstring_to_symbol("define-values");
ptr __static_ffi = Sstring_to_symbol("#%static-ffi");
ptr table_n = Sstring_to_symbol("table");
ptr table_size_n = Sstring_to_symbol("table-size");
ptr arch_n = Sstring_to_symbol("arch");
ptr table_e = scheme_list(2, quote, Sunsigned64((uint64_t) &ffi_table[0]));
ptr table_size_e = scheme_list(2, quote, Sunsigned64(ffi_table_size));
ptr arch_e = scheme_list(2, quote, Sstring(APP_ARCH));
ptr table_d = scheme_list(3, define_values, scheme_list(1, table_n), table_e);
ptr table_size_d = scheme_list(3, define_values, scheme_list(1, table_size_n), table_size_e);
ptr arch_d = scheme_list(3, define_values, scheme_list(1, arch_n), arch_e);
ptr e = scheme_list(4,
module,
__static_ffi,
scheme_list(2, quote, __kernel),
scheme_list(5,
__module_begin,
scheme_list(4, __provide, table_n, table_size_n, arch_n),
table_d,
table_size_d,
arch_d));
// evaling this magically registers it in the default namespace
// it's magic
// don't ask questions
racket_eval(e);
}
int main(int argc, char* argv[]) {
printf("[WRAPPER] booting racket CS\n");
boot_ctx ctx;
if (init_boot_ctx(&ctx) < 0) {
error();
}
racket_boot_arguments_t args;
memset(&args, 0, sizeof(args));
args.boot1_path = ctx.self_name;
args.boot2_path = ctx.self_name;
args.boot3_path = ctx.self_name;
boot_lookup boot1 = boot_ctx_lookup(&ctx, ".csboot1");
boot_lookup boot2 = boot_ctx_lookup(&ctx, ".csboot2");
boot_lookup boot3 = boot_ctx_lookup(&ctx, ".csboot3");
if (boot1.offset == (off_t) -1 ||
boot2.offset == (off_t) -1 ||
boot3.offset == (off_t) -1) {
error();
}
args.boot1_offset = boot1.offset;
args.boot2_offset = boot2.offset;
args.boot3_offset = boot3.offset;
args.exec_file = argv[0];
racket_boot(&args);
boot_lookup app_offset = boot_ctx_lookup(&ctx, ".boot");
if (app_offset.offset == (off_t) -1) {
error();
}
racket_embedded_load_bytes(((uint8_t*) ctx.self_mm) + app_offset.offset,
app_offset.size, false);
unload_boot_ctx(&ctx);
cs_setup_ffi_table();
ptr mod = Scons(Sstring_to_symbol("quote"), Scons(Sstring_to_symbol(APP_NAME), Snil));
racket_dynamic_require(mod, Sfalse);
return 0;
}

View File

@ -18,7 +18,7 @@
(require file/untgz (only-in file/sha1 bytes->hex-string) racket/async-channel racket/bool
racket/cmdline racket/contract racket/fasl racket/file racket/function racket/match
racket/path racket/port racket/string racket/unit srfi/19
racket/path racket/port racket/string racket/unit
"comms.rkt" "info.rkt" "logging.rkt" "not-crypto.rkt" "manifest.rkt" "protocol.rkt"
"pattern.rkt" "static-support.rkt"
(submod "static-support.rkt" misc-calls))
@ -328,6 +328,8 @@
(install-logging!)
(log-agent-info "starting crossfire-agent v~a" (#%info-lookup 'version))
(log-agent-info "ffi mode: ~a" (if (static-ffi-available?) "static" "regular"))
(log-agent-info "hostname ~a" (get-hostname))
(log-agent-info "arch ~a" (static-ffi-arch))
(log-agent-info "~a cpus available" (count-cpus))
(current-queue (make-async-channel))

View File

@ -21,7 +21,7 @@
(define pkg-authors '(haskal))
(define collection "crossfire")
(define deps '("base" "db-lib" "scribble-text-lib" "srfi-lite-lib"
(define deps '("base" "db-lib" "scribble-text-lib"
"north"))
(define build-deps '("scribble-lib" "racket-doc" "rackunit-lib"))
(define scribblings '(("scribblings/crossfire.scrbl" ())))

View File

@ -19,7 +19,7 @@
(require db/base db/sqlite3
data/queue racket/async-channel racket/bool racket/contract racket/fasl racket/file
racket/function racket/list racket/logging racket/match racket/path racket/random
racket/runtime-path racket/set racket/string racket/unit srfi/19
racket/runtime-path racket/set racket/string racket/unit
north/base north/adapter/base north/adapter/sqlite
"comms.rkt" "info.rkt" "logging.rkt" "manifest.rkt" "not-crypto.rkt" "pattern.rkt"
"protocol.rkt"

View File

@ -86,7 +86,7 @@
;; misc ffi calls not provided by racket (because ????)
;; racket should have fsync like cmon
(module+ misc-calls
(require ffi/unsafe/port ffi/vector racket/list racket/match racket/string)
(require ffi/unsafe/port ffi/vector racket/future racket/list racket/match racket/string)
(provide port-fsync count-cpus current-seconds-monotonic get-hostname)
@ -142,8 +142,8 @@
;; this provides an actual count of the number of CPUs on the system, even without
;; --enable-futures by hooking into the underlying rktio call that gets skipped when the current
;; VM is configured without --enable-futures
;; XXX : i'm not entirely sure what the actual processor-count call looks like on chez, and it
;; probably doesn't suffer from the same issue. i'll get back to this when chez is the default vm
;; XXX : on chez scheme, futures are always enabled (?) therefore we just use the futures call
;; directly. however, this hasn't been tested on ARM yet
(define count-cpus/bc
(let ([num-cpus #f])
(lambda ()
@ -162,6 +162,7 @@
(define (count-cpus)
(match (system-type 'vm)
['racket (count-cpus/bc)]
['chez-scheme (processor-count)]
[x (error "don't know how to count-cpus on vm" x)]))