From 280979a479e06f05212cbf44fb265b72225e3404 Mon Sep 17 00:00:00 2001 From: haskal Date: Mon, 28 Dec 2020 03:41:33 -0500 Subject: [PATCH] implement some of the client --- crossfire/agent.rkt | 1 - crossfire/client.rkt | 232 +++++++++++++++++++++---- crossfire/codegen.rkt | 32 ++-- crossfire/manifest.rkt | 5 +- crossfire/not-crypto.rkt | 21 ++- crossfire/server.rkt | 35 ++-- crossfire/templates/Makefile.rkt | 64 +++++++ crossfire/{ => templates}/codegen.rktc | 2 + crossfire/templates/manifest.rktrktd | 59 +++++++ 9 files changed, 388 insertions(+), 63 deletions(-) create mode 100644 crossfire/templates/Makefile.rkt rename crossfire/{ => templates}/codegen.rktc (99%) create mode 100644 crossfire/templates/manifest.rktrktd diff --git a/crossfire/agent.rkt b/crossfire/agent.rkt index 426babb..f6d5b59 100644 --- a/crossfire/agent.rkt +++ b/crossfire/agent.rkt @@ -21,7 +21,6 @@ racket/path racket/port racket/string racket/unit srfi/19 "comms.rkt" "info.rkt" "logging.rkt" "not-crypto.rkt" "manifest.rkt" "protocol.rkt" "pattern.rkt" "static-support.rkt" - ;; port-fsync (submod "static-support.rkt" misc-calls)) (provide agent-main) diff --git a/crossfire/client.rkt b/crossfire/client.rkt index ca94c62..3a2bf5b 100644 --- a/crossfire/client.rkt +++ b/crossfire/client.rkt @@ -16,63 +16,223 @@ ;; You should have received a copy of the GNU Affero General Public License ;; along with this program. If not, see . -(require racket/function racket/list racket/match racket/vector - "comms.rkt" "info.rkt") +(require file/tar racket/bool racket/file racket/function racket/list racket/match racket/path + racket/string racket/unit racket/vector syntax/parse/define + (only-in file/sha1 bytes->hex-string) + (for-syntax racket/base racket/syntax) + "codegen.rkt" "comms.rkt" "info.rkt" "manifest.rkt" "not-crypto.rkt" "protocol.rkt") (define *program* (format "~a-client" (#%info-lookup 'collection))) +;; XXX : slightly unix-oriented +(define *config-root* + (build-path + (or (getenv "XDG_CONFIG_HOME") + (build-path (find-system-path 'home-dir) ".config")) + *program*)) +(define *client-key-file* (build-path *config-root* "config.rktd")) -(define (cmd-new project-name) - (printf "creating new project ~a\n" project-name)) +(define *cf-private-cache* (build-path ".crossfire" "cache.rktd")) +(define *cf-tmp-targz* (build-path ".crossfire" "project.tgz")) -(define (cmd-submit) - (displayln "submitting...")) +(define (cmd-new project-name mode) + (define project-root (build-path project-name)) + (when (or (file-exists? project-root) (directory-exists? project-root) + (link-exists? project-root)) + (error "file or directory already exists named" project-name)) + (define proj-dir (build-path project-name)) + (define priv-cache (build-path proj-dir *cf-private-cache*)) + + (make-directory* proj-dir) + (make-parent-directory* priv-cache) + (call-with-output-file priv-cache (lambda (out) (write (hash) out))) + + (define mf-gen (generate-manifest mode)) + (call-with-output-file + (build-path proj-dir "manifest.rktd") + (lambda (out) (write-string mf-gen out))) + + (void)) + + +(define (cmd-update project-dir) + (define priv-cache (build-path project-dir *cf-private-cache*)) + (define mf (parse-manifest (call-with-input-file (build-path project-dir "manifest.rktd") read))) + (define new-files (generate-support-code mf)) + (define old-hashes (call-with-input-file priv-cache read)) + + (define messages '()) + + (define new-hashes + (for/hash ([(name contents) (in-hash new-files)]) + (define v-hash (port->blake2b-hash (open-input-string contents))) + (define old-hash (hash-ref old-hashes name #f)) + (cond + [(not (file-exists? name)) + (call-with-output-file (build-path project-dir name) + (lambda (out) (write-string contents out))) + (set! messages (cons (format "created file ~a" name) messages))] + [(and old-hash (not (bytes=? old-hash v-hash))) + (define new-name (string-append name "." (substring (bytes->hex-string old-hash) 0 32))) + (rename-file-or-directory (build-path project-dir name) (build-path project-dir new-name)) + (call-with-output-file (build-path project-dir name) + (lambda (out) (write-string contents out))) + (define msg + (format + "because of manifest updates, ~a has been regenerated. the old version was moved to ~a" + name new-name)) + (set! messages (cons msg messages))]) + (values name v-hash))) + + (call-with-output-file priv-cache (lambda (out) (write new-hashes out)) #:exists 'replace) + messages) + + +(define (cmd-check project-dir) + (define messages '()) + + (define mf (parse-manifest (call-with-input-file (build-path project-dir "manifest.rktd") read))) + (define mode (first (manifest-data-ref mf 'mode))) + + (when (and (symbol=? mode 'stdio) + (not (file-exists? (build-path project-dir "crossfire-generator")))) + (set! messages (cons "missing crossfire-generator, make sure to compile it" messages))) + + (unless (file-exists? *client-key-file*) + (set! messages (cons (format "missing configuration to connect to server. use ~a setup" + *program*) + messages))) + + messages) + + +(define (cmd-submit project-dir) + (define mf (parse-manifest (call-with-input-file (build-path project-dir "manifest.rktd") read))) + (match-define (list client-node server-node) (call-with-input-file *client-key-file* read)) + + ;; create targz + (define tmp-targz (build-path project-dir *cf-tmp-targz*)) + (parameterize ([current-directory project-dir]) + (define out (current-output-port)) + (tar-gzip tmp-targz "." + #:exists-ok? #t + #:path-filter (lambda (p) + (match (explode-path p) + [(list 'same (== (build-path ".crossfire")) rst ...) #f] + [(list 'same fst rst ...) #t] + [(list 'same) #t] + [_ (error "not shonks...")])))) + + ;; connect to server + (define server-wrapper@ (make-rpc-wrapper-unit server^)) + (define-values/invoke-unit server-wrapper@ (import) (export server^)) + + (current-to-node server-node) + (current-comms (make-comms client-node)) + (current-tm (make-transaction-manager client-node (current-comms))) + (comms-set-node-info (current-comms) server-node) + (comms-connect (current-comms) (node-id server-node)) + + (displayln "yeet") + (new-project (serialize-manifest mf) (file->bytes tmp-targz)) + + (tm-shutdown (current-tm)) + (comms-shutdown (current-comms)) + (void)) + + +(define (cmd-setup config) + (make-parent-directory* *client-key-file*) + (with-output-to-file *client-key-file* (lambda () (write config)) #:exists 'replace)) -(define (cmd-setup config-file) - (displayln "meow")) (module+ main (require racket/cmdline) - (define (report-fatal-error cmd msg) - (printf "~a ~a: ~a\n" *program* cmd msg) + (define current-subcommand (make-parameter #f)) + + (define (report-status msg . args) + (printf "~a ~a: ~a\n" *program* (current-subcommand) (apply format msg args))) + + (define (report-fatal-error msg . args) + (apply report-status msg args) (exit 1)) - (define (parse-new argv) - (command-line - #:program (format "~a new" *program*) - #:argv argv + (define (interactive-update) + (match (cmd-update (current-directory)) + ['() (report-status "generated files are up-to-date")] + [(list msgs ...) + (for ([msg (in-list msgs)]) + (report-status "~a" msg))])) + + (define (interactive-check) + (match (cmd-check (current-directory)) + ['() (report-status "everything looks good to me!")] + [(list warnings ...) + (for ([warning (in-list warnings)]) + (report-status "error: ~a" warning)) + (report-fatal-error "check was not successful due to above messages")])) + + (define-simple-macro (subcommand (name:id description:str) body ...) + #:with name-str (datum->syntax #'name (symbol->string (syntax-e #'name))) + (list name-str + (lambda (argv) + (parameterize ([current-subcommand name-str]) + (with-handlers ([exn:fail? + (lambda (ex) + (report-fatal-error "~a" (exn-message ex)))]) + (command-line + #:program (format "~a ~a" *program* name-str) + #:argv argv + #:usage-help description + body ...)))) + description)) + + (define-simple-macro (define-commands name:id cmds ...) + (define name (make-hash (list cmds ...)))) + + (define flag-mode (make-parameter "stdio")) + (define-commands *commands* + (subcommand (new "Create a new crossfire project") + #:once-each [("-m" "--mode") mode "Project mode (stdio [default] or callback)" + (flag-mode mode)] #:args (project-name) - (cmd-new project-name))) + (match (flag-mode) + [(or "stdio" "callback") (void)] + [x (report-fatal-error "invalid mode ~a, need stdio or callback" x)]) + (cmd-new project-name (string->symbol (flag-mode))) + (report-status "created new project ~a" project-name)) - (define (parse-submit argv) - (command-line - #:program (format "~a submit" *program*) - #:argv argv + (subcommand (update "Update generated files in a project") #:args () - (cmd-submit))) + (interactive-update)) - (define (parse-setup argv) - (command-line - #:program (format "~a setup" *program*) - #:argv argv + (subcommand (check "Check over a project for issues") + #:args () + (interactive-check)) + + (subcommand (submit "Submit a project for execution") + #:args () + ;; trigger a check before we submit + (interactive-check) + ;; do submit + (report-status "submitting project...!") + (cmd-submit (current-directory))) + + (subcommand (setup "Set up access to a crossfire server") #:args (config-file) (unless (file-exists? config-file) - (report-fatal-error "setup" "provided config file does not exist")) + (report-fatal-error "provided config file does not exist")) (define config - (with-handlers ([exn:fail? - (lambda (ex) (report-fatal-error "setup" (exn-message ex)))]) - (call-with-input-file config-file read))) - (cmd-setup config))) - - (define *commands* (hash "setup" (list parse-setup "Set up server config") - "new" (list parse-new "Create a new project") - "submit" (list parse-submit "Submit a project for execution"))) + (call-with-input-file config-file read)) + (cmd-setup config) + (report-status "successfully imported config"))) (define (print-usage) - (displayln "crossfire-client [ ...]") + (printf "~a [ ...]\n" *program*) + (displayln " The CLI for crossfire") (displayln " where is one of") - (for ([(k v) (in-hash *commands*)]) - (printf " ~a : ~a\n" k (second v)))) + (for ([k (in-list (sort (hash-keys *commands*) stringstring template-path)))))) + (scribble-output output-exp out) + (get-output-string out)) ;; there used to be a racket implementation of the input generator but it majorly violated the ;; design recipe and got obsoleted by the template C version (which is intended to be more portable ;; and faster) ;; ok gamer move time -(define-runtime-path codegen-template "codegen.rktc") -(define (pattern-codegen pattern mode) +(define-runtime-path codegen-template "templates/codegen.rktc") +(define-runtime-path makefile-template "templates/Makefile.rkt") +(define-runtime-path manifest-template "templates/manifest.rktrktd") + +(define (generate-support-code mf) + (define vars (hash 'pattern (vector-map integer-set-contents (manifest-pattern mf)) + 'mode (first (manifest-data-ref mf 'mode)))) + (hash + "crossfire.c" (eval-template codegen-template vars) + "Makefile" (eval-template makefile-template vars))) + +(define (generate-manifest mode) (eval-template - `(file ,(path->string codegen-template)) - (hash 'pattern (vector-map integer-set-contents pattern) - 'mode mode))) + manifest-template + (hash 'mode mode))) diff --git a/crossfire/manifest.rkt b/crossfire/manifest.rkt index 250ac0b..72074e8 100644 --- a/crossfire/manifest.rkt +++ b/crossfire/manifest.rkt @@ -47,7 +47,8 @@ '?d (string->integer-set "0123456789") '?s (string->integer-set " !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") '?a (range->integer-set 32 126) - '?b (range->integer-set 0 255))) + '?b (range->integer-set 0 255) + '?? (char->integer-set #\?))) (provide builtin-isets) ;; manifest contracts and definition @@ -62,7 +63,7 @@ (list/c 'smp boolean?) (list/c 'mode (or/c 'stdio 'callback)) (cons/c 'command (listof string?)) - (cons/c 'iset (listof isub/c)) + (cons/c 'iset (cons/c symbol? (listof isub/c))) (cons/c 'pattern (listof isub/c))))) (struct manifest [data pattern psize] #:transparent) diff --git a/crossfire/not-crypto.rkt b/crossfire/not-crypto.rkt index ad1d86a..11933d3 100644 --- a/crossfire/not-crypto.rkt +++ b/crossfire/not-crypto.rkt @@ -16,7 +16,7 @@ ;; You should have received a copy of the GNU Affero General Public License ;; along with this program. If not, see . -(require ffi/vector ffi/unsafe racket/bool racket/random +(require ffi/vector ffi/unsafe racket/bool racket/match racket/random "static-support.rkt" (for-syntax racket/base racket/string) (only-in racket/contract provide/contract) @@ -137,3 +137,22 @@ (define/ffi crypto-wipe (_fun (buf : _u8vector) (sz : _size = (u8vector-length buf)) -> _void)) (provide crypto-wipe) + +;; helpers + +(define *read-size* 8192) + +(define (port->blake2b-hash port) + (define ctx (crypto-blake2b-init)) + (let loop () + (match (read-bytes *read-size* port) + [(? eof-object?) (void)] + [data (crypto-blake2b-update ctx data) + (loop)])) + (crypto-blake2b-final ctx)) +(provide port->blake2b-hash) + +(define (file->blake2b-hash path) + (call-with-input-file path port->blake2b-hash)) + +(provide file->blake2b-hash) diff --git a/crossfire/server.rkt b/crossfire/server.rkt index fd21097..334642f 100644 --- a/crossfire/server.rkt +++ b/crossfire/server.rkt @@ -21,8 +21,8 @@ racket/function racket/list racket/logging racket/match racket/path racket/random racket/runtime-path racket/set racket/string racket/unit srfi/19 north/base north/adapter/base north/adapter/sqlite - "comms.rkt" "info.rkt" "logging.rkt" "manifest.rkt" "not-crypto.rkt" "pattern.rkt" "protocol.rkt" - ;; port-fsync + "comms.rkt" "info.rkt" "logging.rkt" "manifest.rkt" "not-crypto.rkt" "pattern.rkt" + "protocol.rkt" (submod "static-support.rkt" misc-calls)) ;; logging @@ -98,6 +98,7 @@ (define-stmt q-get-node-resources "select resource from node_resource where nodeid=?") (define-stmt q-edit-node "update node set name=? where id=?") (define-stmt q-delete-node "delete from node where id=?") +(define-stmt q-delete-node-type "delete from node where type=?") (define-stmt q-get-node-type "select type from node where id=?") (define-stmt q-get-node-info "select name, arch, type, secret from node where id=?") @@ -162,16 +163,7 @@ ;; computes a hash of the file identifying its current contents for agents ;; (in case we reuse taskids) (define (server-hash-file taskid) - (define path (get-project-file-path taskid)) - (define *read-size* 8192) - (call-with-input-file path (lambda (in) - (define ctx (crypto-blake2b-init)) - (let loop () - (match (read-bytes *read-size* in) - [(? eof-object?) (void)] - [data (crypto-blake2b-update ctx data) - (loop)])) - (crypto-blake2b-final ctx)))) + (file->blake2b-hash (get-project-file-path taskid))) (define (server-get-file taskid) (define path (get-project-file-path taskid)) @@ -188,7 +180,7 @@ (query (current-db) q-get-all-resources type-str))) (for/list ([(id name arch) (in-query (current-db) q-get-nodes type-str)]) (define online? (and (current-comms) (comms-channel-available? (current-comms) id))) - (node-info id name arch type (hash-ref resources id) online?))) + (node-info id name arch type (hash-ref resources id '()) online?))) (define (make-node name arch type resources) (call-with-transaction (current-db) (lambda () @@ -859,6 +851,11 @@ (write-bytes (crypto-sign-make-key) out) (port-fsync out)))) + ;; create client if it doesn't exist yet + (when (empty? (get-nodes 'client)) + (log-server-info "creating new client") + (make-node "client0" "x-client" 'client '())) + (define seckey (file->bytes *server-seckey-path*)) (define pubkey (crypto-sign-public-key seckey)) (define listen-addr @@ -892,6 +889,13 @@ (lambda (out) (write (list agent-node (current-server-public-node)) out))) (log-server-info "created dev agent ~a" name) (exit)] + [(vector "dev-export-client") + (define client-id (node-info-id (first (get-nodes 'client)))) + (define client-node (load-comms-node client-id #t)) + (define data (list client-node (current-server-public-node))) + (with-output-to-file (build-path *state-root* "client0.rktd") + (lambda () (write data))) + (exit)] [(vector "dev-new-project" tgz) (define id (make-task '((name "test project") (arch "any") (command "./test.sh") @@ -919,6 +923,11 @@ (agent-handler-new-agent (node-info-id agent) (node-info-arch agent) (node-info-resources agent))) + ;; restore clients + (for ([client (in-list (get-nodes 'client))]) + (log-server-info "restoring client ~a" (node-info-id client)) + (restore-comms-node (node-info-id client))) + ;; restore active tasks (for ([(id name manifest-in complete?) (in-query (current-db) q-get-tasks)] #:when (zero? complete?)) diff --git a/crossfire/templates/Makefile.rkt b/crossfire/templates/Makefile.rkt new file mode 100644 index 0000000..93e09cc --- /dev/null +++ b/crossfire/templates/Makefile.rkt @@ -0,0 +1,64 @@ +@;; crossfire: distributed brute force infrastructure +@;; +@;; Copyright (C) 2020 haskal +@;; +@;; This program is free software: you can redistribute it and/or modify +@;; it under the terms of the GNU Affero General Public License as published by +@;; the Free Software Foundation, either version 3 of the License, or +@;; (at your option) any later version. +@;; +@;; This program is distributed in the hope that it will be useful, +@;; but WITHOUT ANY WARRANTY; without even the implied warranty of +@;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@;; GNU Affero General Public License for more details. +@;; +@;; You should have received a copy of the GNU Affero General Public License +@;; along with this program. If not, see . +@;; vim: ft=racket +@;; +.PHONY: all clean + +CC=gcc +CFLAGS=-std=c11 -O3 +LDFLAGS=-static + +@(match mode + ['stdio @list{ +# TODO : your rules here + +all: ... + +clean: ... + + +# crossfire rules + +CF_GEN=crossfire-generator + +all: $(CF_GEN) +$(CF_GEN): crossfire.o +@"\t"$(CC) -o $@"@" $(CFLAGS) $(LDFLAGS) $< + +.PHONY: crossfire-clean +clean: crossfire-clean +crossfire-clean: +@"\t"$(RM) crossfire.o $(CF_GEN) + }] + ['callback @list{ +# TODO: include crossfire.o in your application + +all: ... + +clean: ... + + +# crossfire rules + +.PHONY: crossfire-clean +clean: crossfire-clean +crossfire-clean: +@"\t"$(RM) crossfire.o + }]) + +crossfire.o: crossfire.c +@"\t"$(CC) -c -o $@"@" $(CFLAGS) $< diff --git a/crossfire/codegen.rktc b/crossfire/templates/codegen.rktc similarity index 99% rename from crossfire/codegen.rktc rename to crossfire/templates/codegen.rktc index cfe35f5..e22278f 100644 --- a/crossfire/codegen.rktc +++ b/crossfire/templates/codegen.rktc @@ -98,6 +98,8 @@ int crossfire_main(int argc, char* argv[], callback cb) { @(define fmt (string-join (for/list ([i (in-range (vector-length pattern))]) c-type-fmt) " ")) + void cf_report_success( @arg-vs ); + l_inner: @(define (end-conditionals) diff --git a/crossfire/templates/manifest.rktrktd b/crossfire/templates/manifest.rktrktd new file mode 100644 index 0000000..9aa8497 --- /dev/null +++ b/crossfire/templates/manifest.rktrktd @@ -0,0 +1,59 @@ +@;; crossfire: distributed brute force infrastructure +@;; +@;; Copyright (C) 2020 haskal +@;; +@;; This program is free software: you can redistribute it and/or modify +@;; it under the terms of the GNU Affero General Public License as published by +@;; the Free Software Foundation, either version 3 of the License, or +@;; (at your option) any later version. +@;; +@;; This program is distributed in the hope that it will be useful, +@;; but WITHOUT ANY WARRANTY; without even the implied warranty of +@;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@;; GNU Affero General Public License for more details. +@;; +@;; You should have received a copy of the GNU Affero General Public License +@;; along with this program. If not, see . +@;; vim: ft=racket +@;; +(;; name of the project + (name "my cool project") + ;; architectures it runs on (eg "x86_64-pc-linux-gnu") or "any" + (arch "x86_64-pc-linux-gnu") + ;; resources needed + (resources "cpu") + ;; #t to let crossfire perform multithreading for you + ;; #f if your application performs its own multithreading or should not be multithreaded + (smp #f) + ;; mode + ;; - stdio: pass input to the given command (below) via stdin, space separated hex + ;; - callback: call a C callback function in your program with inputs + ;; in callback mode, you must integrate the provided input generator with your program + (mode @mode) + ;; command to execute (arguments will be appended in callback mode) + (command "/usr/bin/env" "python3" "myscript.py") + + ;; custom pattern definitions + ;; examples: + ;; (iset ?z "0123456789" ?l) + ;; (iset ?z (#x30 #x39) "abcdefghij" "klmn" "opqrstuv") + ;; (iset ?y ?d) + + ;; builtin patterns + ;; ?l - lowercase + ;; ?u - uppercase + ;; ?d - numbers + ;; ?s - symbols + ;; ?a - ascii + ;; ?b - any byte + ;; ?? - escape for literal question mark + + ;; brute force pattern + ;; examples (each element is concatenated) + ;; (pattern "meow?d") + ;; (pattern "meow" ?d) + ;; (pattern "me" "ow" (#x30 #x39)) + ;; (pattern "meow" (48 57)) + ;; up to uint64_t range numbers also supported + ;; (pattern (12345 67890) (10 20) "x") + (pattern "meow?d?d"))