implement some of the client

This commit is contained in:
xenia 2020-12-28 03:41:33 -05:00
parent bcfb26b0d6
commit 280979a479
9 changed files with 388 additions and 63 deletions

View File

@ -21,7 +21,6 @@
racket/path racket/port racket/string racket/unit srfi/19
"comms.rkt" "info.rkt" "logging.rkt" "not-crypto.rkt" "manifest.rkt" "protocol.rkt"
"pattern.rkt" "static-support.rkt"
;; port-fsync
(submod "static-support.rkt" misc-calls))
(provide agent-main)

View File

@ -16,63 +16,223 @@
;; You should have received a copy of the GNU Affero General Public License
;; along with this program. If not, see <https://www.gnu.org/licenses/>.
(require racket/function racket/list racket/match racket/vector
"comms.rkt" "info.rkt")
(require file/tar racket/bool racket/file racket/function racket/list racket/match racket/path
racket/string racket/unit racket/vector syntax/parse/define
(only-in file/sha1 bytes->hex-string)
(for-syntax racket/base racket/syntax)
"codegen.rkt" "comms.rkt" "info.rkt" "manifest.rkt" "not-crypto.rkt" "protocol.rkt")
(define *program* (format "~a-client" (#%info-lookup 'collection)))
;; XXX : slightly unix-oriented
(define *config-root*
(build-path
(or (getenv "XDG_CONFIG_HOME")
(build-path (find-system-path 'home-dir) ".config"))
*program*))
(define *client-key-file* (build-path *config-root* "config.rktd"))
(define (cmd-new project-name)
(printf "creating new project ~a\n" project-name))
(define *cf-private-cache* (build-path ".crossfire" "cache.rktd"))
(define *cf-tmp-targz* (build-path ".crossfire" "project.tgz"))
(define (cmd-submit)
(displayln "submitting..."))
(define (cmd-new project-name mode)
(define project-root (build-path project-name))
(when (or (file-exists? project-root) (directory-exists? project-root)
(link-exists? project-root))
(error "file or directory already exists named" project-name))
(define proj-dir (build-path project-name))
(define priv-cache (build-path proj-dir *cf-private-cache*))
(make-directory* proj-dir)
(make-parent-directory* priv-cache)
(call-with-output-file priv-cache (lambda (out) (write (hash) out)))
(define mf-gen (generate-manifest mode))
(call-with-output-file
(build-path proj-dir "manifest.rktd")
(lambda (out) (write-string mf-gen out)))
(void))
(define (cmd-update project-dir)
(define priv-cache (build-path project-dir *cf-private-cache*))
(define mf (parse-manifest (call-with-input-file (build-path project-dir "manifest.rktd") read)))
(define new-files (generate-support-code mf))
(define old-hashes (call-with-input-file priv-cache read))
(define messages '())
(define new-hashes
(for/hash ([(name contents) (in-hash new-files)])
(define v-hash (port->blake2b-hash (open-input-string contents)))
(define old-hash (hash-ref old-hashes name #f))
(cond
[(not (file-exists? name))
(call-with-output-file (build-path project-dir name)
(lambda (out) (write-string contents out)))
(set! messages (cons (format "created file ~a" name) messages))]
[(and old-hash (not (bytes=? old-hash v-hash)))
(define new-name (string-append name "." (substring (bytes->hex-string old-hash) 0 32)))
(rename-file-or-directory (build-path project-dir name) (build-path project-dir new-name))
(call-with-output-file (build-path project-dir name)
(lambda (out) (write-string contents out)))
(define msg
(format
"because of manifest updates, ~a has been regenerated. the old version was moved to ~a"
name new-name))
(set! messages (cons msg messages))])
(values name v-hash)))
(call-with-output-file priv-cache (lambda (out) (write new-hashes out)) #:exists 'replace)
messages)
(define (cmd-check project-dir)
(define messages '())
(define mf (parse-manifest (call-with-input-file (build-path project-dir "manifest.rktd") read)))
(define mode (first (manifest-data-ref mf 'mode)))
(when (and (symbol=? mode 'stdio)
(not (file-exists? (build-path project-dir "crossfire-generator"))))
(set! messages (cons "missing crossfire-generator, make sure to compile it" messages)))
(unless (file-exists? *client-key-file*)
(set! messages (cons (format "missing configuration to connect to server. use ~a setup"
*program*)
messages)))
messages)
(define (cmd-submit project-dir)
(define mf (parse-manifest (call-with-input-file (build-path project-dir "manifest.rktd") read)))
(match-define (list client-node server-node) (call-with-input-file *client-key-file* read))
;; create targz
(define tmp-targz (build-path project-dir *cf-tmp-targz*))
(parameterize ([current-directory project-dir])
(define out (current-output-port))
(tar-gzip tmp-targz "."
#:exists-ok? #t
#:path-filter (lambda (p)
(match (explode-path p)
[(list 'same (== (build-path ".crossfire")) rst ...) #f]
[(list 'same fst rst ...) #t]
[(list 'same) #t]
[_ (error "not shonks...")]))))
;; connect to server
(define server-wrapper@ (make-rpc-wrapper-unit server^))
(define-values/invoke-unit server-wrapper@ (import) (export server^))
(current-to-node server-node)
(current-comms (make-comms client-node))
(current-tm (make-transaction-manager client-node (current-comms)))
(comms-set-node-info (current-comms) server-node)
(comms-connect (current-comms) (node-id server-node))
(displayln "yeet")
(new-project (serialize-manifest mf) (file->bytes tmp-targz))
(tm-shutdown (current-tm))
(comms-shutdown (current-comms))
(void))
(define (cmd-setup config)
(make-parent-directory* *client-key-file*)
(with-output-to-file *client-key-file* (lambda () (write config)) #:exists 'replace))
(define (cmd-setup config-file)
(displayln "meow"))
(module+ main
(require racket/cmdline)
(define (report-fatal-error cmd msg)
(printf "~a ~a: ~a\n" *program* cmd msg)
(define current-subcommand (make-parameter #f))
(define (report-status msg . args)
(printf "~a ~a: ~a\n" *program* (current-subcommand) (apply format msg args)))
(define (report-fatal-error msg . args)
(apply report-status msg args)
(exit 1))
(define (parse-new argv)
(command-line
#:program (format "~a new" *program*)
#:argv argv
(define (interactive-update)
(match (cmd-update (current-directory))
['() (report-status "generated files are up-to-date")]
[(list msgs ...)
(for ([msg (in-list msgs)])
(report-status "~a" msg))]))
(define (interactive-check)
(match (cmd-check (current-directory))
['() (report-status "everything looks good to me!")]
[(list warnings ...)
(for ([warning (in-list warnings)])
(report-status "error: ~a" warning))
(report-fatal-error "check was not successful due to above messages")]))
(define-simple-macro (subcommand (name:id description:str) body ...)
#:with name-str (datum->syntax #'name (symbol->string (syntax-e #'name)))
(list name-str
(lambda (argv)
(parameterize ([current-subcommand name-str])
(with-handlers ([exn:fail?
(lambda (ex)
(report-fatal-error "~a" (exn-message ex)))])
(command-line
#:program (format "~a ~a" *program* name-str)
#:argv argv
#:usage-help description
body ...))))
description))
(define-simple-macro (define-commands name:id cmds ...)
(define name (make-hash (list cmds ...))))
(define flag-mode (make-parameter "stdio"))
(define-commands *commands*
(subcommand (new "Create a new crossfire project")
#:once-each [("-m" "--mode") mode "Project mode (stdio [default] or callback)"
(flag-mode mode)]
#:args (project-name)
(cmd-new project-name)))
(match (flag-mode)
[(or "stdio" "callback") (void)]
[x (report-fatal-error "invalid mode ~a, need stdio or callback" x)])
(cmd-new project-name (string->symbol (flag-mode)))
(report-status "created new project ~a" project-name))
(define (parse-submit argv)
(command-line
#:program (format "~a submit" *program*)
#:argv argv
(subcommand (update "Update generated files in a project")
#:args ()
(cmd-submit)))
(interactive-update))
(define (parse-setup argv)
(command-line
#:program (format "~a setup" *program*)
#:argv argv
(subcommand (check "Check over a project for issues")
#:args ()
(interactive-check))
(subcommand (submit "Submit a project for execution")
#:args ()
;; trigger a check before we submit
(interactive-check)
;; do submit
(report-status "submitting project...!")
(cmd-submit (current-directory)))
(subcommand (setup "Set up access to a crossfire server")
#:args (config-file)
(unless (file-exists? config-file)
(report-fatal-error "setup" "provided config file does not exist"))
(report-fatal-error "provided config file does not exist"))
(define config
(with-handlers ([exn:fail?
(lambda (ex) (report-fatal-error "setup" (exn-message ex)))])
(call-with-input-file config-file read)))
(cmd-setup config)))
(define *commands* (hash "setup" (list parse-setup "Set up server config")
"new" (list parse-new "Create a new project")
"submit" (list parse-submit "Submit a project for execution")))
(call-with-input-file config-file read))
(cmd-setup config)
(report-status "successfully imported config")))
(define (print-usage)
(displayln "crossfire-client <command> [<arg> ...]")
(printf "~a <command> [<arg> ...]\n" *program*)
(displayln " The CLI for crossfire")
(displayln " where <command> is one of")
(for ([(k v) (in-hash *commands*)])
(printf " ~a : ~a\n" k (second v))))
(for ([k (in-list (sort (hash-keys *commands*) string<?))])
(printf " ~a : ~a\n" k (second (hash-ref *commands* k)))))
(match (current-command-line-arguments)
[(vector (? (curry hash-has-key? *commands*) cmd) argv ...)

View File

@ -18,13 +18,15 @@
(require (only-in data/integer-set integer-set-contents)
racket/match racket/runtime-path racket/vector
scribble/text (rename-in scribble/text/output [output scribble-output]))
scribble/text (rename-in scribble/text/output [output scribble-output])
"manifest.rkt" "pattern.rkt")
(provide pattern-codegen)
(provide generate-support-code generate-manifest)
;; templating infrastructure
(define (eval-template file vars [port (current-output-port)])
(define (eval-template template-path vars)
(define out (open-output-string))
(define cs (current-namespace))
(define output-exp
(parameterize ([current-namespace (make-base-namespace)])
@ -32,17 +34,27 @@
(namespace-attach-module cs mod)
(namespace-require mod))
(hash-for-each vars namespace-set-variable-value!)
(eval `(include/text ,file))))
(scribble-output output-exp port))
(eval `(include/text (file ,(path->string template-path))))))
(scribble-output output-exp out)
(get-output-string out))
;; there used to be a racket implementation of the input generator but it majorly violated the
;; design recipe and got obsoleted by the template C version (which is intended to be more portable
;; and faster)
;; ok gamer move time
(define-runtime-path codegen-template "codegen.rktc")
(define (pattern-codegen pattern mode)
(define-runtime-path codegen-template "templates/codegen.rktc")
(define-runtime-path makefile-template "templates/Makefile.rkt")
(define-runtime-path manifest-template "templates/manifest.rktrktd")
(define (generate-support-code mf)
(define vars (hash 'pattern (vector-map integer-set-contents (manifest-pattern mf))
'mode (first (manifest-data-ref mf 'mode))))
(hash
"crossfire.c" (eval-template codegen-template vars)
"Makefile" (eval-template makefile-template vars)))
(define (generate-manifest mode)
(eval-template
`(file ,(path->string codegen-template))
(hash 'pattern (vector-map integer-set-contents pattern)
'mode mode)))
manifest-template
(hash 'mode mode)))

View File

@ -47,7 +47,8 @@
'?d (string->integer-set "0123456789")
'?s (string->integer-set " !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~")
'?a (range->integer-set 32 126)
'?b (range->integer-set 0 255)))
'?b (range->integer-set 0 255)
'?? (char->integer-set #\?)))
(provide builtin-isets)
;; manifest contracts and definition
@ -62,7 +63,7 @@
(list/c 'smp boolean?)
(list/c 'mode (or/c 'stdio 'callback))
(cons/c 'command (listof string?))
(cons/c 'iset (listof isub/c))
(cons/c 'iset (cons/c symbol? (listof isub/c)))
(cons/c 'pattern (listof isub/c)))))
(struct manifest [data pattern psize] #:transparent)

View File

@ -16,7 +16,7 @@
;; You should have received a copy of the GNU Affero General Public License
;; along with this program. If not, see <https://www.gnu.org/licenses/>.
(require ffi/vector ffi/unsafe racket/bool racket/random
(require ffi/vector ffi/unsafe racket/bool racket/match racket/random
"static-support.rkt"
(for-syntax racket/base racket/string)
(only-in racket/contract provide/contract)
@ -137,3 +137,22 @@
(define/ffi crypto-wipe
(_fun (buf : _u8vector) (sz : _size = (u8vector-length buf)) -> _void))
(provide crypto-wipe)
;; helpers
(define *read-size* 8192)
(define (port->blake2b-hash port)
(define ctx (crypto-blake2b-init))
(let loop ()
(match (read-bytes *read-size* port)
[(? eof-object?) (void)]
[data (crypto-blake2b-update ctx data)
(loop)]))
(crypto-blake2b-final ctx))
(provide port->blake2b-hash)
(define (file->blake2b-hash path)
(call-with-input-file path port->blake2b-hash))
(provide file->blake2b-hash)

View File

@ -21,8 +21,8 @@
racket/function racket/list racket/logging racket/match racket/path racket/random
racket/runtime-path racket/set racket/string racket/unit srfi/19
north/base north/adapter/base north/adapter/sqlite
"comms.rkt" "info.rkt" "logging.rkt" "manifest.rkt" "not-crypto.rkt" "pattern.rkt" "protocol.rkt"
;; port-fsync
"comms.rkt" "info.rkt" "logging.rkt" "manifest.rkt" "not-crypto.rkt" "pattern.rkt"
"protocol.rkt"
(submod "static-support.rkt" misc-calls))
;; logging
@ -98,6 +98,7 @@
(define-stmt q-get-node-resources "select resource from node_resource where nodeid=?")
(define-stmt q-edit-node "update node set name=? where id=?")
(define-stmt q-delete-node "delete from node where id=?")
(define-stmt q-delete-node-type "delete from node where type=?")
(define-stmt q-get-node-type "select type from node where id=?")
(define-stmt q-get-node-info "select name, arch, type, secret from node where id=?")
@ -162,16 +163,7 @@
;; computes a hash of the file identifying its current contents for agents
;; (in case we reuse taskids)
(define (server-hash-file taskid)
(define path (get-project-file-path taskid))
(define *read-size* 8192)
(call-with-input-file path (lambda (in)
(define ctx (crypto-blake2b-init))
(let loop ()
(match (read-bytes *read-size* in)
[(? eof-object?) (void)]
[data (crypto-blake2b-update ctx data)
(loop)]))
(crypto-blake2b-final ctx))))
(file->blake2b-hash (get-project-file-path taskid)))
(define (server-get-file taskid)
(define path (get-project-file-path taskid))
@ -188,7 +180,7 @@
(query (current-db) q-get-all-resources type-str)))
(for/list ([(id name arch) (in-query (current-db) q-get-nodes type-str)])
(define online? (and (current-comms) (comms-channel-available? (current-comms) id)))
(node-info id name arch type (hash-ref resources id) online?)))
(node-info id name arch type (hash-ref resources id '()) online?)))
(define (make-node name arch type resources)
(call-with-transaction (current-db) (lambda ()
@ -859,6 +851,11 @@
(write-bytes (crypto-sign-make-key) out)
(port-fsync out))))
;; create client if it doesn't exist yet
(when (empty? (get-nodes 'client))
(log-server-info "creating new client")
(make-node "client0" "x-client" 'client '()))
(define seckey (file->bytes *server-seckey-path*))
(define pubkey (crypto-sign-public-key seckey))
(define listen-addr
@ -892,6 +889,13 @@
(lambda (out) (write (list agent-node (current-server-public-node)) out)))
(log-server-info "created dev agent ~a" name)
(exit)]
[(vector "dev-export-client")
(define client-id (node-info-id (first (get-nodes 'client))))
(define client-node (load-comms-node client-id #t))
(define data (list client-node (current-server-public-node)))
(with-output-to-file (build-path *state-root* "client0.rktd")
(lambda () (write data)))
(exit)]
[(vector "dev-new-project" tgz)
(define id (make-task '((name "test project") (arch "any")
(command "./test.sh")
@ -919,6 +923,11 @@
(agent-handler-new-agent (node-info-id agent) (node-info-arch agent)
(node-info-resources agent)))
;; restore clients
(for ([client (in-list (get-nodes 'client))])
(log-server-info "restoring client ~a" (node-info-id client))
(restore-comms-node (node-info-id client)))
;; restore active tasks
(for ([(id name manifest-in complete?) (in-query (current-db) q-get-tasks)]
#:when (zero? complete?))

View File

@ -0,0 +1,64 @@
@;; crossfire: distributed brute force infrastructure
@;;
@;; Copyright (C) 2020 haskal
@;;
@;; This program is free software: you can redistribute it and/or modify
@;; it under the terms of the GNU Affero General Public License as published by
@;; the Free Software Foundation, either version 3 of the License, or
@;; (at your option) any later version.
@;;
@;; This program is distributed in the hope that it will be useful,
@;; but WITHOUT ANY WARRANTY; without even the implied warranty of
@;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
@;; GNU Affero General Public License for more details.
@;;
@;; You should have received a copy of the GNU Affero General Public License
@;; along with this program. If not, see <https://www.gnu.org/licenses/>.
@;; vim: ft=racket
@;;
.PHONY: all clean
CC=gcc
CFLAGS=-std=c11 -O3
LDFLAGS=-static
@(match mode
['stdio @list{
# TODO : your rules here
all: ...
clean: ...
# crossfire rules
CF_GEN=crossfire-generator
all: $(CF_GEN)
$(CF_GEN): crossfire.o
@"\t"$(CC) -o $@"@" $(CFLAGS) $(LDFLAGS) $<
.PHONY: crossfire-clean
clean: crossfire-clean
crossfire-clean:
@"\t"$(RM) crossfire.o $(CF_GEN)
}]
['callback @list{
# TODO: include crossfire.o in your application
all: ...
clean: ...
# crossfire rules
.PHONY: crossfire-clean
clean: crossfire-clean
crossfire-clean:
@"\t"$(RM) crossfire.o
}])
crossfire.o: crossfire.c
@"\t"$(CC) -c -o $@"@" $(CFLAGS) $<

View File

@ -98,6 +98,8 @@ int crossfire_main(int argc, char* argv[], callback cb) {
@(define fmt
(string-join (for/list ([i (in-range (vector-length pattern))]) c-type-fmt) " "))
void cf_report_success( @arg-vs );
l_inner:
@(define (end-conditionals)

View File

@ -0,0 +1,59 @@
@;; crossfire: distributed brute force infrastructure
@;;
@;; Copyright (C) 2020 haskal
@;;
@;; This program is free software: you can redistribute it and/or modify
@;; it under the terms of the GNU Affero General Public License as published by
@;; the Free Software Foundation, either version 3 of the License, or
@;; (at your option) any later version.
@;;
@;; This program is distributed in the hope that it will be useful,
@;; but WITHOUT ANY WARRANTY; without even the implied warranty of
@;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
@;; GNU Affero General Public License for more details.
@;;
@;; You should have received a copy of the GNU Affero General Public License
@;; along with this program. If not, see <https://www.gnu.org/licenses/>.
@;; vim: ft=racket
@;;
(;; name of the project
(name "my cool project")
;; architectures it runs on (eg "x86_64-pc-linux-gnu") or "any"
(arch "x86_64-pc-linux-gnu")
;; resources needed
(resources "cpu")
;; #t to let crossfire perform multithreading for you
;; #f if your application performs its own multithreading or should not be multithreaded
(smp #f)
;; mode
;; - stdio: pass input to the given command (below) via stdin, space separated hex
;; - callback: call a C callback function in your program with inputs
;; in callback mode, you must integrate the provided input generator with your program
(mode @mode)
;; command to execute (arguments will be appended in callback mode)
(command "/usr/bin/env" "python3" "myscript.py")
;; custom pattern definitions
;; examples:
;; (iset ?z "0123456789" ?l)
;; (iset ?z (#x30 #x39) "abcdefghij" "klmn" "opqrstuv")
;; (iset ?y ?d)
;; builtin patterns
;; ?l - lowercase
;; ?u - uppercase
;; ?d - numbers
;; ?s - symbols
;; ?a - ascii
;; ?b - any byte
;; ?? - escape for literal question mark
;; brute force pattern
;; examples (each element is concatenated)
;; (pattern "meow?d")
;; (pattern "meow" ?d)
;; (pattern "me" "ow" (#x30 #x39))
;; (pattern "meow" (48 57))
;; up to uint64_t range numbers also supported
;; (pattern (12345 67890) (10 20) "x")
(pattern "meow?d?d"))