diff --git a/crossfire/agent.rkt b/crossfire/agent.rkt
index 426babb..f6d5b59 100644
--- a/crossfire/agent.rkt
+++ b/crossfire/agent.rkt
@@ -21,7 +21,6 @@
racket/path racket/port racket/string racket/unit srfi/19
"comms.rkt" "info.rkt" "logging.rkt" "not-crypto.rkt" "manifest.rkt" "protocol.rkt"
"pattern.rkt" "static-support.rkt"
- ;; port-fsync
(submod "static-support.rkt" misc-calls))
(provide agent-main)
diff --git a/crossfire/client.rkt b/crossfire/client.rkt
index ca94c62..3a2bf5b 100644
--- a/crossfire/client.rkt
+++ b/crossfire/client.rkt
@@ -16,63 +16,223 @@
;; You should have received a copy of the GNU Affero General Public License
;; along with this program. If not, see .
-(require racket/function racket/list racket/match racket/vector
- "comms.rkt" "info.rkt")
+(require file/tar racket/bool racket/file racket/function racket/list racket/match racket/path
+ racket/string racket/unit racket/vector syntax/parse/define
+ (only-in file/sha1 bytes->hex-string)
+ (for-syntax racket/base racket/syntax)
+ "codegen.rkt" "comms.rkt" "info.rkt" "manifest.rkt" "not-crypto.rkt" "protocol.rkt")
(define *program* (format "~a-client" (#%info-lookup 'collection)))
+;; XXX : slightly unix-oriented
+(define *config-root*
+ (build-path
+ (or (getenv "XDG_CONFIG_HOME")
+ (build-path (find-system-path 'home-dir) ".config"))
+ *program*))
+(define *client-key-file* (build-path *config-root* "config.rktd"))
-(define (cmd-new project-name)
- (printf "creating new project ~a\n" project-name))
+(define *cf-private-cache* (build-path ".crossfire" "cache.rktd"))
+(define *cf-tmp-targz* (build-path ".crossfire" "project.tgz"))
-(define (cmd-submit)
- (displayln "submitting..."))
+(define (cmd-new project-name mode)
+ (define project-root (build-path project-name))
+ (when (or (file-exists? project-root) (directory-exists? project-root)
+ (link-exists? project-root))
+ (error "file or directory already exists named" project-name))
+ (define proj-dir (build-path project-name))
+ (define priv-cache (build-path proj-dir *cf-private-cache*))
+
+ (make-directory* proj-dir)
+ (make-parent-directory* priv-cache)
+ (call-with-output-file priv-cache (lambda (out) (write (hash) out)))
+
+ (define mf-gen (generate-manifest mode))
+ (call-with-output-file
+ (build-path proj-dir "manifest.rktd")
+ (lambda (out) (write-string mf-gen out)))
+
+ (void))
+
+
+(define (cmd-update project-dir)
+ (define priv-cache (build-path project-dir *cf-private-cache*))
+ (define mf (parse-manifest (call-with-input-file (build-path project-dir "manifest.rktd") read)))
+ (define new-files (generate-support-code mf))
+ (define old-hashes (call-with-input-file priv-cache read))
+
+ (define messages '())
+
+ (define new-hashes
+ (for/hash ([(name contents) (in-hash new-files)])
+ (define v-hash (port->blake2b-hash (open-input-string contents)))
+ (define old-hash (hash-ref old-hashes name #f))
+ (cond
+ [(not (file-exists? name))
+ (call-with-output-file (build-path project-dir name)
+ (lambda (out) (write-string contents out)))
+ (set! messages (cons (format "created file ~a" name) messages))]
+ [(and old-hash (not (bytes=? old-hash v-hash)))
+ (define new-name (string-append name "." (substring (bytes->hex-string old-hash) 0 32)))
+ (rename-file-or-directory (build-path project-dir name) (build-path project-dir new-name))
+ (call-with-output-file (build-path project-dir name)
+ (lambda (out) (write-string contents out)))
+ (define msg
+ (format
+ "because of manifest updates, ~a has been regenerated. the old version was moved to ~a"
+ name new-name))
+ (set! messages (cons msg messages))])
+ (values name v-hash)))
+
+ (call-with-output-file priv-cache (lambda (out) (write new-hashes out)) #:exists 'replace)
+ messages)
+
+
+(define (cmd-check project-dir)
+ (define messages '())
+
+ (define mf (parse-manifest (call-with-input-file (build-path project-dir "manifest.rktd") read)))
+ (define mode (first (manifest-data-ref mf 'mode)))
+
+ (when (and (symbol=? mode 'stdio)
+ (not (file-exists? (build-path project-dir "crossfire-generator"))))
+ (set! messages (cons "missing crossfire-generator, make sure to compile it" messages)))
+
+ (unless (file-exists? *client-key-file*)
+ (set! messages (cons (format "missing configuration to connect to server. use ~a setup"
+ *program*)
+ messages)))
+
+ messages)
+
+
+(define (cmd-submit project-dir)
+ (define mf (parse-manifest (call-with-input-file (build-path project-dir "manifest.rktd") read)))
+ (match-define (list client-node server-node) (call-with-input-file *client-key-file* read))
+
+ ;; create targz
+ (define tmp-targz (build-path project-dir *cf-tmp-targz*))
+ (parameterize ([current-directory project-dir])
+ (define out (current-output-port))
+ (tar-gzip tmp-targz "."
+ #:exists-ok? #t
+ #:path-filter (lambda (p)
+ (match (explode-path p)
+ [(list 'same (== (build-path ".crossfire")) rst ...) #f]
+ [(list 'same fst rst ...) #t]
+ [(list 'same) #t]
+ [_ (error "not shonks...")]))))
+
+ ;; connect to server
+ (define server-wrapper@ (make-rpc-wrapper-unit server^))
+ (define-values/invoke-unit server-wrapper@ (import) (export server^))
+
+ (current-to-node server-node)
+ (current-comms (make-comms client-node))
+ (current-tm (make-transaction-manager client-node (current-comms)))
+ (comms-set-node-info (current-comms) server-node)
+ (comms-connect (current-comms) (node-id server-node))
+
+ (displayln "yeet")
+ (new-project (serialize-manifest mf) (file->bytes tmp-targz))
+
+ (tm-shutdown (current-tm))
+ (comms-shutdown (current-comms))
+ (void))
+
+
+(define (cmd-setup config)
+ (make-parent-directory* *client-key-file*)
+ (with-output-to-file *client-key-file* (lambda () (write config)) #:exists 'replace))
-(define (cmd-setup config-file)
- (displayln "meow"))
(module+ main
(require racket/cmdline)
- (define (report-fatal-error cmd msg)
- (printf "~a ~a: ~a\n" *program* cmd msg)
+ (define current-subcommand (make-parameter #f))
+
+ (define (report-status msg . args)
+ (printf "~a ~a: ~a\n" *program* (current-subcommand) (apply format msg args)))
+
+ (define (report-fatal-error msg . args)
+ (apply report-status msg args)
(exit 1))
- (define (parse-new argv)
- (command-line
- #:program (format "~a new" *program*)
- #:argv argv
+ (define (interactive-update)
+ (match (cmd-update (current-directory))
+ ['() (report-status "generated files are up-to-date")]
+ [(list msgs ...)
+ (for ([msg (in-list msgs)])
+ (report-status "~a" msg))]))
+
+ (define (interactive-check)
+ (match (cmd-check (current-directory))
+ ['() (report-status "everything looks good to me!")]
+ [(list warnings ...)
+ (for ([warning (in-list warnings)])
+ (report-status "error: ~a" warning))
+ (report-fatal-error "check was not successful due to above messages")]))
+
+ (define-simple-macro (subcommand (name:id description:str) body ...)
+ #:with name-str (datum->syntax #'name (symbol->string (syntax-e #'name)))
+ (list name-str
+ (lambda (argv)
+ (parameterize ([current-subcommand name-str])
+ (with-handlers ([exn:fail?
+ (lambda (ex)
+ (report-fatal-error "~a" (exn-message ex)))])
+ (command-line
+ #:program (format "~a ~a" *program* name-str)
+ #:argv argv
+ #:usage-help description
+ body ...))))
+ description))
+
+ (define-simple-macro (define-commands name:id cmds ...)
+ (define name (make-hash (list cmds ...))))
+
+ (define flag-mode (make-parameter "stdio"))
+ (define-commands *commands*
+ (subcommand (new "Create a new crossfire project")
+ #:once-each [("-m" "--mode") mode "Project mode (stdio [default] or callback)"
+ (flag-mode mode)]
#:args (project-name)
- (cmd-new project-name)))
+ (match (flag-mode)
+ [(or "stdio" "callback") (void)]
+ [x (report-fatal-error "invalid mode ~a, need stdio or callback" x)])
+ (cmd-new project-name (string->symbol (flag-mode)))
+ (report-status "created new project ~a" project-name))
- (define (parse-submit argv)
- (command-line
- #:program (format "~a submit" *program*)
- #:argv argv
+ (subcommand (update "Update generated files in a project")
#:args ()
- (cmd-submit)))
+ (interactive-update))
- (define (parse-setup argv)
- (command-line
- #:program (format "~a setup" *program*)
- #:argv argv
+ (subcommand (check "Check over a project for issues")
+ #:args ()
+ (interactive-check))
+
+ (subcommand (submit "Submit a project for execution")
+ #:args ()
+ ;; trigger a check before we submit
+ (interactive-check)
+ ;; do submit
+ (report-status "submitting project...!")
+ (cmd-submit (current-directory)))
+
+ (subcommand (setup "Set up access to a crossfire server")
#:args (config-file)
(unless (file-exists? config-file)
- (report-fatal-error "setup" "provided config file does not exist"))
+ (report-fatal-error "provided config file does not exist"))
(define config
- (with-handlers ([exn:fail?
- (lambda (ex) (report-fatal-error "setup" (exn-message ex)))])
- (call-with-input-file config-file read)))
- (cmd-setup config)))
-
- (define *commands* (hash "setup" (list parse-setup "Set up server config")
- "new" (list parse-new "Create a new project")
- "submit" (list parse-submit "Submit a project for execution")))
+ (call-with-input-file config-file read))
+ (cmd-setup config)
+ (report-status "successfully imported config")))
(define (print-usage)
- (displayln "crossfire-client [ ...]")
+ (printf "~a [ ...]\n" *program*)
+ (displayln " The CLI for crossfire")
(displayln " where is one of")
- (for ([(k v) (in-hash *commands*)])
- (printf " ~a : ~a\n" k (second v))))
+ (for ([k (in-list (sort (hash-keys *commands*) string))])
+ (printf " ~a : ~a\n" k (second (hash-ref *commands* k)))))
(match (current-command-line-arguments)
[(vector (? (curry hash-has-key? *commands*) cmd) argv ...)
diff --git a/crossfire/codegen.rkt b/crossfire/codegen.rkt
index 4aa6e70..bc9a317 100644
--- a/crossfire/codegen.rkt
+++ b/crossfire/codegen.rkt
@@ -18,13 +18,15 @@
(require (only-in data/integer-set integer-set-contents)
racket/match racket/runtime-path racket/vector
- scribble/text (rename-in scribble/text/output [output scribble-output]))
+ scribble/text (rename-in scribble/text/output [output scribble-output])
+ "manifest.rkt" "pattern.rkt")
-(provide pattern-codegen)
+(provide generate-support-code generate-manifest)
;; templating infrastructure
-(define (eval-template file vars [port (current-output-port)])
+(define (eval-template template-path vars)
+ (define out (open-output-string))
(define cs (current-namespace))
(define output-exp
(parameterize ([current-namespace (make-base-namespace)])
@@ -32,17 +34,27 @@
(namespace-attach-module cs mod)
(namespace-require mod))
(hash-for-each vars namespace-set-variable-value!)
- (eval `(include/text ,file))))
- (scribble-output output-exp port))
+ (eval `(include/text (file ,(path->string template-path))))))
+ (scribble-output output-exp out)
+ (get-output-string out))
;; there used to be a racket implementation of the input generator but it majorly violated the
;; design recipe and got obsoleted by the template C version (which is intended to be more portable
;; and faster)
;; ok gamer move time
-(define-runtime-path codegen-template "codegen.rktc")
-(define (pattern-codegen pattern mode)
+(define-runtime-path codegen-template "templates/codegen.rktc")
+(define-runtime-path makefile-template "templates/Makefile.rkt")
+(define-runtime-path manifest-template "templates/manifest.rktrktd")
+
+(define (generate-support-code mf)
+ (define vars (hash 'pattern (vector-map integer-set-contents (manifest-pattern mf))
+ 'mode (first (manifest-data-ref mf 'mode))))
+ (hash
+ "crossfire.c" (eval-template codegen-template vars)
+ "Makefile" (eval-template makefile-template vars)))
+
+(define (generate-manifest mode)
(eval-template
- `(file ,(path->string codegen-template))
- (hash 'pattern (vector-map integer-set-contents pattern)
- 'mode mode)))
+ manifest-template
+ (hash 'mode mode)))
diff --git a/crossfire/manifest.rkt b/crossfire/manifest.rkt
index 250ac0b..72074e8 100644
--- a/crossfire/manifest.rkt
+++ b/crossfire/manifest.rkt
@@ -47,7 +47,8 @@
'?d (string->integer-set "0123456789")
'?s (string->integer-set " !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~")
'?a (range->integer-set 32 126)
- '?b (range->integer-set 0 255)))
+ '?b (range->integer-set 0 255)
+ '?? (char->integer-set #\?)))
(provide builtin-isets)
;; manifest contracts and definition
@@ -62,7 +63,7 @@
(list/c 'smp boolean?)
(list/c 'mode (or/c 'stdio 'callback))
(cons/c 'command (listof string?))
- (cons/c 'iset (listof isub/c))
+ (cons/c 'iset (cons/c symbol? (listof isub/c)))
(cons/c 'pattern (listof isub/c)))))
(struct manifest [data pattern psize] #:transparent)
diff --git a/crossfire/not-crypto.rkt b/crossfire/not-crypto.rkt
index ad1d86a..11933d3 100644
--- a/crossfire/not-crypto.rkt
+++ b/crossfire/not-crypto.rkt
@@ -16,7 +16,7 @@
;; You should have received a copy of the GNU Affero General Public License
;; along with this program. If not, see .
-(require ffi/vector ffi/unsafe racket/bool racket/random
+(require ffi/vector ffi/unsafe racket/bool racket/match racket/random
"static-support.rkt"
(for-syntax racket/base racket/string)
(only-in racket/contract provide/contract)
@@ -137,3 +137,22 @@
(define/ffi crypto-wipe
(_fun (buf : _u8vector) (sz : _size = (u8vector-length buf)) -> _void))
(provide crypto-wipe)
+
+;; helpers
+
+(define *read-size* 8192)
+
+(define (port->blake2b-hash port)
+ (define ctx (crypto-blake2b-init))
+ (let loop ()
+ (match (read-bytes *read-size* port)
+ [(? eof-object?) (void)]
+ [data (crypto-blake2b-update ctx data)
+ (loop)]))
+ (crypto-blake2b-final ctx))
+(provide port->blake2b-hash)
+
+(define (file->blake2b-hash path)
+ (call-with-input-file path port->blake2b-hash))
+
+(provide file->blake2b-hash)
diff --git a/crossfire/server.rkt b/crossfire/server.rkt
index fd21097..334642f 100644
--- a/crossfire/server.rkt
+++ b/crossfire/server.rkt
@@ -21,8 +21,8 @@
racket/function racket/list racket/logging racket/match racket/path racket/random
racket/runtime-path racket/set racket/string racket/unit srfi/19
north/base north/adapter/base north/adapter/sqlite
- "comms.rkt" "info.rkt" "logging.rkt" "manifest.rkt" "not-crypto.rkt" "pattern.rkt" "protocol.rkt"
- ;; port-fsync
+ "comms.rkt" "info.rkt" "logging.rkt" "manifest.rkt" "not-crypto.rkt" "pattern.rkt"
+ "protocol.rkt"
(submod "static-support.rkt" misc-calls))
;; logging
@@ -98,6 +98,7 @@
(define-stmt q-get-node-resources "select resource from node_resource where nodeid=?")
(define-stmt q-edit-node "update node set name=? where id=?")
(define-stmt q-delete-node "delete from node where id=?")
+(define-stmt q-delete-node-type "delete from node where type=?")
(define-stmt q-get-node-type "select type from node where id=?")
(define-stmt q-get-node-info "select name, arch, type, secret from node where id=?")
@@ -162,16 +163,7 @@
;; computes a hash of the file identifying its current contents for agents
;; (in case we reuse taskids)
(define (server-hash-file taskid)
- (define path (get-project-file-path taskid))
- (define *read-size* 8192)
- (call-with-input-file path (lambda (in)
- (define ctx (crypto-blake2b-init))
- (let loop ()
- (match (read-bytes *read-size* in)
- [(? eof-object?) (void)]
- [data (crypto-blake2b-update ctx data)
- (loop)]))
- (crypto-blake2b-final ctx))))
+ (file->blake2b-hash (get-project-file-path taskid)))
(define (server-get-file taskid)
(define path (get-project-file-path taskid))
@@ -188,7 +180,7 @@
(query (current-db) q-get-all-resources type-str)))
(for/list ([(id name arch) (in-query (current-db) q-get-nodes type-str)])
(define online? (and (current-comms) (comms-channel-available? (current-comms) id)))
- (node-info id name arch type (hash-ref resources id) online?)))
+ (node-info id name arch type (hash-ref resources id '()) online?)))
(define (make-node name arch type resources)
(call-with-transaction (current-db) (lambda ()
@@ -859,6 +851,11 @@
(write-bytes (crypto-sign-make-key) out)
(port-fsync out))))
+ ;; create client if it doesn't exist yet
+ (when (empty? (get-nodes 'client))
+ (log-server-info "creating new client")
+ (make-node "client0" "x-client" 'client '()))
+
(define seckey (file->bytes *server-seckey-path*))
(define pubkey (crypto-sign-public-key seckey))
(define listen-addr
@@ -892,6 +889,13 @@
(lambda (out) (write (list agent-node (current-server-public-node)) out)))
(log-server-info "created dev agent ~a" name)
(exit)]
+ [(vector "dev-export-client")
+ (define client-id (node-info-id (first (get-nodes 'client))))
+ (define client-node (load-comms-node client-id #t))
+ (define data (list client-node (current-server-public-node)))
+ (with-output-to-file (build-path *state-root* "client0.rktd")
+ (lambda () (write data)))
+ (exit)]
[(vector "dev-new-project" tgz)
(define id (make-task '((name "test project") (arch "any")
(command "./test.sh")
@@ -919,6 +923,11 @@
(agent-handler-new-agent (node-info-id agent) (node-info-arch agent)
(node-info-resources agent)))
+ ;; restore clients
+ (for ([client (in-list (get-nodes 'client))])
+ (log-server-info "restoring client ~a" (node-info-id client))
+ (restore-comms-node (node-info-id client)))
+
;; restore active tasks
(for ([(id name manifest-in complete?) (in-query (current-db) q-get-tasks)]
#:when (zero? complete?))
diff --git a/crossfire/templates/Makefile.rkt b/crossfire/templates/Makefile.rkt
new file mode 100644
index 0000000..93e09cc
--- /dev/null
+++ b/crossfire/templates/Makefile.rkt
@@ -0,0 +1,64 @@
+@;; crossfire: distributed brute force infrastructure
+@;;
+@;; Copyright (C) 2020 haskal
+@;;
+@;; This program is free software: you can redistribute it and/or modify
+@;; it under the terms of the GNU Affero General Public License as published by
+@;; the Free Software Foundation, either version 3 of the License, or
+@;; (at your option) any later version.
+@;;
+@;; This program is distributed in the hope that it will be useful,
+@;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+@;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+@;; GNU Affero General Public License for more details.
+@;;
+@;; You should have received a copy of the GNU Affero General Public License
+@;; along with this program. If not, see .
+@;; vim: ft=racket
+@;;
+.PHONY: all clean
+
+CC=gcc
+CFLAGS=-std=c11 -O3
+LDFLAGS=-static
+
+@(match mode
+ ['stdio @list{
+# TODO : your rules here
+
+all: ...
+
+clean: ...
+
+
+# crossfire rules
+
+CF_GEN=crossfire-generator
+
+all: $(CF_GEN)
+$(CF_GEN): crossfire.o
+@"\t"$(CC) -o $@"@" $(CFLAGS) $(LDFLAGS) $<
+
+.PHONY: crossfire-clean
+clean: crossfire-clean
+crossfire-clean:
+@"\t"$(RM) crossfire.o $(CF_GEN)
+ }]
+ ['callback @list{
+# TODO: include crossfire.o in your application
+
+all: ...
+
+clean: ...
+
+
+# crossfire rules
+
+.PHONY: crossfire-clean
+clean: crossfire-clean
+crossfire-clean:
+@"\t"$(RM) crossfire.o
+ }])
+
+crossfire.o: crossfire.c
+@"\t"$(CC) -c -o $@"@" $(CFLAGS) $<
diff --git a/crossfire/codegen.rktc b/crossfire/templates/codegen.rktc
similarity index 99%
rename from crossfire/codegen.rktc
rename to crossfire/templates/codegen.rktc
index cfe35f5..e22278f 100644
--- a/crossfire/codegen.rktc
+++ b/crossfire/templates/codegen.rktc
@@ -98,6 +98,8 @@ int crossfire_main(int argc, char* argv[], callback cb) {
@(define fmt
(string-join (for/list ([i (in-range (vector-length pattern))]) c-type-fmt) " "))
+ void cf_report_success( @arg-vs );
+
l_inner:
@(define (end-conditionals)
diff --git a/crossfire/templates/manifest.rktrktd b/crossfire/templates/manifest.rktrktd
new file mode 100644
index 0000000..9aa8497
--- /dev/null
+++ b/crossfire/templates/manifest.rktrktd
@@ -0,0 +1,59 @@
+@;; crossfire: distributed brute force infrastructure
+@;;
+@;; Copyright (C) 2020 haskal
+@;;
+@;; This program is free software: you can redistribute it and/or modify
+@;; it under the terms of the GNU Affero General Public License as published by
+@;; the Free Software Foundation, either version 3 of the License, or
+@;; (at your option) any later version.
+@;;
+@;; This program is distributed in the hope that it will be useful,
+@;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+@;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+@;; GNU Affero General Public License for more details.
+@;;
+@;; You should have received a copy of the GNU Affero General Public License
+@;; along with this program. If not, see .
+@;; vim: ft=racket
+@;;
+(;; name of the project
+ (name "my cool project")
+ ;; architectures it runs on (eg "x86_64-pc-linux-gnu") or "any"
+ (arch "x86_64-pc-linux-gnu")
+ ;; resources needed
+ (resources "cpu")
+ ;; #t to let crossfire perform multithreading for you
+ ;; #f if your application performs its own multithreading or should not be multithreaded
+ (smp #f)
+ ;; mode
+ ;; - stdio: pass input to the given command (below) via stdin, space separated hex
+ ;; - callback: call a C callback function in your program with inputs
+ ;; in callback mode, you must integrate the provided input generator with your program
+ (mode @mode)
+ ;; command to execute (arguments will be appended in callback mode)
+ (command "/usr/bin/env" "python3" "myscript.py")
+
+ ;; custom pattern definitions
+ ;; examples:
+ ;; (iset ?z "0123456789" ?l)
+ ;; (iset ?z (#x30 #x39) "abcdefghij" "klmn" "opqrstuv")
+ ;; (iset ?y ?d)
+
+ ;; builtin patterns
+ ;; ?l - lowercase
+ ;; ?u - uppercase
+ ;; ?d - numbers
+ ;; ?s - symbols
+ ;; ?a - ascii
+ ;; ?b - any byte
+ ;; ?? - escape for literal question mark
+
+ ;; brute force pattern
+ ;; examples (each element is concatenated)
+ ;; (pattern "meow?d")
+ ;; (pattern "meow" ?d)
+ ;; (pattern "me" "ow" (#x30 #x39))
+ ;; (pattern "meow" (48 57))
+ ;; up to uint64_t range numbers also supported
+ ;; (pattern (12345 67890) (10 20) "x")
+ (pattern "meow?d?d"))