rework posts compiling

This commit is contained in:
xenia 2020-05-09 18:52:25 -04:00
parent 247356cd92
commit f1d0e122c2
2 changed files with 43 additions and 11 deletions

View File

@ -14,6 +14,17 @@
/taskq.sqlite3: sqlite database for naive persistent task queue
stores the activitypub stuff to in a crash-resistant way
## Building the static content
1. checks posts that got posted but that have no corresponding source. deletes and sends out delete
activities. if any posts got deleted it could touch a fake flag file that causes the next step to
rebuild all indices even if there are no new or updated posts
2. main pass that builds all the main activitypub objects and html pages and css. queues create or
update activities if necessary
3. second pass based on tag info extracted during previous step, regenerates tag pages and deletes
ones that don't exist anymore. tag pages don't have corresponding AP objects so no activities are
required here
## User object
endpoint: https://my.instance.tld/
@ -54,6 +65,8 @@ be filled in with dummy data indicating the resource could not be fetched
to recompile posts from markdown multiple times because that's super unnecessary. it also contains
front matter with the post metadata and parsed hashtags
/cache/tagindex.rktb stores an index of all hashtags and which posts contain them
## Task queue
stores queued outgoing requests

View File

@ -6,6 +6,7 @@
net/url-string
markdown
markdown/toc
racket/fasl
"compile.rkt"
"prefs.rkt")
@ -62,7 +63,6 @@
(reverse (cons (substring str last-pos) items))]))
(first (process+ xexpr)))
(define compile-index-json
(rule '("db/actorkey.pub" "src/instance.rktd" "src/bio.md") "public/index.json"
(lambda (in out)
@ -147,17 +147,36 @@
(define post-htmls (map (lambda (b)
(process-hashtags! b instance-url hashtags))
(parse-markdown post-content)))
(write `((is-meow #t)
(hashtags ,(set->list hashtags))
(toc ,(toc post-htmls))
(content ,post-htmls)) out))))
(write-bytes
(s-exp->fasl
(hash 'is-meow #t
'hashtags (set->list hashtags)
'toc (toc post-htmls)
'content post-htmls))
out))))
(for/fold ([posts (set)]) ([post-name (in-list (directory-list "src/posts"))]
#:when (regexp-match? #px"\\.md$" post-name))
(define post-src (build-path "src/posts" post-name))
(define cache-dst (build-path "cache/posts"
(path-replace-extension (last (explode-path post-src)) ".rktd")))
(set-add posts (make-post-cache-rule post-src cache-dst))))
(define-values [posts-set cache-set]
(for/fold ([posts (set)] [caches (set)]) ([post-name (in-list (directory-list "src/posts"))]
#:when (regexp-match? #px"\\.md$" post-name))
(define post-src (build-path "src/posts" post-name))
(define cache-dst (build-path "cache/posts"
(path-replace-extension post-name ".rktb")))
(define json-dst (build-path "public/posts/"))
(values
(set-add posts (make-post-cache-rule post-src cache-dst))
(set-add caches cache-dst))))
;; use the cache set to make a tags rule
(set-add
posts-set
(rule (set->list cache-set) "cache/tagindex.rktb"
(lambda (in out)
(define index (make-hash))
(for ([(fname port) (in-hash in)])
(define meta (fasl->s-exp (port->bytes port)))
(for ([tag (in-list (hash-ref meta 'hashtags))])
(hash-update! index tag (lambda (s) (set-add s fname)) set)))
(write-bytes (s-exp->fasl (for/hash ([(k v) (in-hash index)])
(values k (set->list v)))) out)))))
(define compiler-rules