rework posts compiling

NOTES.md

@@ -14,6 +14,17 @@
 /taskq.sqlite3: sqlite database for naive persistent task queue
     stores the activitypub stuff to in a crash-resistant way
 
+## Building the static content
+
+1. checks posts that got posted but that have no corresponding source. deletes and sends out delete
+   activities. if any posts got deleted it could touch a fake flag file that causes the next step to
+   rebuild all indices even if there are no new or updated posts
+2. main pass that builds all the main activitypub objects and html pages and css. queues create or
+   update activities if necessary
+3. second pass based on tag info extracted during previous step, regenerates tag pages and deletes
+   ones that don't exist anymore. tag pages don't have corresponding AP objects so no activities are
+   required here
+
 ## User object
 
 endpoint: https://my.instance.tld/
@@ -54,6 +65,8 @@ be filled in with dummy data indicating the resource could not be fetched
 to recompile posts from markdown multiple times because that's super unnecessary. it also contains
 front matter with the post metadata and parsed hashtags
 
+/cache/tagindex.rktb stores an index of all hashtags and which posts contain them
+
 ## Task queue
 
 stores queued outgoing requests

private/rules.rkt

@@ -6,6 +6,7 @@
          net/url-string
          markdown
          markdown/toc
+         racket/fasl
          "compile.rkt"
          "prefs.rkt")
 
@@ -62,7 +63,6 @@
        (reverse (cons (substring str last-pos) items))]))
   (first (process+ xexpr)))
 
-
 (define compile-index-json
   (rule '("db/actorkey.pub" "src/instance.rktd" "src/bio.md") "public/index.json"
         (lambda (in out)
@@ -147,17 +147,36 @@
             (define post-htmls (map (lambda (b)
                                       (process-hashtags! b instance-url hashtags))
                                     (parse-markdown post-content)))
-            (write `((is-meow #t)
+            (write-bytes
-                     (hashtags ,(set->list hashtags))
+              (s-exp->fasl
-                     (toc ,(toc post-htmls))
+                (hash 'is-meow #t
-                     (content ,post-htmls)) out))))
+                      'hashtags (set->list hashtags)
+                      'toc (toc post-htmls)
+                      'content post-htmls))
+              out))))
 
-  (for/fold ([posts (set)]) ([post-name (in-list (directory-list "src/posts"))]
+  (define-values [posts-set cache-set]
-            #:when (regexp-match? #px"\\.md$" post-name))
+    (for/fold ([posts (set)] [caches (set)]) ([post-name (in-list (directory-list "src/posts"))]
-    (define post-src (build-path "src/posts" post-name))
+                                              #:when (regexp-match? #px"\\.md$" post-name))
-    (define cache-dst (build-path "cache/posts"
+      (define post-src (build-path "src/posts" post-name))
-                                  (path-replace-extension (last (explode-path post-src)) ".rktd")))
+      (define cache-dst (build-path "cache/posts"
-    (set-add posts (make-post-cache-rule post-src cache-dst))))
+                                    (path-replace-extension post-name ".rktb")))
+      (define json-dst (build-path "public/posts/"))
+      (values
+        (set-add posts (make-post-cache-rule post-src cache-dst))
+        (set-add caches cache-dst))))
+  ;; use the cache set to make a tags rule
+  (set-add
+    posts-set
+    (rule (set->list cache-set) "cache/tagindex.rktb"
+          (lambda (in out)
+            (define index (make-hash))
+            (for ([(fname port) (in-hash in)])
+              (define meta (fasl->s-exp (port->bytes port)))
+              (for ([tag (in-list (hash-ref meta 'hashtags))])
+                (hash-update! index tag (lambda (s) (set-add s fname)) set)))
+            (write-bytes (s-exp->fasl (for/hash ([(k v) (in-hash index)])
+                                        (values k (set->list v)))) out)))))
 
 
 (define compiler-rules