A mess of LALRPOP.

macros/Cargo.toml

@@ -2,9 +2,14 @@
 name = "typed-html-macros"
 version = "0.1.0"
 authors = ["Bodil Stokke <bodil@bodil.org>"]
+build = "build.rs"
 
 [lib]
 proc-macro = true
 
 [dependencies]
 pom = "2.0.1"
+lalrpop-util = "0.16.1"
+
+[build-dependencies]
+lalrpop = "0.16.1"

macros/build.rs

@@ -0,0 +1,5 @@
+extern crate lalrpop;
+
+fn main() {
+    lalrpop::process_root().unwrap();
+}

macros/src/declare.rs

@@ -3,21 +3,22 @@ use pom::Parser;
 use proc_macro::{quote, Group, Ident, Literal, TokenStream, TokenTree};
 
 use config::global_attrs;
+use lexer::{Lexer, ParseError, Token};
 use map::StringyMap;
-use parser::*;
+use parser::{self, *};
 
 // State
 
-struct Declare {
+pub struct Declare {
-    name: Ident,
+    pub name: Ident,
-    attrs: StringyMap<Ident, TokenStream>,
+    pub attrs: StringyMap<Ident, TokenStream>,
-    req_children: Vec<Ident>,
+    pub req_children: Vec<Ident>,
-    opt_children: Option<TokenStream>,
+    pub opt_children: Option<TokenStream>,
-    traits: Vec<TokenStream>,
+    pub traits: Vec<TokenStream>,
 }
 
 impl Declare {
-    fn new(name: Ident) -> Self {
+    pub fn new(name: Ident) -> Self {
         Declare {
             attrs: global_attrs(name.span()),
             req_children: Vec::new(),
@@ -45,8 +46,7 @@ impl Declare {
 
     fn attrs(&self) -> impl Iterator<Item = (TokenTree, TokenStream, TokenTree)> + '_ {
         self.attrs.iter().map(|(key, value)| {
-            let attr_name: TokenTree =
+            let attr_name: TokenTree = Ident::new_raw(&key.to_string(), key.span()).into();
-                Ident::new(&format!("attr_{}", key.to_string()), key.span()).into();
             let attr_type = value.clone();
             let attr_str = Literal::string(&key.to_string()).into();
             (attr_name, attr_type, attr_str)
@@ -64,7 +64,7 @@ impl Declare {
         })
     }
 
-    fn into_token_stream(self) -> TokenStream {
+    pub fn into_token_stream(self) -> TokenStream {
         let mut stream = TokenStream::new();
         stream.extend(self.attr_struct());
         stream.extend(self.struct_());
@@ -380,3 +380,7 @@ fn declare<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Declare>> {
 pub fn expand_declare(input: &[TokenTree]) -> pom::Result<TokenStream> {
     declare().parse(input).map(|decl| decl.into_token_stream())
 }
+
+pub fn expand_declare_lalrpop(input: &[Token]) -> Result<Vec<Declare>, ParseError> {
+    parser::grammar::DeclarationsParser::new().parse(Lexer::new(input))
+}

macros/src/grammar.lalrpop

@@ -0,0 +1,287 @@
+use lexer::{Token, to_stream, HtmlParseError, Keyword};
+use html::{Node, Element};
+use declare::Declare;
+use map::StringyMap;
+use proc_macro::{Delimiter, Ident, Literal, Group, TokenTree};
+use lalrpop_util::ParseError;
+
+grammar;
+
+/// Match a B separated list of zero or more A, return a list of A.
+Separated<A, B>: Vec<A> = {
+    <v:(<A> B)*> <e:A?> => match e {
+        None => v,
+        Some(e) => {
+            let mut v = v;
+            v.push(e);
+            v
+        }
+    }
+}
+
+/// Match a B separated list of one or more A, return a list of tokens, including the Bs.
+/// Both A and B must resolve to a Token.
+SeparatedInc<A, B>: Vec<Token> = {
+    <v:(A B)*> <e:A> => {
+        let mut out = Vec::new();
+        for (a, b) in v {
+            out.push(a);
+            out.push(b);
+        }
+        out.push(e);
+        out
+    }
+}
+
+Ident: Ident = IdentToken => {
+    match <> {
+        Token::Ident(ident) => ident,
+        _ => unreachable!()
+    }
+};
+
+Literal: Literal = LiteralToken => {
+    match <> {
+        Token::Literal(literal) => literal,
+        _ => unreachable!()
+    }
+};
+
+GroupToken = {
+    BraceGroupToken,
+    BracketGroupToken,
+    ParenGroupToken,
+};
+
+/// A kebab case HTML ident, converted to a snake case ident.
+HtmlIdent: Ident = {
+    <init:(<Ident> "-")*> <last:Ident> => {
+        let mut init = init;
+        init.push(last);
+        let (span, name) = init.into_iter().fold((None, String::new()), |(span, name), token| {
+            (
+                match span {
+                    None => Some(token.span()),
+                    Some(span) => span.join(token.span()),
+                },
+                if name.is_empty() {
+                    name + &token.to_string()
+                } else {
+                    name + "_" + &token.to_string()
+                }
+            )
+        });
+        Ident::new(&name, span.unwrap())
+    }
+};
+
+
+
+// The HTML macro
+
+/// An approximation of a Rust expression.
+BareExpression: Token = "&"? (IdentToken ":" ":")* SeparatedInc<IdentToken, "."> ParenGroupToken? => {
+    let (reference, left, right, args) = (<>);
+    let mut out = Vec::new();
+    if let Some(reference) = reference {
+        out.push(reference);
+    }
+    for (ident, c1, c2) in left {
+        out.push(ident);
+        out.push(c1);
+        out.push(c2);
+    }
+    out.extend(right);
+    if let Some(args) = args {
+        out.push(args);
+    }
+    Group::new(Delimiter::Brace, to_stream(out)).into()
+};
+
+AttrValue: Token = {
+    LiteralToken,
+    GroupToken,
+    BareExpression,
+};
+
+Attr: (Ident, Token) = <name:HtmlIdent> "=" <value:AttrValue> => (name, value);
+
+Attrs: StringyMap<Ident, TokenTree> = Attr* => <>.into();
+
+OpeningTag: (Ident, StringyMap<Ident, TokenTree>) = "<" <HtmlIdent> <Attrs> ">";
+
+ClosingTag: Ident = "<" "/" <HtmlIdent> ">";
+
+SingleTag: Element = "<" <name:HtmlIdent> <attributes:Attrs> "/" ">" => {
+    Element {
+        name,
+        attributes,
+        children: Vec::new(),
+    }
+};
+
+ParentTag: Element = <opening:OpeningTag> <children:Node*> <closing:ClosingTag> =>? {
+    let (name, attributes) = opening;
+    let closing_name = closing.to_string();
+    if closing_name == name.to_string() {
+        Ok(Element {
+            name,
+            attributes,
+            children,
+        })
+    } else {
+        Err(ParseError::User { error: HtmlParseError {
+            token: closing.into(),
+            message: format!("expected closing tag '</{}>', found '</{}>'", name.to_string(), closing_name),
+        }})
+    }
+};
+
+Element = {
+    SingleTag,
+    ParentTag,
+};
+
+TextNode = Literal;
+
+CodeBlock: Group = BraceGroupToken => match <> {
+    Token::Group(_, group) => group,
+    _ => unreachable!()
+};
+
+pub Node: Node = {
+    Element => Node::Element(<>),
+    TextNode => Node::Text(<>),
+    CodeBlock => Node::Block(<>),
+};
+
+
+
+// The declare macro
+
+TypePath: Vec<Token> = {
+    IdentToken => vec![<>],
+    TypePath ":" ":" IdentToken => {
+        let (mut path, c1, c2, last) = (<>);
+        path.push(c1);
+        path.push(c2);
+        path.push(last);
+        path
+    }
+};
+
+Reference: Vec<Token> = "&" ("'" IdentToken)? => {
+    let (amp, lifetime) = (<>);
+    let mut out = vec![amp];
+    if let Some((tick, ident)) = lifetime {
+        out.push(tick);
+        out.push(ident);
+    }
+    out
+};
+
+TypeArgs: Vec<Token> = {
+    TypeSpec,
+    TypeArgs "," TypeSpec => {
+        let (mut args, comma, last) = (<>);
+        args.push(comma);
+        args.extend(last);
+        args
+    }
+};
+
+TypeArgList: Vec<Token> = "<" TypeArgs ">" => {
+    let (left, mut args, right) = (<>);
+    args.insert(0, left);
+    args.push(right);
+    args
+};
+
+TypeSpec: Vec<Token> = Reference? TypePath TypeArgList? => {
+    let (reference, path, args) = (<>);
+    let mut out = Vec::new();
+    if let Some(reference) = reference {
+        out.extend(reference);
+    }
+    out.extend(path);
+    if let Some(args) = args {
+        out.extend(args);
+    }
+    out
+};
+
+TypeDecl: (Ident, Vec<Token>) = <HtmlIdent> ":" <TypeSpec>;
+
+TypeDecls: Vec<(Ident, Vec<Token>)> = {
+    TypeDecl => vec![<>],
+    <decls:TypeDecls> "," <decl:TypeDecl> => {
+        let mut decls = decls;
+        decls.push(decl);
+        decls
+    },
+};
+
+Attributes = "{" <TypeDecls> ","? "}";
+
+TypePathList = "[" <Separated<TypePath, ",">> "]";
+
+IdentList = "[" <Separated<Ident, ",">> "]";
+
+Groups = "in" <TypePathList>;
+
+Children: (Vec<Ident>, Option<Vec<Token>>) = "with" <req:IdentList?> <opt:TypePath?> => {
+    (req.unwrap_or_else(|| Vec::new()), opt)
+};
+
+Declaration: Declare = <name:HtmlIdent> <attrs:Attributes?> <groups:Groups?> <children:Children?> ";" => {
+    let mut decl = Declare::new(name);
+    if let Some(attrs) = attrs {
+        for (key, value) in attrs {
+            decl.attrs.insert(key, to_stream(value));
+        }
+    }
+    if let Some(groups) = groups {
+        for group in groups {
+            decl.traits.push(to_stream(group));
+        }
+    }
+    if let Some((req_children, opt_children)) = children {
+        decl.req_children = req_children;
+        decl.opt_children = opt_children.map(to_stream);
+    }
+    decl
+};
+
+pub Declarations = Declaration*;
+
+
+
+extern {
+    type Location = usize;
+    type Error = HtmlParseError;
+
+    enum Token {
+        "<" => Token::Punct('<', _),
+        ">" => Token::Punct('>', _),
+        "/" => Token::Punct('/', _),
+        "=" => Token::Punct('=', _),
+        "-" => Token::Punct('-', _),
+        ":" => Token::Punct(':', _),
+        "." => Token::Punct('.', _),
+        "," => Token::Punct(',', _),
+        "&" => Token::Punct('&', _),
+        "'" => Token::Punct('\'', _),
+        ";" => Token::Punct(';', _),
+        "{" => Token::GroupOpen(Delimiter::Brace, _),
+        "}" => Token::GroupClose(Delimiter::Brace, _),
+        "[" => Token::GroupOpen(Delimiter::Bracket, _),
+        "]" => Token::GroupClose(Delimiter::Bracket, _),
+        "in" => Token::Keyword(Keyword::In, _),
+        "with" => Token::Keyword(Keyword::With, _),
+        IdentToken => Token::Ident(_),
+        LiteralToken => Token::Literal(_),
+        ParenGroupToken => Token::Group(Delimiter::Parenthesis, _),
+        BraceGroupToken => Token::Group(Delimiter::Brace, _),
+        BracketGroupToken => Token::Group(Delimiter::Bracket, _),
+    }
+}

macros/src/html.rs

@@ -1,20 +1,21 @@
-use pom::combinator::*;
+use proc_macro::{
-use pom::Parser;
+    quote, Delimiter, Diagnostic, Group, Ident, Level, Literal, TokenStream, TokenTree,
-use proc_macro::{quote, Delimiter, Group, Ident, Literal, TokenStream, TokenTree};
+};
 
 use config::required_children;
+use lexer::{Lexer, ParseError, Token};
 use map::StringyMap;
-use parser::*;
+use parser::grammar;
 
 #[derive(Clone)]
-enum Node {
+pub enum Node {
     Element(Element),
     Text(Literal),
     Block(Group),
 }
 
 impl Node {
-    fn into_token_stream(self) -> TokenStream {
+    pub fn into_token_stream(self) -> TokenStream {
         match self {
             Node::Element(el) => el.into_token_stream(),
             Node::Text(text) => {
@@ -52,10 +53,10 @@ impl Node {
 }
 
 #[derive(Clone)]
-struct Element {
+pub struct Element {
-    name: Ident,
+    pub name: Ident,
-    attributes: StringyMap<Ident, TokenTree>,
+    pub attributes: StringyMap<Ident, TokenTree>,
-    children: Vec<Node>,
+    pub children: Vec<Node>,
 }
 
 fn extract_data_attrs(attrs: &mut StringyMap<Ident, TokenTree>) -> StringyMap<String, TokenTree> {
@@ -92,32 +93,30 @@ fn is_string_literal(literal: &Literal) -> bool {
 }
 
 impl Element {
-    fn new(name: Ident) -> Self {
-        Element {
-            name,
-            attributes: StringyMap::new(),
-            children: Vec::new(),
-        }
-    }
-
     fn into_token_stream(mut self) -> TokenStream {
         let name = self.name;
         let name_str = name.to_string();
         let typename: TokenTree = Ident::new(&format!("Element_{}", &name_str), name.span()).into();
         let req_names = required_children(&name_str);
         if req_names.len() > self.children.len() {
-            panic!(
+            Diagnostic::spanned(
-                "<{}> requires {} children but found only {}",
+                name.span(),
-                name_str,
+                Level::Error,
-                req_names.len(),
+                format!(
-                self.children.len()
+                    "<{}> requires {} children but there are only {}",
-            );
+                    name_str,
+                    req_names.len(),
+                    self.children.len()
+                ),
+            )
+            .emit();
+            panic!();
         }
         let data_attrs = extract_data_attrs(&mut self.attributes);
         let attrs = self.attributes.iter().map(|(key, value)| {
             (
                 key.to_string(),
-                TokenTree::Ident(Ident::new(&format!("attr_{}", key), key.span())),
+                TokenTree::Ident(Ident::new_raw(&key.to_string(), key.span())),
                 value,
             )
         });
@@ -145,9 +144,9 @@ impl Element {
                     let pos_str: TokenTree = Literal::string(&pos).into();
                     body.extend(quote!(
                         element.attrs.$key = Some($value.parse().unwrap_or_else(|err| {
-                            eprintln!("ERROR: {}: <{} {}={:?}> attribute value was not accepted: {:?}",
+                            eprintln!("ERROR: {}: <{} {}={:?}> failed to parse attribute value: {}",
                                       $pos_str, $tag_name, $attr_str, $value, err);
-                            panic!();
+                            panic!("failed to parse string literal");
                         }));
                     ));
                 }
@@ -184,57 +183,6 @@ impl Element {
     }
 }
 
-fn element_start<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Element>> {
+pub fn expand_html(input: &[Token]) -> Result<Node, ParseError> {
-    (punct('<') * html_ident()).map(Element::new)
+    grammar::NodeParser::new().parse(Lexer::new(input))
-}
-
-fn attr_value<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = TokenTree>> {
-    literal().map(TokenTree::Literal) | dotted_ident() | group().map(TokenTree::Group)
-}
-
-fn attr<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = (Ident, TokenTree)>> {
-    html_ident() + (punct('=') * attr_value())
-}
-
-fn element_with_attrs<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Element>> {
-    (element_start() + attr().repeat(0..)).map(|(mut el, attrs)| {
-        for (name, value) in attrs {
-            el.attributes.insert(name, value);
-        }
-        el
-    })
-}
-
-fn element_single<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Element>> {
-    element_with_attrs() - punct('/') - punct('>')
-}
-
-fn element_open<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Element>> {
-    element_with_attrs() - punct('>')
-}
-
-fn element_close<'a>(name: &str) -> Combinator<impl Parser<'a, TokenTree, Output = ()>> {
-    let name = name.to_lowercase();
-    // TODO make this return an error message containing the tag name
-    punct('<') * punct('/') * ident_match(name) * punct('>').discard()
-}
-
-fn element_with_children<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Element>> {
-    (element_open() + comb(node).repeat(0..)).map(|(mut el, children)| {
-        el.children.extend(children.into_iter());
-        el
-    }) >> |el: Element| element_close(&el.name.to_string()).expect("closing tag") * unit(el)
-}
-
-fn node(input: &[TokenTree], start: usize) -> pom::Result<(Node, usize)> {
-    (element_single().map(Node::Element)
-        | element_with_children().map(Node::Element)
-        | literal().map(Node::Text)
-        | group().map(Node::Block))
-    .0
-    .parse(input, start)
-}
-
-pub fn expand_html(input: &[TokenTree]) -> pom::Result<TokenStream> {
-    comb(node).parse(input).map(|el| el.into_token_stream())
 }

macros/src/lexer.rs

@@ -0,0 +1,227 @@
+use lalrpop_util::ParseError::*;
+use proc_macro::{
+    Delimiter, Diagnostic, Group, Ident, Level, Literal, Punct, Span, TokenStream, TokenTree,
+};
+
+pub type Spanned<Tok, Loc, Error> = Result<(Loc, Tok, Loc), Error>;
+pub type ParseError = lalrpop_util::ParseError<usize, Token, HtmlParseError>;
+
+#[derive(Clone, Debug)]
+pub enum Token {
+    Ident(Ident),
+    Literal(Literal),
+    Punct(char, Punct),
+    Group(Delimiter, Group),
+    GroupOpen(Delimiter, Span),
+    GroupClose(Delimiter, Span),
+    Keyword(Keyword, Ident),
+}
+
+impl Token {
+    pub fn span(&self) -> Span {
+        match self {
+            Token::Ident(ident) => ident.span(),
+            Token::Literal(literal) => literal.span(),
+            Token::Punct(_, punct) => punct.span(),
+            Token::Group(_, group) => group.span(),
+            Token::GroupOpen(_, span) => *span,
+            Token::GroupClose(_, span) => *span,
+            Token::Keyword(_, ident) => ident.span(),
+        }
+    }
+
+    pub fn is_ident(&self) -> bool {
+        match self {
+            Token::Ident(_) => true,
+            _ => false,
+        }
+    }
+}
+
+impl From<Token> for TokenTree {
+    fn from(token: Token) -> Self {
+        match token {
+            Token::Ident(ident) => TokenTree::Ident(ident),
+            Token::Literal(literal) => TokenTree::Literal(literal),
+            Token::Punct(_, punct) => TokenTree::Punct(punct),
+            Token::Group(_, group) => TokenTree::Group(group),
+            Token::GroupOpen(_, _) => panic!("Can't convert a GroupOpen token to a TokenTree"),
+            Token::GroupClose(_, _) => panic!("Can't convert a GroupClose token to a TokenTree"),
+            Token::Keyword(_, ident) => TokenTree::Ident(ident),
+        }
+    }
+}
+
+impl From<Token> for TokenStream {
+    fn from(token: Token) -> Self {
+        TokenTree::from(token).into()
+    }
+}
+
+impl From<Ident> for Token {
+    fn from(ident: Ident) -> Self {
+        Token::Ident(ident)
+    }
+}
+
+impl From<Literal> for Token {
+    fn from(literal: Literal) -> Self {
+        Token::Literal(literal)
+    }
+}
+
+impl From<Punct> for Token {
+    fn from(punct: Punct) -> Self {
+        Token::Punct(punct.as_char(), punct)
+    }
+}
+
+impl From<Group> for Token {
+    fn from(group: Group) -> Self {
+        Token::Group(group.delimiter(), group)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum Keyword {
+    In,
+    With,
+}
+
+pub fn keywordise(tokens: Vec<Token>) -> Vec<Token> {
+    tokens
+        .into_iter()
+        .map(|token| match token {
+            Token::Ident(ident) => {
+                let name = ident.to_string();
+                if name == "in" {
+                    Token::Keyword(Keyword::In, ident)
+                } else if name == "with" {
+                    Token::Keyword(Keyword::With, ident)
+                } else {
+                    Token::Ident(ident)
+                }
+            }
+            t => t,
+        })
+        .collect()
+}
+
+#[derive(Debug)]
+pub struct HtmlParseError {
+    pub token: Token,
+    pub message: String,
+}
+
+fn pprint_token(token: &str) -> &str {
+    match token {
+        "BraceGroupToken" => "code block",
+        "LiteralToken" => "literal",
+        "IdentToken" => "identifier",
+        a => a,
+    }
+}
+
+fn pprint_tokens(tokens: &[String]) -> String {
+    let tokens: Vec<&str> = tokens.iter().map(|s| pprint_token(&s)).collect();
+    if tokens.len() > 1 {
+        let start = tokens[..tokens.len() - 1].join(", ");
+        let end = &tokens[tokens.len() - 1];
+        format!("{} or {}", start, end)
+    } else {
+        tokens[0].to_string()
+    }
+}
+
+fn is_in_node_position(tokens: &[String]) -> bool {
+    use std::collections::HashSet;
+    let input: HashSet<&str> = tokens.iter().map(String::as_str).collect();
+    let output: HashSet<&str> = ["\"<\"", "BraceGroupToken", "LiteralToken"]
+        .iter()
+        .cloned()
+        .collect();
+    input == output
+}
+
+pub fn parse_error(input: &[Token], error: &ParseError) -> Diagnostic {
+    match error {
+        InvalidToken { location } => {
+            let loc = &input[*location];
+            Diagnostic::spanned(loc.span(), Level::Error, "invalid token")
+        }
+        UnrecognizedToken {
+            token: None,
+            expected,
+        } => panic!(
+            "unexpected end of macro: expecting {}",
+            pprint_tokens(&expected)
+        ),
+        UnrecognizedToken {
+            token: Some((_, token, _)),
+            expected,
+        } => {
+            let mut msg = format!("expected {}", pprint_tokens(&expected));
+            if is_in_node_position(expected) && token.is_ident() {
+                // special case: you probably meant to quote that text
+                msg += "; looks like you forgot to put \"quotes\" around your text nodes";
+            }
+            Diagnostic::spanned(token.span(), Level::Error, msg)
+        }
+        ExtraToken {
+            token: (_, token, _),
+        } => Diagnostic::spanned(token.span(), Level::Error, "superfluous token"),
+        User { error } => {
+            Diagnostic::spanned(error.token.span(), Level::Error, error.message.to_owned())
+        }
+    }
+}
+
+pub fn to_stream<I: IntoIterator<Item = Token>>(tokens: I) -> TokenStream {
+    let mut stream = TokenStream::new();
+    stream.extend(tokens.into_iter().map(TokenTree::from));
+    stream
+}
+
+pub fn unroll_stream(stream: TokenStream, deep: bool) -> Vec<Token> {
+    let mut vec = Vec::new();
+    for tt in stream {
+        match tt {
+            TokenTree::Ident(ident) => vec.push(ident.into()),
+            TokenTree::Literal(literal) => vec.push(literal.into()),
+            TokenTree::Punct(punct) => vec.push(punct.into()),
+            TokenTree::Group(ref group) if deep => {
+                vec.push(Token::GroupOpen(group.delimiter(), group.span()));
+                let sub = unroll_stream(group.stream(), deep);
+                vec.extend(sub);
+                vec.push(Token::GroupClose(group.delimiter(), group.span()));
+            }
+            TokenTree::Group(group) => vec.push(group.into()),
+        }
+    }
+    vec
+}
+
+pub struct Lexer<'a> {
+    stream: &'a [Token],
+    pos: usize,
+}
+
+impl<'a> Lexer<'a> {
+    pub fn new(stream: &'a [Token]) -> Self {
+        Lexer { stream, pos: 0 }
+    }
+}
+
+impl<'a> Iterator for Lexer<'a> {
+    type Item = Spanned<Token, usize, HtmlParseError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.stream.get(self.pos) {
+            None => None,
+            Some(token) => {
+                self.pos += 1;
+                Some(Ok((self.pos - 1, token.clone(), self.pos)))
+            }
+        }
+    }
+}

macros/src/lib.rs

@@ -2,7 +2,9 @@
 #![feature(proc_macro_quote)]
 #![feature(proc_macro_span)]
 #![feature(proc_macro_diagnostic)]
+#![feature(proc_macro_raw_ident)]
 
+extern crate lalrpop_util;
 extern crate pom;
 extern crate proc_macro;
 
@@ -11,19 +13,39 @@ use proc_macro::{TokenStream, TokenTree};
 mod config;
 mod declare;
 mod html;
+mod lexer;
 mod map;
 mod parser;
 
 #[proc_macro]
 pub fn html(input: TokenStream) -> TokenStream {
-    let input: Vec<TokenTree> = input.into_iter().collect();
+    let stream = lexer::unroll_stream(input, false);
-    let result = html::expand_html(&input);
+    let result = html::expand_html(&stream);
     match result {
         Err(error) => {
-            parser::parse_error(&input, &error).emit();
+            lexer::parse_error(&stream, &error).emit();
             panic!("macro expansion produced errors; see above.")
         }
-        Ok(ts) => ts,
+        Ok(node) => node.into_token_stream(),
+    }
+}
+
+#[proc_macro]
+pub fn declalrpop_element(input: TokenStream) -> TokenStream {
+    let stream = lexer::keywordise(lexer::unroll_stream(input, true));
+    let result = declare::expand_declare_lalrpop(&stream);
+    match result {
+        Err(error) => {
+            lexer::parse_error(&stream, &error).emit();
+            panic!("macro expansion produced errors; see above.")
+        }
+        Ok(decls) => {
+            let mut out = TokenStream::new();
+            for decl in decls {
+                out.extend(decl.into_token_stream());
+            }
+            out
+        }
     }
 }
 

macros/src/map.rs

@@ -29,3 +29,18 @@ where
         self.0.values().map(|(k, _)| k)
     }
 }
+
+impl<K, V, OK, OV> From<Vec<(OK, OV)>> for StringyMap<K, V>
+where
+    OK: Into<K>,
+    OV: Into<V>,
+    K: ToString,
+{
+    fn from(vec: Vec<(OK, OV)>) -> Self {
+        let mut out = Self::new();
+        for (key, value) in vec {
+            out.insert(key.into(), value.into());
+        }
+        out
+    }
+}

macros/src/parser.rs

@@ -1,12 +1,9 @@
+use lalrpop_util::lalrpop_mod;
 use pom::combinator::*;
 use pom::{Error, Parser};
-use proc_macro::{
+use proc_macro::{Diagnostic, Group, Ident, Level, Punct, TokenStream, TokenTree};
-    Delimiter, Diagnostic, Group, Ident, Level, Literal, Punct, TokenStream, TokenTree,
-};
 
-pub fn unit<'a, I: 'a, A: Clone>(value: A) -> Combinator<impl Parser<'a, I, Output = A>> {
+lalrpop_mod!(pub grammar);
-    comb(move |_, start| Ok((value.clone(), start)))
-}
 
 pub fn punct<'a>(punct: char) -> Combinator<impl Parser<'a, TokenTree, Output = Punct>> {
     comb(move |input: &[TokenTree], start| match input.get(start) {
@@ -28,35 +25,6 @@ pub fn ident<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Ident>> {
     })
 }
 
-pub fn ident_match<'a>(name: String) -> Combinator<impl Parser<'a, TokenTree, Output = ()>> {
-    comb(move |input: &[TokenTree], start| match input.get(start) {
-        Some(TokenTree::Ident(i)) => {
-            if i.to_string() == name {
-                Ok(((), start + 1))
-            } else {
-                Err(Error::Mismatch {
-                    message: format!("expected '</{}>', found '</{}>'", name, i.to_string()),
-                    position: start,
-                })
-            }
-        }
-        _ => Err(Error::Mismatch {
-            message: "expected identifier".to_string(),
-            position: start,
-        }),
-    })
-}
-
-pub fn literal<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Literal>> {
-    comb(|input: &[TokenTree], start| match input.get(start) {
-        Some(TokenTree::Literal(l)) => Ok((l.clone(), start + 1)),
-        _ => Err(Error::Mismatch {
-            message: "expected literal".to_string(),
-            position: start,
-        }),
-    })
-}
-
 pub fn group<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Group>> {
     comb(|input: &[TokenTree], start| match input.get(start) {
         Some(TokenTree::Group(g)) => Ok((g.clone(), start + 1)),
@@ -83,45 +51,6 @@ pub fn type_spec<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = TokenSt
     valid.repeat(1..).collect().map(to_stream)
 }
 
-pub fn dotted_ident<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = TokenTree>> {
-    (ident()
-        + ((punct('.') + ident()).discard() | (punct(':').repeat(2) + ident()).discard())
-            .repeat(0..))
-    .collect()
-    .map(|tokens| {
-        if tokens.len() == 1 {
-            tokens[0].clone()
-        } else {
-            Group::new(Delimiter::Brace, to_stream(tokens)).into()
-        }
-    })
-}
-
-/// Read a sequence of idents and dashes, and merge them into a single ident
-/// with the dashes replaced by underscores.
-pub fn html_ident<'a>() -> Combinator<impl Parser<'a, TokenTree, Output = Ident>> {
-    let start = ident();
-    let next = punct('-') * ident();
-    (start * next.repeat(0..)).collect().map(|stream| {
-        let (span, name) = stream
-            .iter()
-            .fold((None, String::new()), |(span, name), token| {
-                (
-                    match span {
-                        None => Some(token.span()),
-                        Some(span) => span.join(token.span()),
-                    },
-                    match token {
-                        TokenTree::Ident(ident) => name + &ident.to_string(),
-                        TokenTree::Punct(_) => name + "_",
-                        _ => unreachable!(),
-                    },
-                )
-            });
-        Ident::new(&name, span.unwrap())
-    })
-}
-
 /// Turn a parser error into a proc_macro diagnostic.
 pub fn parse_error(input: &[TokenTree], error: &pom::Error) -> Diagnostic {
     match error {

typed-html/src/elements.rs

@@ -2,7 +2,7 @@
 #![allow(dead_code)]
 
 use std::fmt::Display;
-use typed_html_macros::declare_element;
+use typed_html_macros::{declalrpop_element, declare_element};
 
 use super::types::*;
 
@@ -99,102 +99,106 @@ impl Node for TextNode {
 impl FlowContent for TextNode {}
 impl PhrasingContent for TextNode {}
 
-declare_element!(html {
+declalrpop_element!{
-    xmlns: Uri,
+    html {
-} [head, body]);
+        xmlns: Uri,
-declare_element!(head {} [title] MetadataContent);
+    } with [head, body];
-declare_element!(body {} [] FlowContent);
+    head with [title] MetadataContent;
+    body with FlowContent;
 
-// Metadata content
+    // Metadata
-declare_element!(base {
+    base {
-    href: Uri,
+        href: Uri,
-    target: Target,
+        target: Target,
-} [] [MetadataContent]);
+    } in [MetadataContent];
-declare_element!(link {
+    link {
-    as: Mime,
+        as: Mime,
-    crossorigin: CrossOrigin,
+        crossorigin: CrossOrigin,
-    href: Uri,
+        href: Uri,
-    hreflang: LanguageTag,
+        hreflang: LanguageTag,
-    media: String, // FIXME media query
+        media: String, // FIXME media query
-    rel: LinkType,
+        rel: LinkType,
-    sizes: String, // FIXME
+        sizes: String, // FIXME
-    title: String, // FIXME
+        title: String, // FIXME
-    type: Mime,
+        type: Mime,
-} [] [MetadataContent]);
+    } in [MetadataContent];
-declare_element!(meta {
+    meta {
-    charset: String, // FIXME IANA standard names
+        charset: String, // FIXME IANA standard names
-    content: String,
+        content: String,
-    http_equiv: String, // FIXME string enum
+        http_equiv: String, // FIXME string enum
-    name: String, // FIXME string enum
+        name: String, // FIXME string enum
-} [] [MetadataContent]);
+    } in [MetadataContent];
-declare_element!(style {
+    style {
-    type: Mime,
+        type: Mime,
-    media: String, // FIXME media query
+        media: String, // FIXME media query
-    nonce: Nonce,
+        nonce: Nonce,
-    title: String, // FIXME
+        title: String, // FIXME
-} [] [MetadataContent] TextNode);
+    } in [MetadataContent] with TextNode;
-declare_element!(title {} [] [MetadataContent] TextNode);
+    title in [MetadataContent] with TextNode;
+
+    // Flow
+    a {
+        download: String,
+        href: Uri,
+        hreflang: LanguageTag,
+        ping: SpacedList<Uri>,
+        rel: SpacedList<LinkType>,
+        target: Target,
+        type: Mime,
+    } in [FlowContent, PhrasingContent, InteractiveContent] with FlowContent;
+    abbr in [FlowContent, PhrasingContent] with PhrasingContent;
+    address in [FlowContent] with FlowContent;
+    article in [FlowContent, SectioningContent] with FlowContent;
+    aside in [FlowContent, SectioningContent] with FlowContent;
+    audio {
+        autoplay: bool,
+        controls: bool,
+        crossorigin: CrossOrigin,
+        loop: bool,
+        muted: bool,
+        preload: Preload,
+        src: Uri,
+    } in [FlowContent, PhrasingContent, EmbeddedContent] with MediaContent;
+    b in [FlowContent, PhrasingContent] with PhrasingContent;
+    bdo in [FlowContent, PhrasingContent] with PhrasingContent;
+    bdi in [FlowContent, PhrasingContent] with PhrasingContent;
+    blockquote {
+        cite: Uri,
+    } in [FlowContent] with FlowContent;
+    br in [FlowContent, PhrasingContent];
+    button {
+        autofocus: bool,
+        disabled: bool,
+        form: Id,
+        formaction: Uri,
+        formenctype: FormEncodingType,
+        formmethod: FormMethod,
+        formnovalidate: bool,
+        formtarget: Target,
+        name: Id,
+        type: ButtonType,
+        value: String,
+    } in [FlowContent, PhrasingContent, InteractiveContent, FormContent] with PhrasingContent;
+    canvas {
+        height: usize,
+        width: usize,
+    } in [FlowContent, PhrasingContent, EmbeddedContent] with FlowContent;
+    cite in [FlowContent, PhrasingContent] with PhrasingContent;
+    code in [FlowContent, PhrasingContent] with PhrasingContent;
+    data {
+        value: String,
+    } in [FlowContent, PhrasingContent] with PhrasingContent;
+    datalist in [FlowContent, PhrasingContent] with Element_option;
+    del {
+        cite: Uri,
+        datetime: Datetime,
+    } in [FlowContent, PhrasingContent] with FlowContent;
+    details {
+        open: bool,
+    } in [FlowContent, SectioningContent, InteractiveContent] with [summary] FlowContent;
+}
 
 // Flow content
-declare_element!(a {
-    download: String,
-    href: Uri,
-    hreflang: LanguageTag,
-    ping: SpacedList<Uri>,
-    rel: SpacedList<LinkType>,
-    target: Target,
-    type: Mime,
-} [] [FlowContent, PhrasingContent, InteractiveContent] FlowContent);
-declare_element!(abbr {} [] [FlowContent, PhrasingContent] PhrasingContent);
-declare_element!(address {} [] [FlowContent] FlowContent); // FIXME it has additional constraints on FlowContent
-declare_element!(article {} [] [FlowContent, SectioningContent] FlowContent);
-declare_element!(aside {} [] [FlowContent, SectioningContent] FlowContent);
-declare_element!(audio {
-    autoplay: bool,
-    controls: bool,
-    crossorigin: CrossOrigin,
-    loop: bool,
-    muted: bool,
-    preload: Preload,
-    src: Uri,
-} [] [FlowContent, PhrasingContent, EmbeddedContent] MediaContent);
-declare_element!(b {} [] [FlowContent, PhrasingContent] PhrasingContent);
-declare_element!(bdo {} [] [FlowContent, PhrasingContent] PhrasingContent);
-declare_element!(bdi {} [] [FlowContent, PhrasingContent] PhrasingContent);
-declare_element!(blockquote {
-    cite: Uri,
-} [] [FlowContent] FlowContent);
-declare_element!(br {} [] [FlowContent, PhrasingContent]);
-declare_element!(button {
-    autofocus: bool,
-    disabled: bool,
-    form: Id,
-    formaction: Uri,
-    formenctype: FormEncodingType,
-    formmethod: FormMethod,
-    formnovalidate: bool,
-    formtarget: Target,
-    name: Id,
-    type: ButtonType,
-    value: String,
-} [] [FlowContent, PhrasingContent, InteractiveContent, FormContent] PhrasingContent);
-declare_element!(canvas {
-    height: usize,
-    width: usize,
-} [] [FlowContent, PhrasingContent, EmbeddedContent] FlowContent); // FIXME has additional child constraints
-declare_element!(cite {} [] [FlowContent, PhrasingContent] PhrasingContent);
-declare_element!(code {} [] [FlowContent, PhrasingContent] PhrasingContent);
-declare_element!(data {
-    value: String,
-} [] [FlowContent, PhrasingContent] PhrasingContent);
-declare_element!(datalist {} [] [FlowContent, PhrasingContent] Element_option);
-declare_element!(del {
-    cite: Uri,
-    datetime: Datetime,
-} [] [FlowContent, PhrasingContent] FlowContent);
-declare_element!(details {
-    open: bool,
-} [summary] [FlowContent, SectioningContent, InteractiveContent] FlowContent);
 declare_element!(dfn {} [] [FlowContent, PhrasingContent] PhrasingContent);
 declare_element!(div {} [] [FlowContent] FlowContent);
 declare_element!(dl {} [] [FlowContent] DescriptionListContent);

typed-html/src/types/spacedset.rs

@@ -37,6 +37,19 @@ impl<'a, A: 'a + Ord + Clone> FromIterator<&'a A> for SpacedSet<A> {
     }
 }
 
+impl<'a, A: Ord + FromStr> FromStr for SpacedSet<A>
+where
+    <A as FromStr>::Err: Debug,
+{
+    type Err = <A as FromStr>::Err;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let result: Result<Vec<A>, Self::Err> =
+            s.split_whitespace().map(|s| FromStr::from_str(s)).collect();
+        result.map(|items| Self::from_iter(items))
+    }
+}
+
 impl<'a, A: Ord + FromStr> From<&'a str> for SpacedSet<A>
 where
     <A as FromStr>::Err: Debug,