PER: tokenizer
This commit is contained in:
parent
53d3501604
commit
07ae409fdf
|
|
@ -0,0 +1,218 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from enum import Enum
|
||||
from typing import *
|
||||
|
||||
class PerTokenName(Enum):
|
||||
Title = "; @Title"
|
||||
Props = "; @Props"
|
||||
Author = "; @Author"
|
||||
Changelog = "; @Changelog"
|
||||
Manufacturer = "; @Manufacturer"
|
||||
Doc = "; @Doc"
|
||||
Core = "; @Core"
|
||||
Chip = "; @Chip"
|
||||
Chiplist = "; @Chiplist"
|
||||
Copyright = "; @Copyright"
|
||||
Description = "; @Description"
|
||||
Keywords = "; @Keywords"
|
||||
Date = "; @Date"
|
||||
#HeaderSep = "; -------"
|
||||
Id = "; @Id"
|
||||
|
||||
CONFIG = "config" # what is this
|
||||
BASE = "base" # base address, base addrspace:expr
|
||||
# expr can be int or (d.l(addrspace:off)) -> ???
|
||||
WIDTH = "width" # what is this
|
||||
SAVEINDEX = "saveindex" # what is this
|
||||
|
||||
TREE_OPEN = "tree.open"
|
||||
TREE_CLOSE = "tree.close" # like tree.open
|
||||
TREE_END = "tree.end"
|
||||
TREE = "tree"
|
||||
|
||||
# group of stuff (sometimes one reg?)
|
||||
# group (addrspace:)start--end
|
||||
# addrspaces: c15 c14 ad(=what?) <none> EE(avr eeprom) D(avr data?) d(msp430) CSR(riscv) NAR(xtensa shit), "e:comp.base('name',-1)"(also xtensa) SPR(xtensa)
|
||||
# also seen: group iospace() (teaklite)
|
||||
GROUP = "group"
|
||||
HGROUP = "hgroup"
|
||||
RGROUP = "rgroup"
|
||||
SGROUP = "sgroup"
|
||||
WGROUP = "wgroup"
|
||||
# a register (sometimes hidden)
|
||||
# line.qual offset "CODENAME,description"
|
||||
HIDE = "hide"
|
||||
LINE = "line"
|
||||
# bitfields and stuff in registers
|
||||
# bitfld.qual IDK start(--end) "CODE,description" ("if 0","if 1",...)
|
||||
BITFLD = "bitfld"
|
||||
ABITFLD = "abitfld"
|
||||
RBITFLD = "rbitfld"
|
||||
HEXFLD = "hexfld"
|
||||
#RHEXFLD? WHEXFLD??
|
||||
EVENTFLD = "eventfld"
|
||||
SETCLRFLD = "setclrfld"
|
||||
# masks in registers?
|
||||
# hexmask.qual IDK start--end MASK "CODE,description"
|
||||
DECMASK = "decmask"
|
||||
#RDECMASK? WDECMASK??
|
||||
HEXMASK = "hexmask"
|
||||
RHEXMASK = "rhexmask"
|
||||
#WHEXMASK??
|
||||
|
||||
COPY = "copy"
|
||||
REPEAT_END = "repeat.end"
|
||||
REPEAT_REPLAY = "repeat.replay"
|
||||
REPEAT = "repeat"
|
||||
|
||||
# to ignore: assert autoindent.{on,off} button elif else endif if in
|
||||
# newline sif textline textfld x
|
||||
# IDK: entry, read, wait, saveindex, saveout, set, getx, register
|
||||
# EH: include, endian.{be,le} (-> conditional endianness)
|
||||
# TODO: copy: copy from previous group? (->derivedFrom whee)
|
||||
# TODO: repeat{,.end}: repeat macro stuff (cf. dimIndexElement)
|
||||
# TODO: repeat.replay: copy+replay
|
||||
|
||||
PER_TOKEN_IGNORE = {
|
||||
'assert','autoindent.on','autoindent.off','button','elif','else','endif',
|
||||
'if','in','newline','sif','textline','textfld','x',
|
||||
'entry','read','wait','saveindex','saveout','set','getx','register',
|
||||
'include','endian.be','endian.le'
|
||||
}
|
||||
PER_TOKEN_HEADER = [PerTokenName.__members__[x] for x in (
|
||||
'Title','Props','Author','Changelog','Manufacturer','Doc','Core','Chiplist',
|
||||
'Copyright','Id','Chip','Description','Keywords','Date',#'HeaderSep',
|
||||
)]
|
||||
PER_TOKEN_BODY = [v for k, v in PerTokenName.__members__.items() if v not in PER_TOKEN_HEADER]
|
||||
|
||||
class PerTokenQual(Enum):
|
||||
# also .<hex>?
|
||||
BYTE = "byte"
|
||||
WORD = "word"
|
||||
LONG = "long"
|
||||
QUAD = "quad"
|
||||
SHORT = "short"
|
||||
SBYTE = "sbyte"
|
||||
TBYTE = "tbyte"
|
||||
# used for mask stuff
|
||||
"""
|
||||
LONG_TBYTE = "long.tbyte"
|
||||
LONG_BYTE = "long.byte"
|
||||
LONG_WORD = "long.word"
|
||||
LONG_LONG = "long.long"
|
||||
WORD_BYTE = "word.byte"
|
||||
WORD_WORD = "word.word"
|
||||
BYTE_BYTE = "byte.byte"
|
||||
QUAD_BYTE = "quad.byte"
|
||||
QUAD_SBYTE = "quad.sbyte"
|
||||
QUAD_TBYTE = "quad.tbyte"
|
||||
QUAD_WORD = "quad.word"
|
||||
QUAD_SHORT = "quad.short"
|
||||
QUAD_LONG = "quad.long"
|
||||
QUAD_QUAD = "quad.quad"
|
||||
TBYTE_BYTE = "tbyte.byte"
|
||||
TBYTE_WORD = "tbyte.word"
|
||||
"""
|
||||
|
||||
|
||||
class PerToken(NamedTuple):
|
||||
name: PerTokenName
|
||||
qual: Union[PerTokenQual, Tuple[PerTokenQual, PerTokenQual]] = None
|
||||
args: List[str] = []
|
||||
|
||||
|
||||
# TODO: tokenize into data stream with useful arguments
|
||||
# TODO: data stream -> tree structure
|
||||
|
||||
def find_tok(l: str, hdr: bool) -> PerTokenName:
|
||||
ll=l.lower() if hdr else l
|
||||
for h in (PER_TOKEN_HEADER if hdr else PER_TOKEN_BODY):
|
||||
#print("ll='%s', h='%s'"%(ll,h.value))
|
||||
if ll.startswith(h.value.lower() if hdr else h.value):
|
||||
return h
|
||||
assert not hdr or l.split()[0] in PER_TOKEN_IGNORE, "Unknown token on line: %s"%l
|
||||
return None
|
||||
|
||||
|
||||
def tokenize_body(f, l=None):
|
||||
prevtell = -1
|
||||
while True:
|
||||
if l is None: l = f.readline().strip()
|
||||
if len(l) == 0:
|
||||
tell = f.tell()
|
||||
if tell == prevtell: break # EOF
|
||||
prevtell = tell
|
||||
continue
|
||||
ll = l.lower()
|
||||
sp = l.split()
|
||||
# regular token
|
||||
t = find_tok(ll, False)
|
||||
#print("t",t)
|
||||
if t is not None:
|
||||
yield PerToken(t, args=sp[1:])
|
||||
l = None
|
||||
continue
|
||||
|
||||
ht = sp[0].lower().split('.')
|
||||
assert len(ht) > 0 or sp[0].lower() in PER_TOKEN_IGNORE, "Unknown token on line: %s"%l
|
||||
t = find_tok(ht[0], False)
|
||||
if t is not None:
|
||||
assert len(ht) in {2,3},"bad qual %s in line %s"%(repr(ht),l)
|
||||
quals = [PerTokenQual.__members__[k] for k in ht[1:]]
|
||||
if len(quals) == 1: quals = quals[0]
|
||||
else: quals = tuple(quals)
|
||||
yield PerToken(t, qual=quals, args=sp[1:])
|
||||
l = None
|
||||
continue
|
||||
|
||||
l = None
|
||||
|
||||
|
||||
def tokenize(f):
|
||||
curtok = None
|
||||
curlines = []
|
||||
|
||||
prevtell = -1
|
||||
while True:
|
||||
l = f.readline().strip()
|
||||
if len(l) == 0:
|
||||
tell = f.tell()
|
||||
if tell == prevtell: break # EOF
|
||||
prevtell = tell
|
||||
continue
|
||||
if l[0] != ';':
|
||||
yield from tokenize_body(f,l)
|
||||
break
|
||||
if l.startswith('; -------'): continue
|
||||
|
||||
if l.startswith('; @') and l[3] != ' ':
|
||||
# new token!
|
||||
# flush old one
|
||||
if curtok is not None:
|
||||
yield PerToken(curtok, args=curlines)
|
||||
# start new
|
||||
curtok = find_tok(l, True)
|
||||
curlines = [l[len(curtok.value)+1:].strip()]
|
||||
else:
|
||||
curlines.append(l[3:].strip())
|
||||
|
||||
|
||||
# flatten copy and repeat statements
|
||||
def tokenize_flatten(itor):
|
||||
yield from itor # TODO
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import glob
|
||||
for p in glob.glob('t/*/*.per'):
|
||||
print(p)
|
||||
with open(p,'r') as f:
|
||||
for x in tokenize(f): pass
|
||||
|
||||
with open('t/arm/peram65xx.per','r') as f:
|
||||
for x in tokenize(f): print(x)
|
||||
with open('t/arm/perfm0p.per','r') as f:
|
||||
for x in tokenize(f): print(x)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue