stuff

.gitignore

@@ -0,0 +1,3 @@
+t/
+__pycache__/
+*.pyc

dvf/dvf2h.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+
+import argparse, sys
+from typing import *
+
+import dvfparse
+from dvfparse import Dvf, DvfMI, DvfVN, DvfSN
+
+
+def comment_lines(t: str, lines: Sequence[str]) -> Sequence[str]:
+    if t in ('c', 'gas-with-cpp', 'gas', 'ld-script'):
+        return ["/* ", *[" * " + l for l in lines], " */"]
+    else:
+        return ["; " + l for l in lines]
+
+
+def format_define(t: str, pfix: str, name: str, addr: str, width: int, deref: bool) -> str:
+    name = name.upper()
+    if t == 'c':
+        return "#define %s%s (%s(volatile uint%d_t*)0x%04x)" % \
+            (pfix, name, "*" if deref else "", width, addr)
+    elif t in ('gas-with-cpp', 'gas', 'nasm'):
+        PFIX = {
+            'gas-with-cpp': '#',
+            'gas': '.',
+            'nasm': '%'
+        }
+        return "%cdefine %s%s 0x%04x" % (PFIX[t], pfix, name, addr)
+    elif t in ('nasm-equ', 'asxxxx', 'elflift'):
+        EQU = {
+            'nasm-equ': " equ ",
+            'asxxxx': " = ",
+            'elflift': "@",
+        }
+        return "%s%s%s0x%04x" % (pfix, name, EQU[t], addr)
+    else:
+        assert False, ("bad format type %s" % t)
+
+
+
+def format_defines(t: str, pfixr: str, pfixv: str,
+                   regs: List[DvfSN], vt: List[DvfVN]) -> str:
+    return '\n'.join(format_define(t, pfixr, reg.name, reg.address,
+                                   (reg.flg2 & 0xf) * 2, True) for reg in regs) \
+        + '\n'.join(
+            format_define(t, pfixv, ev.name, ev.address, 16, False) for ev in vt) + '\n'
+
+
+
+def format_mem_line(mi: DvfMI) -> str:
+    # FIXME: handle multiple of each existing
+    NAMES = {
+        DvfMI.MREG_FLASH: "flash",
+        DvfMI.MREG_RAM: "ram",
+        DvfMI.MREG_VECTOR: "vectors",
+        DvfMI.MREG_CALLT: "callt",
+        DvfMI.MREG_CALLF: "callf"
+    }
+
+    return "\t%s (%s%s%s) : ORIGIN = 0x%04x, LENGTH = 0x%04x" % \
+        (NAMES.get(mi.mreg_type, "//unk%02x"%mi.mreg_type),
+         "r" if (mi.access & 1) != 0 else "",
+         "w" if (mi.access & 2) != 0 else "",
+         "x" if (mi.access & 4) != 0 else "",
+         mi.addr_start, mi.addr_end - mi.addr_start + 1)
+
+def format_linker_script(mi: List[DvfMI]) -> str:
+    useful_types = {
+        DvfMI.MREG_FLASH,
+        DvfMI.MREG_RAM,
+        DvfMI.MREG_VECTOR,
+        DvfMI.MREG_CALLT,
+        DvfMI.MREG_CALLF,
+    }
+
+    #mi = [m for m in mi if m.mreg_type in useful_types]
+
+    flashreg = [m for m in mi if m.mreg_type == DvfMI.MREG_FLASH]
+    flashreg.sort(key=lambda m: m.addr_start)
+    assert len(flashreg) > 0
+    flashreg = flashreg[0]
+
+    if not any(m.mreg_type == DvfMI.MREG_VECTOR for m in mi):
+        # let's guess something
+        lowest_next = [m for m in mi if m.addr_start > flashreg.addr_start]
+        lowest_next.sort(key=lambda m: m.addr_start)
+        assert len(lowest_next) > 0
+        if lowest_next[0].addr_start < flashreg.addr_end:
+            # we need a subregion of flash so this should be usable
+            mi.append(DvfMI(0, lowest_next[0].addr_start-1,
+                            DvfMI.MREG_VECTOR, 1))
+
+            # now modify the flash so that it only starts AFTER the lowest_next
+            i = 0
+            oldorg = flashreg.addr_start
+            while any(m.addr_start == flashreg.addr_start
+                      and m.mreg_type != DvfMI.MREG_FLASH for m in mi):
+                flashreg = flashreg._replace(addr_start = lowest_next[i].addr_end + 1)
+                i = i = 1
+
+            for i in range(len(mi)):
+                if mi[i].addr_start == oldorg and mi[i].mreg_type == DvfMI.MREG_FLASH:
+                    mi[i] = flashreg
+                    break
+
+    return "MEMORY {\n" + '\n'.join(format_mem_line(m) for m in mi) + "\n}\n"
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(prog="dvf2h",
+                                     description="Convert NEC/Renesas Device "+\
+                                     "Files to C/asm defines and linker scripts")
+
+    parser.add_argument('-p', '--use-reg-prefix', dest='reg_prefix',
+                        default=None, action='store_true',
+                        help="Use a REG_ prefix in the names for registers. "+\
+                        "Defaults to on for the 'c' output format, but off otherwise")
+    parser.add_argument('-P', '--no-use-reg-prefix', dest='reg_prefix',
+                        default=None, action='store_false',
+                        help="Don't use a REG_ prefix in the names for registers. "+\
+                        "See the help for `--use-reg-prefix' for more info.")
+
+    parser.add_argument('-f', '--out-format', default="c",
+                        choices=["c", "gas-with-cpp", "gas", "nasm-def",
+                                 "nasm-equ", "asxxxx", "elflift", "ld-script"],
+                        help="Output format. 'c' is a C/C++ header (default), "+\
+                        "'gas-with-cpp' is a GNU as-compatible header, "+\
+                        "'gas' uses GNU as .define directives, "+\
+                        "'nasm-def' outputs NASM %%define directives, "+\
+                        "'nasm-equ' outputs defines using the EQU syntax, "+\
+                        "'asxxxx' outputs ASxxxx-compatible include files, "+\
+                        "'elflift' outputs an elflift-compatible symbol table, "+\
+                        "'ld-script' outputs the MEMORY part of a GNU ld script.")
+
+    parser.add_argument('-o', '--output', type=str, default=None)
+
+    parser.add_argument('input', type=argparse.FileType('rb'))
+
+    args = parser.parse_args()
+
+    reg_prefix = args.reg_prefix or (args.out_format == 'c')
+    output = sys.stdout if args.output is None else open(args.output, 'w')
+
+    try:
+        dvf = Dvf.parse(args.input.read())
+        md = dvf.sections[dvfparse.name2int('MD')]
+        mi = dvf.sections[dvfparse.name2int('MI')]
+        sn = dvf.sections[dvfparse.name2int('SN')]
+        es = [v for k, v in dvf.sections.items() if k == dvfparse.name2int('ES')]
+        vn = dvf.sections[dvfparse.name2int('VN')]
+        assert len(md) == 1
+        md = md[0]
+        assert len(es) == 0 or len(es) == 1
+        es = es[0] if len(es) != 0 else None
+
+        # FIXME: do something about bitflags
+        #        these are flags of another listed register, you have to find
+        #        the non-1bit SN entry with the same address as this one to
+        #        figure out the name of the register it belogs to
+        sn = [DvfSN(s.name, s.flg0, s.flg1, s.flg2, s.address, s.flg3, s.flg4)
+              for s in sn if (s.flg2 & 0xf) != 1]
+        if es is not None:
+            es = [DvfSN(s.name, s.flg0, s.flg1, s.flg2, s.address, s.flg3,
+                        s.flg4) for s in es if (s.flg2 & 0xf) != 1]
+        if (dvf.mcutype & 0xF0) == Dvf.MCU_78K0R:
+            # FIXME: ugly hack
+            sn = [DvfSN(s.name, s.flg0, s.flg1, s.flg2,
+                        s.address | 0xf0000, s.flg3, s.flg4) for s in sn]
+            if es is not None:
+                es = [DvfSN(s.name, s.flg0, s.flg1, s.flg2,
+                            s.address | 0xf0000, s.flg3, s.flg4) for s in es]
+
+        output.write('\n'.join(comment_lines(args.out_format, [
+            "%s automatically generated by dvf2h.py" % \
+                ("Linker script" if args.out_format == 'ld-script'
+                                 else "Register include file"),
+            "For the %s (%s)" % (dvf.chipname_sym, dvf.chipname_disp),
+            "",
+            "Original: DVF version %s from %s" % (md.version, md.timestamp),
+            dvf.copyright_notice
+        ])))
+        output.write("\n\n")
+
+        if args.out_format == 'ld-script':
+            output.write(format_linker_script(mi))
+        else:
+            rpf = "REG_" if reg_prefix else ""
+            vpf = "VEC_" if reg_prefix else ""
+            output.write(format_defines(args.out_format, rpf, vpf, sn, []))
+            output.write("\n\n\n")
+            if es is not None:
+                output.write(format_defines(args.out_format, rpf, vpf, es, []))
+                output.write("\n\n\n")
+            output.write(format_defines(args.out_format, rpf, vpf, [], vn))
+            output.write("\n\n\n")
+
+    finally:
+        if args.output is not None:
+            output.close()
+

dvf/dvfparse.py

@@ -0,0 +1,301 @@
+
+from __future__ import annotations
+import argparse, os, os.path, struct, sys
+from typing import *
+
+def name2int(s: Union[str, bytes]):
+    if isinstance(s, str): return ord(s[0]) + ord(s[1])*256
+    else:
+        assert isinstance(s, bytes)
+        return s[0] + s[1]*256
+
+# supported section types:
+# * SN: SFRs
+# * VN: vectors
+# * MI: memory layout
+# * ES: 2nd SFR
+# * MD: probably "metadata", version & timestam
+#
+# can be parsed:
+# * SC: something with SFRs
+# * FL: flash stuff
+# * OD: debug stuff
+#
+# I know about their existence:
+# * CG, XD, CT, QB, EV, EM, BS, UE, PC
+#
+# I have no clue:
+# * VI, MC, SS, IO, PT, MK, MT, ID, BI, LD, PL, PI, SI, CI, SX, PE, PS
+
+class DvfSN(NamedTuple):
+    name: str
+    flg0: int
+    flg1: int
+    flg2: int
+    address: int
+    flg3: int
+    flg4: int
+
+    def parse(b: bytes) -> 'DvfSN':
+        name, flg0, flg1, flg2, addr, nul1, flg3, flg4, nulls = struct.unpack(
+            '<16sH2HHH2H10s', b)
+        name = name.rstrip(b'\0').decode('utf-8')
+        assert nul1 == 0
+        assert all(n == 0 for n in nulls)
+
+        return DvfSN(name, flg0, flg1, flg2, addr, flg3, flg4)
+
+DvfES = DvfSN
+
+class DvfVN(NamedTuple):
+    name: str
+    address: int
+    int_type: int
+    enable_reg_addr: int
+    enable_bit_flag: int
+
+    def parse(b: bytes) -> DvfVN:
+        name, addr, ityp, enrg, enbt = struct.unpack('<16sHHHH', b)
+        name = name.rstrip(b'\0').decode('utf-8')
+
+        return DvfVN(name, addr, ityp, enrg, enbt)
+
+class DvfMI(NamedTuple):
+    ACCESS_R = 1
+    ACCESS_W = 2
+    ACCESS_X = 4
+    ACCESS_RSV = 8 # reserved
+
+    MREG_FLASH     = 0x00
+    MREG_RAM       = 0x01
+    # TODO: rama again? (0x2)
+    # TODO: 0..ffffe, 0..fffff regions (0xb, 0xc)
+    MREG_RESERVED  = 0x0e
+    MREG_CPUREGS   = 0x0f
+    MREG_SFR       = 0x10
+    # TODO ffd0..ffdf (0x11)
+    MREG_VECTOR    = 0x12
+    MREG_CALLT     = 0x13
+    MREG_CALLF     = 0x14
+    MREG_SHORT     = 0x16 # "short direct addressing" in instruction set
+    MREG_SHORT_SFR = 0x18 # above + is SFR
+    MREG_SFR2      = 0x24
+    MREG_ROM_ENTRY = 0x25
+    MREG_MIRROR    = 0x26
+    MREG_PRIVRAM   = 0x27
+    MREG_ROM       = 0x28
+
+    addr_start: int
+    addr_end: int
+    mreg_type: int
+    access: int
+
+    def parse(b: bytes) -> DvfMI:
+        start, end, typ, acc, zeros = struct.unpack('<IIHHI', b)
+        assert zeros == 0
+        return DvfMI(start, end, typ, acc)
+
+class DvfMD(NamedTuple):
+    version: str # file format version?
+    timestamp: str # also includes author
+
+    def parse(b: bytes) -> DvfMD:
+        assert len(b) >= 40
+        ver, ts = b[:8], b[8:]
+        ver = ver.rstrip(b'\0').decode('utf-8')
+        ts  = ts.rstrip( b'\0').decode('utf-8')
+        return DvfMD(ver, ts)
+
+class DvfSC(NamedTuple):
+    name: str
+    addr1: int
+    addr2: int
+
+    def parse(b: bytes) -> DvfSC:
+        name, a1, a2 = struct.unpack('<16sII', b)
+        name = name.rstrip(b'\0').decode('utf-8')
+        return DvfSC(name, a1, a2)
+
+class DvfFL(NamedTuple):
+    FL_TYPE_NBLOCKS = 1
+    FL_TYPE_BLOCK   = 2
+
+    fl_type: int
+    index: int
+    address: int
+    extra: int
+
+    def parse(b: bytes) -> DvfFL:
+        typ, ind, addr, ex = struct.unpack('<HHHH', b)
+        return DvfFL(typ, ind, addr, ex)
+
+class DvfOD(NamedTuple):
+    index: int
+    subindex: int
+    value: int
+
+    def parse(b: bytes) -> DvfOD:
+        ind, subind, val = struct.unpack('<HHI', b)
+        return DvfOD(ind, subind, val)
+
+class DvfSect(NamedTuple):
+    SN = name2int("SN")
+    VN = name2int("VN")
+    MI = name2int("MI")
+    ES = name2int("ES")
+    MD = name2int("MD")
+
+    SC = name2int("SC")
+    FL = name2int("FL")
+    OD = name2int("OD")
+
+    def ENTSIZE(dvftype: int) -> Dict[int, int]:
+        return {
+            name2int(b'SN'):40,
+            name2int(b'VN'):24,
+            name2int(b'MI'):16,
+            name2int(b'CG'):80,
+            name2int(b'ES'):40, # 78K0R only?
+            name2int(b'XD'):12, # 78K0R only?
+            name2int(b'CT'):96,
+            name2int(b'SC'):24,
+            name2int(b'QB'):136 if (dvftype & 0xF0) == 0x10 else 108,
+            name2int(b'EV'):8,
+            name2int(b'FL'):8,
+            name2int(b'OD'):8,
+            name2int(b'AF'):16,
+            name2int(b'SD'):16,
+            name2int(b'EM'):12,
+            name2int(b'MD'):48,
+            name2int(b'BS'):16, # 78K0 only?
+            name2int(b'UE'):12, # RL78 only?
+            name2int(b'PC'):20, # idk
+        }
+
+    TYPES = {
+        name2int('SN'): DvfSN,
+        name2int('ES'): DvfES,
+        name2int('VN'): DvfVN,
+        name2int('MI'): DvfMI,
+        name2int('MD'): DvfMD,
+        name2int('SC'): DvfSC,
+        name2int('FL'): DvfFL,
+        name2int('OD'): DvfOD
+    }
+
+    name: int
+    nentries: int
+    offset: int
+
+class Dvf(NamedTuple):
+    MAGIC = 0x1a
+
+    # TODO: add support for V850/RH850, RX
+    MCU_78K0    =    0
+    MCU_78K0R   = 0x10
+    MCU_RL78x1y = 0x20
+    MCU_RL78x2y = 0x30
+
+    IDK_KNOWN_78K0 = {
+        b'\0\01\x02!\x02\0\0H\x11', # Kx2
+        b'\0\01\x02\0\x01\0\0H\x11', # Kx2-A, uPD78F8039
+        b'\0\01\x02\x01\x01\0\0H\x11', # Fx2
+        b'\0\01\x02\x01\x02\0\0H\x11', # Kx2-L
+        b'\0\01\x02\x10\x01\0\0H\x11', # misc (Ix2, KC2-A, uPD78F0730)
+        b'\0\01\x02\x11\x01\0\0H\x11', # FC2
+        b'\0\01\x02\x12\x01\0\0H\x11', # Lx3
+        b'\0\02\x02\0\x01\0\0H\x11', # misc (uPD78F8077, Dx2, Kx2-C, uPD78F8039)
+        b'\0\02\x02\x02\x01\0\0H\x11', # Fx2-L
+        b'\0\02\x02\x10\x01\0\0H\x11', # Lx3-M
+    }
+    IDK_KNOWN_78K0R = {
+        b'\x0b\0\0\x03 \x02\0\0H\x11', # KC3-L
+        b'\x0b\0\0\x03!\x01\0\0H\x11', # Lx3
+        b'\x0b\0\0\x03\0\x01\0\0H\x11', # misc (Kx3-L(USB), Hx3, LG3-M, uPD78F80xx)
+        b'\x0b\0\0\x03\0\x02\0\0H\x11', # Kx3-L, Ix3
+        b'\x0b\0\0\x03\0\x03\0\0H\x11', # Kx3
+        b'\x0b\0\0\x03\x01\x01\0\0H\x11', # Fx3, Kx3-C
+        b'\x0b\0\0\x03\x01\x02\0\0H\x11', # Kx3-L
+        b'\x0b\0\0\x03\x10\x01\0\0H\x11', # Lx3, Kx3-A
+        b'\x0b\0\0\x03\x10\x02\0\0H\x11', # Kx3-L
+        b'\x0b\0\0\x03\x11\x01\0\0H\x11', # KF3-C
+    }
+    IDK_KNOWN_RL78x1y = {
+        b'\x0c\0\0\x03 \x01\0\0H\x11', # G11, G1D
+        b'\x0c\0\0\x03!\x01\0\0H\x11', # I1A, L1C
+        b'\x0c\0\0\x030\x02\0\0H\x11', # G14
+        b'\x0c\0\0\x03\0\x01\0\0H\x11', # misc (G13A, G1H, F15, I1E, H1D, I1C)
+        b'\x0c\0\0\x03\x01\x01\0\0H\x11', # misc (L1A, I1D, G1G, G1A, L13, I1C, D1A, G1F, F1A)
+        b'\x0c\0\0\x03\x01\x02\0\0H\x11', # G1E
+        b'\x0c\0\0\x03\x02\x01\0\0H\x11', # I1B, F12
+        b'\x0c\0\0\x03\x03\x01\0\0H\x11', # G10, L12
+        b'\x0c\0\0\x03\x10\x01\0\0H\x11', # misc (G1N, F1E, G1P, G1C, D1A, G1M, G1K)
+        b'\x0c\0\0\x03\x11\x01\0\0H\x11', # L1C, F13, F14
+        b'\x0c\0\0\x03\x12\x01\0\0H\x11', # G12
+        b'\x0c\0\0\x03\x14\x01\0\0H\x11', # G13
+    }
+    IDK_KNOWN_RL78x2y = {
+        b'\x0c\0 \x03\0\x01\0\0H\x11', # F24
+        b'\x0c\0\0\x030\x02\0\0H\x11', # FGIC
+        b'\x0c\0\0\x03\x02\x01\0\0H\x11', # FGIC
+        b'\x0c\0\x10\x03\x10\x01\0\0H\x11', # G23
+    }
+
+    magic: int
+    copyright_notice: str
+    idk: bytes
+    chipname_disp: str
+    chipname_sym: str
+
+    mcutype: int
+    sections: Dict[int, Any]
+    sections_raw: List[DvfSect]
+
+    @staticmethod
+    def parse(data: bytes, mcutype = None) -> Dvf:
+        magic, crnot, idk, name_disp, name_sym, nsect = struct.unpack(
+            '<B35s16s16s8sI', data[:80])
+        assert magic == Dvf.MAGIC
+        assert idk[14:16] in { b"\x02\0", b"\x03\0", b"\x04\0" }, str(idk)
+        crnot = crnot.rstrip(b'\0').decode('utf-8')
+        name_disp = name_disp.rstrip(b'\0').decode('utf-8')
+        name_sym  = name_sym.rstrip(b'\0').decode('utf-8')
+
+        #mcutype = None
+        if mcutype is None:
+            idkfix = idk[0:10]
+            if idkfix in Dvf.IDK_KNOWN_78K0: mcutype = Dvf.MCU_78K0
+            elif idkfix in Dvf.IDK_KNOWN_78K0R: mcutype = Dvf.MCU_78K0R
+            elif idkfix in Dvf.IDK_KNOWN_RL78x1y: mcutype = Dvf.MCU_RL78x1y
+            elif idkfix in Dvf.IDK_KNOWN_RL78x2y: mcutype = Dvf.MCU_RL78x2y
+            else: assert False, ("unknown MCU type", idk)
+
+        sects = {}
+        sraw = []
+        for i in range(nsect):
+            shblob = data[(80+i*12):(80+(i+1)*12)]
+            name, nentries, offset, zeros = struct.unpack('<HHH6s', shblob)
+            if not all(z == 0 for z in zeros):
+                continue # idk
+
+            sraw.append(DvfSect(name, nentries, offset))
+            if nentries == 0: continue
+
+            #print("%c%c" % (chr(name&0xff),chr(name>>8)))
+            entsize = DvfSect.ENTSIZE(mcutype)[name]
+
+            sdblob = data[offset:(offset+entsize*nentries)]
+
+            styp = DvfSect.TYPES.get(name)
+            if styp is None: sects[name] = sdblob
+            else:
+                l = []
+                for j in range(nentries):
+                    sdiblob = sdblob[(j*entsize):((j+1)*entsize)]
+                    #if name == name2int("MD"):
+                    #    print("esz",entsize,"j",j,"blob",sdiblob)
+                    l.append(DvfSect.TYPES[name].parse(sdiblob))
+                sects[name] = l
+
+        return Dvf(magic, crnot, idk, name_disp, name_sym, mcutype, sects, sraw)
+

dvf/xmlmeta.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+
+import os, struct, sys
+import json, xml, xml.etree
+import xml.etree.ElementTree as ET
+from typing import *
+
+import dvfparse
+from dvfparse import Dvf, DvfMI, DvfVN, DvfSN
+
+
+class DRec(NamedTuple):
+    path: str
+    magic: bytes
+    name: str
+    bromsize: int
+
+class XRec(NamedTuple):
+    #path: str # is mostly prodname anyway
+    nickname: str # "product series"
+    prodname: str
+    proddispname: str
+    dvfname: str
+    romsz: int
+    ramsz: int
+    npins: int
+    dvf: DRec
+
+
+def get_common_masks(l: List[bytes]) -> (bytes, bytes):
+    #return (l[:10], l[:10])
+    bits = 16*8
+    mask = (1<<bits)-1
+    negb = mask
+    posb = mask
+
+    for b in l:
+        i = int.from_bytes(b, byteorder='big')
+        posb =  posb &  i
+        negb = (negb & ~i) & mask
+
+    negb = (~negb & mask).to_bytes(16, byteorder='big')
+    posb = posb.to_bytes(16, byteorder='big')
+    #negb = ''.join("%02x"%x for x in negb)
+    #posb = ''.join("%02x"%x for x in posb)
+    return (negb, posb)
+
+
+def parsedvf(path: str, is78k: bool) -> DRec:
+    typ = Dvf.MCU_RL78G13
+    if is78k: typ = Dvf.MCU_78K0R if "78K0R" in path else Dvf.MCU_78K0
+
+    data = None
+    with open(path, "rb") as f: data = f.read()
+    magic = data[0x24:0x34]
+    name = data[0x34:0x44]
+    name = name.strip(b'\0').decode('utf-8')
+
+    #print(filename)
+    dvf = Dvf.parse(data, typ)
+    mi = dvf.sections[dvfparse.name2int('MI')]
+
+    brom_size = None
+    for m in mi:
+        if m.mreg_type == 0x28: # bootrom
+            brom_size = m.addr_end - m.addr_start + 1
+            break
+
+    #mag1 = ''.join("%02x"%x for x in magic[:10])
+    #mag2 = ''.join("%02x"%x for x in magic[10:])
+    #d2 = stuff.setdefault(mag1, {})
+    #d3 = d2.setdefault(name[:6], {})
+    #d3.setdefault(mag2, []).append((name, brom_size))
+
+    #magic = ''.join("%02x"%x for x in magic)
+    return DRec(os.path.split(path)[-1], magic, name, brom_size)
+
+
+def parsexml(basepath: str, is78k: bool) -> Sequence[XRec]:
+    xmlfile = basepath + "_Productlist.xml"
+
+    xmlroot = ET.parse(xmlfile).getroot()
+
+    curnick, curprod, curproddisp, curdvf, currom, curram, curpins = \
+        None, None, None, None, None, None, None
+    for node in xmlroot[1]:
+        if node.tag == 'Nickname':
+            if curnick is not None:
+                ext = "78K" if is78k else "DVF"
+                if is78k:
+                    if "78K0R" in curnick:
+                        root2 = basepath + "0R"
+                    elif "78K0" in curnick:
+                        root2 = basepath + "0"
+                    else: assert False, curnick
+                else: root2 = basepath
+                dvfdir = root2 + "/Devicefile/"
+
+                assert curprod
+                filename = "%s_common.xml" % curprod
+                filepath = "%s/%s" % (root2, filename)
+                if curdvf is not None:
+                    pass#assert curdvf in {"DFR5F100", "DF780495", "DF780547", "DF780893", "DF780893", "DF780893__V110", "DF781009", "DF781009__V101"}, curdvf
+                #if curdvf is None or curdvf == 'DFR5F100':
+                #try:
+                xml2 = ET.parse(filepath).getroot()
+                dvfi = xml2.find("DEVICEFILEINFORMATION")
+                folder = dvfi.find("DeviceFileInformationFolderName").text
+                filen = dvfi.find("DeviceFileInformationFileName").text
+                folder = folder or "Devicefile"
+
+                curdvf = "%s/%s/%s" % (root2, folder, filen)
+                #print("curdvf:%s"% curdvf)
+                #except Exception: curdvf = None
+                if curdvf is None:
+                    #print("curprod: %s"%curprod)
+                    curdvf = "%s/D%s.%s" % (dvfdir, curprod, ext) # shrug
+                #else:
+                #    if is78k:
+                #        if "78K0R" in curnick: pass
+                #        elif "78K0" in curnick: curdvf = curdvf.replace("DF78", "DF")
+                #    print("curdvf k:%s"% curdvf, locals())
+                #    curdvf = "%s/%s.%s" % (dvfdir, curdvf, ext)
+                yield XRec(curnick, curprod, curproddisp,
+                           os.path.split(curdvf)[-1], currom, curram, curpins,
+                           parsedvf(curdvf, is78k))
+                curnick, curprod, curproddisp, curdvf, currom, curram, curpins = \
+                    None, None, None, None, None, None, None
+                curnick = node.text
+            else: curnick = node.text
+        if node.tag == 'ProductName': curprod = node.text
+        if node.tag == 'ROMSize': currom = int(node.text or "0")
+        if node.tag == 'RAMSize': curram = int(node.text or "0")
+        if node.tag == 'DeviceFileProductName': curdvf = node.text
+        if node.tag == 'ProductDisplayName': curproddisp = node.text
+        if node.tag == 'PinCount': curpins = int(node.text or "0")
+
+
+if __name__ == '__main__':
+    root_rl = sys.argv[1]
+    xmls_rl = list(parsexml(root_rl, False))
+    xmls_rl.append(XRec("RL78/L1C", "R5F111PJ", "R5F111PJ", None, 256, 16*1024, 100,
+                        parsedvf(root_rl+"/Devicefile/DR5F111PJ.DVF", False)))
+    xmld_rl = {}
+    for x in xmls_rl:
+        l = xmld_rl.setdefault(x.nickname, []).append(x)
+
+    root_k = sys.argv[2]
+    xmls_k = list(parsexml(root_k, True))
+    xmls_k.append(XRec('78K0R/UPD78F8069', "UPD78F806964", "UPD78F8069_64", None, 128+16, 8*1024, 64,
+                       parsedvf(root_k+"0R/Devicefile/DF806964.78K", True)))
+    xmld_k = {}
+    for x in xmls_k:
+        l = xmld_k.setdefault(x.nickname, []).append(x)
+
+    #print(json.dumps(xmld_rl, sort_keys=True, indent=4))
+    #print(json.dumps(xmld_k , sort_keys=True, indent=4))
+
+    # verify that all DVF files on disk are used!
+
+    def dictl(ps, useset: bool=False):
+        if useset:
+            d = {}
+            for x, y in ps: d.setdefault(x, set()).add(y)
+            return dict((x, list(y)) for x, y in d.items())
+        else:
+            d = {}
+            for x, y in ps: d.setdefault(x, []).append(y)
+            return d
+
+    """
+    unk_rl, unk_k0, unk_k0r = [], [], []
+    for entry in os.scandir(root_rl+"/Devicefile"):
+        if not any(any(entry.name == y.dvf.path for y in x) for x in xmld_rl.values()):
+            unk_rl.append(entry.name)
+    for entry in os.scandir(root_k+"0/Devicefile"):
+        if not any(any(entry.name == y.dvf.path for y in x) for x in xmld_k.values()):
+            unk_k0.append(entry.name)
+    for entry in os.scandir(root_k+"0R/Devicefile"):
+        if not any(any(entry.name == y.dvf.path for y in x) for x in xmld_k.values()):
+            unk_k0r.append(entry.name)
+    print(unk_rl)
+    print(unk_k0)
+    print(unk_k0r)
+    """
+
+    """
+    maskd = {}
+    for k, l in xmld_k.items():
+        mask = get_common_masks([x.dvf.magic for x in l])
+        maskd.setdefault(str(mask[1][:10]), []).append(k)
+    for k, l in xmld_rl.items():
+        mask = get_common_masks([x.dvf.magic for x in l])
+        maskd.setdefault(str(mask[1][:10]), []).append(k)
+    print(json.dumps(maskd, sort_keys=True, indent=4))
+    """
+
+    """
+    print()
+
+    k78k0_bytes  = sum([[l.dvf.magic for l in v] for k, v in xmld_k.items() if "78K0R" not in k], start=[])
+    k78k0r_bytes = sum([[l.dvf.magic for l in v] for k, v in xmld_k.items() if "78K0R"     in k], start=[])
+    k78k_bytes   = sum([[l.dvf.magic for l in v] for k, v in xmld_k.items()], start=[])
+    rl78s1_bytes = sum([[l.dvf.magic for l in v] for k, v in xmld_rl.items() if all(x not in k for x in {"G23","F24","FGIC"})], start=[])
+    rl78s2_bytes = sum([[l.dvf.magic for l in v] for k, v in xmld_rl.items() if any(x     in k for x in {"G23","F24","FGIC"})], start=[])
+    rl78_bytes   = sum([[l.dvf.magic for l in v] for k, v in xmld_rl.items()], start=[])
+
+    print("78K0", get_common_masks(k78k0_bytes))
+    print("78K0R", get_common_masks(k78k0r_bytes))
+    print("78K", get_common_masks(k78k_bytes))
+    print("RL78/x1y", get_common_masks(rl78s1_bytes))
+    print("RL78/x2y", get_common_masks(rl78s2_bytes))
+    print("RL78", get_common_masks(rl78_bytes))
+    """
+
+    k78k0_bytes  = dictl(sum([[(repr(l.dvf.magic[:10]), l.nickname) for l in v] for k, v in xmld_k.items() if "78K0R" not in k], start=[]), True)
+    k78k0r_bytes = dictl(sum([[(repr(l.dvf.magic[:10]), l.nickname) for l in v] for k, v in xmld_k.items() if "78K0R"     in k], start=[]), True)
+    k78k_bytes   = dictl(sum([[(repr(l.dvf.magic[:10]), l.nickname) for l in v] for k, v in xmld_k.items()], start=[]), True)
+    rl78s1_bytes = dictl(sum([[(repr(l.dvf.magic[:10]), l.nickname) for l in v] for k, v in xmld_rl.items() if all(x not in k for x in {"G23","F24","FGIC"})], start=[]), True)
+    rl78s2_bytes = dictl(sum([[(repr(l.dvf.magic[:10]), l.nickname) for l in v] for k, v in xmld_rl.items() if any(x     in k for x in {"G23","F24","FGIC"})], start=[]), True)
+    rl78_bytes   = dictl(sum([[(repr(l.dvf.magic[:10]), l.nickname) for l in v] for k, v in xmld_rl.items()], start=[]), True)
+
+    print("78K0", json.dumps(k78k0_bytes, sort_keys=True, indent=4))
+    print("78K0R", json.dumps(k78k0r_bytes, sort_keys=True, indent=4))
+    #print("78K", json.dumps(k78k_bytes, sort_keys=True, indent=4))
+    print("RL78/x1y", json.dumps(rl78s1_bytes, sort_keys=True, indent=4))
+    print("RL78/x2y", json.dumps(rl78s2_bytes, sort_keys=True, indent=4))
+    #print("RL78", json.dumps(rl78_bytes, sort_keys=True, indent=4))
+

svd/svd.py

@@ -0,0 +1,207 @@
+
+from typing import *
+
+
+SvdEndian = Literal["little", "big", "selectable", "other"]
+SvdSauAccess = Literal["n", "c", "s"]
+SvdProtection = Literal["s", "n", "p"]
+SvdAddressUsage = Literal["registers", "buffer", "reserved"]
+SvdUsage = Literal["read", "write", "read-write"]
+SvdDataType = Literal[
+    "uint8_t","uint16_t","uint32_t","uint64_t",
+    "int8_t","int16_t","int32_t","int64_t",
+    "uint8_t*","uint16_t*","uint32_t*","uint64_t*",
+    "int8_t*","int16_t*","int32_t*","int64_t*"
+]
+SvdModifiedWriteValues = Literal[
+    "oneToClear", "oneToSet", "oneToToggle",
+    "zeroToSet", "zeroToClear", "zeroToToggle",
+    "clear", "set", "modify"
+]
+SvdReadAction = Literal["clear", "set", "modify", "modifyExternal"]
+SvdAccess = Literal["read-only", "write-only", "read-write", "writeOnce", "read-writeOnce"]
+
+
+class SvdSauRegion(NamedTuple):
+    enabled: bool
+    name: str
+    base: int
+    limit: int
+    access: SvdSauAccess
+
+
+class SvdSauRegionsConfig(NamedTuple):
+    enabled: bool
+    protectionWhenDisabled: SvdSauAccess
+    regions: Sequence[SvdSauRegion]
+
+
+class SvdCpu(NamedTuple):
+    name: str # explicitly not restricting to standard CMSIS names, because we'll be doing non-ARM stuff anyway
+    revision: Tuple[int, int]
+    endian: SvdEndian
+    mpuPresent: bool
+    fpuPresent: bool
+    fpuDP: bool
+    dspPresent: bool
+    icachePresent: bool
+    dcachePresent: bool
+    itcmPresent: bool
+    dtcmPresent: bool
+    vtorPresent: bool
+    nvicPrioBits: int
+    vendorSystickConfig: bool
+    deviceNumInterrupts: int
+    sauNumRegions: int
+    sauRegionsConfig: SvdSauRegionsConfig
+
+
+class SvdRegisterProperties(NamedTuple):
+    size: int
+    access: SvdAccess
+    protection: SvdProtection
+    resetValue: int
+    resetMask: int
+
+
+class SvdEnumeratedValue(NamedTuple):
+    name: str
+    description: str
+    value: int
+    isDefault: bool
+
+
+class SvdEnumeratedValues(NamedTuple):
+    derivedFrom: str
+    name: str
+    usage: SvdUsage
+    enumeratedValue: Sequence[SvdEnumeratedValue]
+
+
+class SvdDimArrayIndex(NamedTuple):
+    headerEnumName: str
+    enumeratedValue: SvdEnumeratedValue
+
+
+class SvdDimElement(NamedTuple):
+    dim: int
+    dimIncrement: int
+    dimIndex: int
+    dimName: str
+    dimArrayIndex: SvdDimArrayIndex
+
+
+class SvdAddressBlock(NamedTuple):
+    offset: int
+    size: int
+    usage: SvdAddressUsage
+    protection: SvdProtection
+
+
+class SvdInterrupt(NamedTuple):
+    name: str
+    description: str
+    value: int
+
+
+class SvdCluster(NamedTuple):
+    derivedFrom: str
+    dimElement: SvdDimElement
+    name: str
+    description: str
+    alternateCluster: str
+    headerStructName: str
+    addressOffset: int
+    registerProperties: SvdRegisterProperties
+    register: Sequence[Any]
+    cluster: Sequence[Any]
+
+
+class SvdWriteConstraintRange(NamedTuple):
+    min: int
+    max: int
+
+class SvdWriteConstraint(NamedTuple):
+    writeAsRead: bool
+    useEnumeratedValues: bool
+    range: SvdWriteConstraintRange
+
+
+class SvdBitRangeLsbMsb(NamedTuple):
+    lsb: int
+    msb: int
+class SvdBitRangeOffsetWidth(NamedTuple):
+    bitOffset: int
+    bitWidth: int
+
+
+SvdBitRangePattern = str
+
+
+class SvdField(NamedTuple):
+    derivedFrom: str
+    dimElement: SvdDimElement
+    name: str
+    description: str
+    bitRange: Union[SvdBitRangeLsbMsb, SvdBitRangeOffsetWidth, SvdBitRangePattern]
+    access: SvdAccess
+    modifiedWriteValues: SvdModifiedWriteValues
+    writeConstraint: SvdWriteConstraint
+    readAction: SvdReadAction
+    enumeratedValues: SvdEnumeratedValues
+
+
+class SvdRegister(NamedTuple):
+    derivedFrom: str
+    dimElement: SvdDimElement
+    name: str
+    displayName: str
+    description: str
+    alternateGroup: str
+    alternateRegister: str
+    addressOffset: int
+    registerProperties: SvdRegisterProperties
+    dataType: SvdDataType
+    modifiedWriteValues: SvdModifiedWriteValues
+    writeConstraint: SvdWriteConstraint
+    readAction: SvdReadAction
+    fields: Sequence[SvdField]
+
+
+class SvdPeripheral(NamedTuple):
+    derivedFrom: str
+    name: str
+    version: str
+    description: str
+    alternatePeripheral: str
+    groupName: str
+    prependToName: str
+    appendToName: str
+    headerStructName: str
+    disableCondition: str
+    baseAddress: int
+    registerProperties: SvdRegisterProperties
+    addressBlock: SvdAddressBlock
+    interrupt: SvdInterrupt
+    registers: Sequence[Union[SvdRegister, SvdCluster]]
+
+
+class SvdDevice(NamedTuple):
+    vendor: str
+    vendorID: str
+    name: str
+    series: str
+    version: str
+    description: str
+    licenseText: str
+    cpu: SvdCpu
+    headerSystemFilename: str
+    headerDefinitionsPrefix: str
+    addressUnitBits: int
+    width: int
+    registerProperties: SvdRegisterProperties
+    peripherals: Sequence[SvdPeripheral]
+    vendorExtensions: Any
+
+    # TODO: functions for expanding registerproperties, dimelements and derivedfrom?
+

svd/svdparse.py

@@ -0,0 +1,194 @@
+
+import xml, xml.etree
+import xml.etree.ElementTree as ET
+import typing
+
+from svd import *
+from typing import get_type_hints
+
+
+def parse_one(ch, t):
+    k, v = None, None
+    if isinstance(ch, ET.Element):
+        k, v = ch.tag, ch
+    else:
+        k, v = ch
+
+    field = get_type_hints(t)[k]
+    #print("%s -> %s [%s]" %(k, v, repr(field)))
+
+    known_types = {
+        SvdDevice: parse_device,
+        SvdCpu: parse_cpu,
+        SvdPeripheral: parse_peripheral,
+        SvdRegister: parse_register,
+        SvdField: parse_field,
+        SvdEnumeratedValues: parse_enumeratedValues,
+        SvdCluster: parse_cluster,
+        SvdSauRegionsConfig: parse_sauRegionsConfig,
+        SvdWriteConstraint: parse_writeConstraint,
+        SvdEnumeratedValue: parse_enumeratedValue,
+        SvdWriteConstraintRange: parse_writeConstraintRange,
+        SvdInterrupt: parse_interrupt,
+        SvdAddressBlock: parse_addressBlock,
+        SvdSauRegion: parse_sauRegion,
+    }
+
+    vv = known_types.get(field)
+    if vv:
+        return vv(v)
+    elif isinstance(ch, ET.Element):
+        v = ch.text
+
+    if field == int:
+        return int(v, 0)
+    elif field == bool:
+        assert v in {'true','false','1','0'}, ("invalid boolean value %s"%v)
+        return v in {'true','1'}
+    elif field == str or field.__origin__ == typing.Literal:
+        return v
+    elif field.__origin__ == tuple:
+        if k == 'revision':
+            return tuple(int(x) for x in v[1:].split('p'))
+        else:
+            assert False, ("unknown Tuple type %s for field %s" % (field, k))
+    elif field.__origin__ == typing.Union:
+        return v  # good enough for now
+    else:
+        assert False, ("unknown type %s for field %s" % (field, k))
+
+
+REG_PROP_NAMES = {'size', 'access', 'protection', 'resetValue', 'resetMask'}
+DIM_ELEM_NAMES = {'dim','dimIncrement','dimIndex','dimName','dimArrayIndex'}
+
+def try_parse_registerProperties(xd, part, dimElem) -> SvdRegisterProperties:
+    d = dict((n, None) for n in SvdRegisterProperties._fields)
+    for ch in xd:
+        if ch.tag in d:
+            d[ch.tag] = parse_one(ch, SvdRegisterProperties)
+        else:
+            assert ch.tag in part._fields or (dimElem and ch.tag in DIM_ELEM_NAMES), \
+                ("unknown child %s" % repr(ch))
+
+    return SvdRegisterProperties(**d)
+
+def try_parse_dimElement(xd, part, regProp) -> SvdDimElement:
+    d = dict((n, None) for n in SvdDimElement._fields)
+    for ch in xd:
+        if ch.tag == 'dimArrayIndex':
+            d[ch.tag] = parse_dimArrayIndex(ch)
+        elif ch.tag in d:
+            d[ch.tag] = parse_one(ch, SvdDimElement)
+        else:
+            assert ch.tag in part._fields or (regProp and ch.tag in REG_PROP_NAMES), \
+                ("unknown child %s" % repr(ch))
+
+    return SvdDimElement(**d)
+
+
+def parse_generic(xd, T: type, tag: str,
+                  hasRegProps: bool = False, hasAttrib: bool = False,
+                  hasDimElem: bool = False, specials: Dict[str, Any] = None,
+                  lists: Dict[str, Any] = None):
+    assert xd.tag == tag, ("expected %s, got %s" % (tag, xd.tag))
+    d = dict((n, None) for n in T._fields)
+
+    if hasRegProps:
+        d['registerProperties'] = try_parse_registerProperties(xd, T, hasDimElem)
+    if hasDimElem:
+        d['dimElement'] = try_parse_dimElement(xd, T, hasRegProps)
+    if hasAttrib == True:
+        for k, v in xd.attrib.items():
+            if k in d:
+                d[k] = v
+            else:
+                assert False, ("unknown attrib %s" % repr((k, v)))
+    elif hasAttrib == False:
+        assert len(xd.attrib) == 0
+
+    for ch in xd:
+        if lists and ch.tag in lists:
+            if d[ch.tag] == None: d[ch.tag] = []
+            d[ch.tag].append(lists[ch.tag](ch))
+        elif specials and ch.tag in specials:
+            d[ch.tag] = specials[ch.tag](ch)
+        elif ch.tag in d:
+            d[ch.tag] = parse_one(ch, T)
+        else:
+            assert (hasRegProps and ch.tag in REG_PROP_NAMES) \
+                    or (hasDimElem and ch.tag in DIM_ELEM_NAMES), \
+                ("unknown child %s" % repr(ch))
+
+    return T(**d)
+
+
+def parse_sauRegion(xd): return parse_generic(xd, SvdSauRegion, 'region')
+def parse_addressBlock(xd): return parse_generic(xd, SvdAddressBlock, 'addressBlock')
+def parse_interrupt(xd): return parse_generic(xd, SvdInterrupt, 'interrupt')
+def parse_writeConstraintRange(xd): return parse_generic(xd, SvdWriteConstraintRange, 'range')
+def parse_enumeratedValue(xd): return parse_generic(xd, SvdEnumeratedValue, 'enumeratedValue')
+def parse_writeConstraint(xd):
+    return parse_writeConstraint(xd, SvdWriteConstraint, 'writeConstraint')
+def parse_dimArrayIndex(xd): return parse_generic(xd, SvdDimArrayIndex, 'dimArrayIndex')
+
+
+def parse_sauRegionsConfig(xd):
+    return parse_generic(xd, SvdSauRegionsConfig, 'sauRegionsConfig', lists={
+        'regions': parse_sauRegion,
+    })
+
+
+def parse_cluster(xd) -> SvdCluster:
+    return parse_generic(xd, SvdCluster, 'cluster', True, True, True, lists={
+        'register': parse_register,
+        'cluster': parse_cluster,
+    })
+
+
+def parse_enumeratedValues(xd) -> SvdEnumeratedValues:
+    return parse_generic(xd, SvdEnumeratedValues, 'enumeratedValues', hasAttrib=True, lists={
+        'enumeratedValue': parse_enumeratedValue,
+    })
+
+
+def parse_cpu(xd): return parse_generic(xd, SvdCpu, 'cpu')
+def parse_field(xd): return parse_generic(xd, SvdField, 'field', False, True, True)
+
+
+def parse_register(xd) -> SvdRegister:
+    return parse_generic(xd, SvdRegister, 'register', True, True, True, {
+        'writeConstraint': parse_writeConstraint,
+        'fields': (lambda ch: [parse_field(xch) for xch in ch])
+    })
+
+
+def parse_registers(xd) -> Sequence[Union[SvdCluster, SvdRegister]]:
+    rr = []
+    for ch in xd:
+        if ch.tag == 'cluster':
+            rr.append(parse_cluster(ch))
+        elif ch.tag == 'register':
+            rr.append(parse_register(ch))
+        else:
+            assert False, ("expected cluster or register, got %s"%ch)
+
+
+def parse_peripheral(xd) -> SvdPeripheral:
+    return parse_generic(xd, SvdPeripheral, 'peripheral', True, True, False, {
+        'registers': parse_registers, # FIXME
+    })
+
+
+def parse_device(xd) -> SvdDevice:
+    return parse_generic(xd, SvdDevice, 'device', True, 2, False, {
+        'cpu': parse_cpu,
+        'peripherals': (lambda ch: [parse_peripheral(xch) for xch in ch]),
+        'vendorExtensions': (lambda ch: ch)
+    })
+
+
+def parse(path) -> SvdDevice:
+    xmlroot = ET.parse(path).getroot()
+
+    return parse_device(xmlroot)
+