Dynamic syntax: grammar extensions as runtime data
The DSL page above declares its syntax { ... } block as a literal, expanded once before the compiler ever runs — indistinguishable from a macro. This one instead computes its trigger keyword and rules from data at runtime, calls expand_syntax_dsls_with(src, existing_defs) itself to define the grammar extension, keeps only the returned defs list, then makes a SEPARATE, later call — on a string containing only the usage line, no syntax declaration anywhere in it — passing those defs forward. Two independent calls, composing through nothing but an ordinary list value, exactly like passing along any other piece of program state. The assembled result runs through the same tokenize/parse_program/lower_program/run_ir chain that already powers the playground (see self_hosting/examples/playground_main.patlang) — PatLang's compiler has always been callable from PatLang code; this demo is the first to close the loop and use that to synthesize a language construct that couldn't have existed until the program ran. Native transcript only, no "run in browser" button: because this demo calls the compiler pipeline from inside itself, running it live means the playground's own run_ir (a nested interpreter, isolated from the outer compiled binary's functions) has to tokenize/parse/lower the entire ~85KB pipeline-plus-demo bundle a second time, INSIDE that nested interpreter — measured at several orders of magnitude slower than the ~2.5 seconds the same bundle takes through the top-level interpreter directly. The underlying capability is proven correct (this transcript matches the interpreted result exactly, and a compiled-native build agrees too); the live-in-browser path for this specific self-referential case is simply not practical yet — an honest limitation of run_ir's current performance on large inputs, not of the dynamic-syntax feature itself.
PatLang source
# =============================================================================
# Stage 1 self-hosted lexer library (Stage 0 compilable subset).
# Tokens are lists: [type, text, line] with types NUM, IDENT, STR, OP, NL, UNK, EOF.
# =============================================================================
make a function called is_digit_code takes c returns r
return (c >= 48) and (c <= 57)
end
make a function called is_alpha_code takes c returns r
if (c >= 65) and (c <= 90) then
return true
else
if (c >= 97) and (c <= 122) then
return true
else
return c == 95
end
end
end
make a function called is_op_code takes c returns r
if (c == 43) or (c == 45) or (c == 42) or (c == 47) or (c == 37) then
return true
else
if (c == 61) or (c == 60) or (c == 62) or (c == 33) then
return true
else
if (c == 40) or (c == 41) or (c == 91) or (c == 93) or (c == 44) or (c == 46) or (c == 124) or (c == 123) or (c == 125) or (c == 58) or (c == 59) then
return true
else
return false
end
end
end
end
make a function called tokenize takes src returns tokens
let tokens = vec_new()
let h = str_intern(src)
let n = sc_len(h)
let i = 0
let line = 1
while i < n do
let c = sc_code(h, i)
if c == 10 then
vec_push(tokens, ["NL", "", line])
let line = line + 1
let i = i + 1
else
if (c == 32) or (c == 9) or (c == 13) then
let i = i + 1
else
if c == 35 then
while (i < n) and (sc_code(h, i) != 10) do
let i = i + 1
end
else
if is_digit_code(c) then
# sb_new/sb_push/sb_str, NOT `txt = txt + ...` -- string
# concatenation in a loop copies the whole accumulated string
# on every append (PatLang strings are immutable), turning an
# O(n) scan into O(n^2). Numbers/identifiers are usually
# short so this rarely mattered in practice, but STRING
# literals (the branch below) can be arbitrarily long --
# self_hosting/lib/runtime_rs.patlang embeds large chunks of
# literal Rust source as single PatLang string literals,
# tens of thousands of characters each -- so this loop is a
# real, not just theoretical, O(n^2) risk on every compile
# that includes that file. Found via a new compiler warning
# (ir/lowering.rs) added specifically to catch this shape,
# while investigating an unrelated 30+-minute self-compile
# regression in a different file (self_hosting/lib/
# syntax_dsl.patlang) that turned out to have the identical
# anti-pattern.
let txt_b = sb_new()
let dots = 0
let scanning = true
while (i < n) and scanning do
let d = sc_code(h, i)
if is_digit_code(d) then
sb_push(txt_b, sc_char(h, i))
let i = i + 1
else
if (d == 46) and (dots == 0) then
let dots = 1
sb_push(txt_b, sc_char(h, i))
let i = i + 1
else
let scanning = false
end
end
end
vec_push(tokens, ["NUM", sb_str(txt_b), line])
else
if is_alpha_code(c) then
let txt_b = sb_new()
let scanning = true
while (i < n) and scanning do
let d = sc_code(h, i)
if is_alpha_code(d) or is_digit_code(d) then
sb_push(txt_b, sc_char(h, i))
let i = i + 1
else
let scanning = false
end
end
vec_push(tokens, ["IDENT", sb_str(txt_b), line])
else
if c == 34 then
let i = i + 1
let txt_b = sb_new()
let scanning = true
while (i < n) and scanning do
let d = sc_code(h, i)
if d == 34 then
let scanning = false
let i = i + 1
else
if d == 92 then
# escape sequences: n t r quote backslash (unknown kept raw)
let e = sc_code(h, i + 1)
if e == 110 then
sb_push(txt_b, chr(10))
let i = i + 2
else
if e == 116 then
sb_push(txt_b, chr(9))
let i = i + 2
else
if e == 114 then
sb_push(txt_b, chr(13))
let i = i + 2
else
if e == 34 then
sb_push(txt_b, chr(34))
let i = i + 2
else
if e == 92 then
sb_push(txt_b, chr(92))
let i = i + 2
else
sb_push(txt_b, sc_char(h, i))
let i = i + 1
end
end
end
end
end
else
sb_push(txt_b, sc_char(h, i))
let i = i + 1
end
end
end
vec_push(tokens, ["STR", sb_str(txt_b), line])
else
if is_op_code(c) then
let txt = sc_char(h, i)
let first = c
let i = i + 1
if i < n then
let d = sc_code(h, i)
if (d == 61) and ((first == 61) or (first == 33) or (first == 60) or (first == 62)) then
let txt = txt + sc_char(h, i)
let i = i + 1
else
# ':-' -- rule turnstile
if (d == 45) and (first == 58) then
let txt = txt + sc_char(h, i)
let i = i + 1
end
end
end
vec_push(tokens, ["OP", txt, line])
else
vec_push(tokens, ["UNK", sc_char(h, i), line])
let i = i + 1
end
end
end
end
end
end
end
end
vec_push(tokens, ["EOF", "", line])
return tokens
end
make a function called print_tokens takes tokens returns done
let n = vec_len(tokens)
let i = 0
while i < n do
let t = vec_get(tokens, i)
print(t[0] + " '" + t[1] + "' @" + t[2])
let i = i + 1
end
return true
end
# =============================================================================
# Stage 1 self-hosted parser library (Stage 0 compilable subset).
# Consumes tokens from lib/lexer.patlang, produces list-shaped AST nodes.
#
# Statements:
# ["Let", name, expr] let NAME = expr
# ["Expr", expr] call statements, e.g. print(x), emit(e, p)
# ["If", cond, [then], [else]] if expr then ... [else ...] end
# ["While", cond, [body]] while expr do ... end
# ["Func", name, [params], [b]] make a function called N takes a, b returns r ... end
# ["Return", expr] return expr
# ["When", event, [body], line] when EVENT do ... end (event handler)
# ["Err", message, line] parse error placeholder (1-indexed line)
#
# Expressions:
# ["Num", text] ["Str", text] ["Bool", "true"/"false"] ["Var", name]
# ["Bin", op, lhs, rhs] op: + - * / % == != < <= > >= and or
# ["Un", op, expr] op: not -
# ["Call", name, [args]]
# ["List", [items]]
# ["Index", obj, idx]
# ["Member", obj, prop]
#
# All parse functions return [node, next_pos] pairs; parse_args and
# parse_stmts_until return [list, next_pos].
# =============================================================================
make a function called tok_is takes t, ty, tx returns r
return (t[0] == ty) and (t[1] == tx)
end
make a function called skip_nl takes toks, pos returns p
# Skips newlines AND a bare ';' -- native parser.rs treats Semicolon as
# a fully optional statement separator right alongside Newline (never
# required; see its "while matches!(self.curr, Token::Semicolon |
# Token::Newline | ...)" loop), so this self-hosted mirror needs to
# tolerate the same already-existing, already-optional token, not
# introduce any new requirement into the grammar.
let p = pos
let looping = true
while looping do
let t = vec_get(toks, p)
if t[0] == "NL" then
let p = p + 1
else
if tok_is(t, "OP", ";") then
let p = p + 1
else
let looping = false
end
end
end
return p
end
# ---- expressions ----
make a function called parse_args takes toks, pos returns r
# pos points just after '('; returns [args, pos_after_rparen]
# tolerates newlines around '(', ',', and ')' so multi-line calls parse
let args = []
let p = skip_nl(toks, pos)
let t = vec_get(toks, p)
if tok_is(t, "OP", ")") then
return [args, p + 1]
else
let looping = true
while looping do
let e = parse_expr(toks, p)
let args = list_push(args, e[0])
let p = skip_nl(toks, e[1])
let t2 = vec_get(toks, p)
if tok_is(t2, "OP", ",") then
let p = skip_nl(toks, p + 1)
else
let looping = false
end
end
let t3 = vec_get(toks, p)
if tok_is(t3, "OP", ")") then
return [args, p + 1]
else
return [[["Err", "expected ) in argument list", vec_get(toks, p)[2]]], p]
end
end
end
make a function called parse_primary takes toks, pos returns r
let t = vec_get(toks, pos)
let ty = t[0]
let tx = t[1]
if ty == "NUM" then
return [["Num", tx], pos + 1]
else
if ty == "STR" then
return [["Str", tx], pos + 1]
else
if ty == "IDENT" then
if (tx == "true") or (tx == "false") then
return [["Bool", tx], pos + 1]
else
if (tx == "pursue") and (tok_is(vec_get(toks, pos + 1), "OP", "(") == false) then
# `pursue GOAL` -- GOAL is a bare name, taken as a string
# literal argument to the `pursue` host fn, mirroring native
# parser.rs's parse_primary special case exactly (the
# parenthesized form `pursue(x)` falls through to ordinary
# call parsing below instead).
let nameTok = vec_get(toks, pos + 1)
if nameTok[0] == "IDENT" then
return [["Call", "pursue", [["Str", nameTok[1]]]], pos + 2]
else
return [["Err", "expected goal name after 'pursue'", nameTok[2]], pos + 1]
end
else
if (tx == "activate") and (tok_is(vec_get(toks, pos + 1), "OP", "(") == false) then
# `activate PLAN` -- PLAN is a general expression (typically a
# variable holding a previous `pursue` result), unlike
# `pursue`'s bare goal-name.
let planR = parse_expr(toks, pos + 1)
return [["Call", "activate", [planR[0]]], planR[1]]
else
if (tx == "budgeted") and tok_is(vec_get(toks, pos + 1), "OP", "(") then
let msR = parse_expr(toks, pos + 2)
let msNode = msR[0]
let p = msR[1]
let existingNode = ["Bool", "false"]
let p2 = p
if tok_is(vec_get(toks, p), "OP", ",") then
let exR = parse_expr(toks, p + 1)
let existingNode = exR[0]
let p2 = exR[1]
end
if tok_is(vec_get(toks, p2), "OP", ")") then
let p3 = p2 + 1
let opener = vec_get(toks, p3)
if tok_is(opener, "OP", "{") then
let bodyR = parse_stmts_until(toks, p3 + 1)
let p4 = bodyR[1]
if tok_is(vec_get(toks, p4), "OP", "}") then
return [["Budgeted", msNode, existingNode, bodyR[0]], p4 + 1]
else
return [["Err", "expected } after budgeted body", vec_get(toks, p4)[2]], p4]
end
else
if (opener[0] == "IDENT") and (opener[1] == "do") then
let bodyR = parse_stmts_until(toks, p3 + 1)
let p4 = bodyR[1]
if tok_is(vec_get(toks, p4), "IDENT", "end") then
return [["Budgeted", msNode, existingNode, bodyR[0]], p4 + 1]
else
return [["Err", "expected end after budgeted body", vec_get(toks, p4)[2]], p4]
end
else
return [["Err", "expected { or do after budgeted(...)", vec_get(toks, p3)[2]], p3]
end
end
else
return [["Err", "expected ) after budgeted arguments", vec_get(toks, p2)[2]], p2]
end
else
let nx = vec_get(toks, pos + 1)
if tok_is(nx, "OP", "(") then
let a = parse_args(toks, pos + 2)
return [["Call", tx, a[0]], a[1]]
else
return [["Var", tx], pos + 1]
end
end
end
end
end
else
if tok_is(t, "OP", "(") then
let inner = parse_expr(toks, pos + 1)
let p = inner[1]
let t2 = vec_get(toks, p)
if tok_is(t2, "OP", ")") then
return [inner[0], p + 1]
else
return [["Err", "expected )", vec_get(toks, p)[2]], p]
end
else
if tok_is(t, "OP", "[") then
# tolerates newlines around '[', ',', and ']' for multi-line lists
let items = []
let p = skip_nl(toks, pos + 1)
let t2 = vec_get(toks, p)
if tok_is(t2, "OP", "]") then
return [["List", items], p + 1]
else
let looping = true
while looping do
let e = parse_expr(toks, p)
let items = list_push(items, e[0])
let p = skip_nl(toks, e[1])
let t3 = vec_get(toks, p)
if tok_is(t3, "OP", ",") then
let p = skip_nl(toks, p + 1)
else
let looping = false
end
end
let t4 = vec_get(toks, p)
if tok_is(t4, "OP", "]") then
return [["List", items], p + 1]
else
return [["Err", "expected ] in list", vec_get(toks, p)[2]], p]
end
end
else
if tok_is(t, "OP", "|") then
# Closure literal: |params| do body end, or |params| { body }
let p = pos + 1
let params = []
let looping = true
while looping do
let pt = vec_get(toks, p)
if tok_is(pt, "OP", "|") then
let p = p + 1
let looping = false
else
if pt[0] == "IDENT" then
let params = list_push(params, pt[1])
let p = p + 1
let nt = vec_get(toks, p)
if tok_is(nt, "OP", ",") then
let p = p + 1
end
else
let looping = false
end
end
end
let p = skip_nl(toks, p)
let dt = vec_get(toks, p)
if (dt[0] == "IDENT") and (dt[1] == "do") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
let et = vec_get(toks, p2)
if (et[0] == "IDENT") and (et[1] == "end") then
return [["Closure", params, body[0]], p2 + 1]
else
return [["Err", "expected end after closure body", vec_get(toks, p2)[2]], p2]
end
else
if tok_is(dt, "OP", "{") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
let et = vec_get(toks, p2)
if tok_is(et, "OP", "}") then
return [["Closure", params, body[0]], p2 + 1]
else
return [["Err", "expected } after closure body", vec_get(toks, p2)[2]], p2]
end
else
return [["Err", "expected do or { after closure params", vec_get(toks, p)[2]], p]
end
end
else
if tok_is(t, "OP", "{") then
# A bare `{ ... }` in expression position is a real,
# deliberate zero-param function literal -- the same
# value-position meaning `|params| { }` has with an empty
# param list, e.g. `let f = { return 42 }` then `f()`.
# Mirrors native parser.rs's parse_primary (Token::BlockStart)
# exactly. NOT legal as its own statement (see parse_stmt's
# dedicated rejection) -- only as a value.
let body = parse_stmts_until(toks, pos + 1)
let p2 = body[1]
let et = vec_get(toks, p2)
if tok_is(et, "OP", "}") then
return [["Closure", [], body[0]], p2 + 1]
else
return [["Err", "expected } after block body", vec_get(toks, p2)[2]], p2]
end
else
return [["Err", "unexpected " + ty + " '" + tx + "'", vec_get(toks, pos)[2]], pos + 1]
end
end
end
end
end
end
end
end
make a function called parse_postfix takes toks, pos returns r
let r1 = parse_primary(toks, pos)
let node = r1[0]
let p = r1[1]
let looping = true
while looping do
let t = vec_get(toks, p)
if tok_is(t, "OP", "[") then
let idx = parse_expr(toks, p + 1)
let p2 = idx[1]
let t2 = vec_get(toks, p2)
if tok_is(t2, "OP", "]") then
let node = ["Index", node, idx[0]]
let p = p2 + 1
else
let node = ["Err", "expected ] after index", vec_get(toks, p2)[2]]
let looping = false
end
else
if tok_is(t, "OP", ".") then
let nameTok = vec_get(toks, p + 1)
if nameTok[0] == "IDENT" then
let node = ["Member", node, nameTok[1]]
let p = p + 2
else
let node = ["Err", "expected name after .", nameTok[2]]
let looping = false
end
else
# obj.method(args) -- a call whose callee is a Member expression,
# e.g. AST.RegisterRoute(...) (the RouterDSL example's own return
# template). The existing "Call" AST shape (used for bare
# NAME(args)) always treats its callee as a plain STRING, resolved
# by name against known functions/locals in lower.patlang -- there
# was no way to compose a Call around an arbitrary expression the
# way native parser.rs's Expr::Call{function: Box<Expr>, args}
# does. A dedicated MethodCall node (object expr + method name +
# args) sidesteps that rather than overloading "Call"'s callee
# slot with two different shapes; lower.patlang routes it to
# send(object, "method", ...args), the same host call obj.prop =
# value already uses on the write side.
if (node[0] == "Member") and tok_is(t, "OP", "(") then
let a = parse_args(toks, p + 1)
let node = ["MethodCall", node[1], node[2], a[0]]
let p = a[1]
else
let looping = false
end
end
end
end
return [node, p]
end
make a function called parse_unary takes toks, pos returns r
let t = vec_get(toks, pos)
if tok_is(t, "IDENT", "not") then
let e = parse_unary(toks, pos + 1)
return [["Un", "not", e[0]], e[1]]
else
if tok_is(t, "IDENT", "bnot") then
let e = parse_unary(toks, pos + 1)
return [["Un", "bnot", e[0]], e[1]]
else
if tok_is(t, "OP", "-") then
let e = parse_unary(toks, pos + 1)
return [["Un", "-", e[0]], e[1]]
else
return parse_postfix(toks, pos)
end
end
end
end
# GitHub #28: the binary-operator precedence chain used to be seven
# hand-cascaded functions (parse_mul -> parse_shift -> parse_add ->
# parse_bitwise -> parse_cmp -> parse_and -> parse_expr's own `or`
# tier), each re-implementing its OWN copy of "peek for a continuation
# operator, optionally past a newline, recurse." Every one of those
# seven copies was a separate place to forget the fix, which is
# exactly what happened (`let x =\n "a"` and several tiers were still
# missing it even after `+`/`-`/`*`/`/`/`%` had already been patched).
# Replaced with ONE table-driven precedence-climbing (Pratt) loop,
# `parse_binary`, so there is only one place left where "does this
# expression continue" is decided at all -- matching the native Rust
# frontend's own architecture (a single unified `parse_expression`,
# not a cascade), not just its behavior.
#
# binop_info(t) -> ["", 0, false] if t isn't a binary operator at all,
# else [canonical_op_name, precedence (1=loosest .. 7=tightest),
# continues_across_a_newline]. Precedence numbers match the OLD
# cascade's own nesting exactly (parse_expr=or=1 was outermost/
# loosest, parse_mul=7 was innermost/tightest) so existing precedence
# stays identical; only the MECHANISM collapsed to one function.
# `continues_across_a_newline` mirrors the native frontend's own
# whitelist (parser.rs's parse_expression) exactly: every arithmetic/
# comparison/bitwise/shift operator continues across a newline: `and`/
# `or` deliberately do NOT (matching that this is a real, general
# design DECISION already made by the reference implementation, not
# an oversight to "complete" -- `let ok = a\nand b` stays two separate
# things, same as it always has been in both frontends).
make a function called binop_info takes t returns info
if t[0] == "OP" then
if t[1] == "*" then
return ["*", 7, true]
end
if t[1] == "/" then
return ["/", 7, true]
end
if t[1] == "%" then
return ["%", 7, true]
end
if t[1] == "+" then
return ["+", 5, true]
end
if t[1] == "-" then
return ["-", 5, true]
end
if t[1] == "==" then
return ["==", 3, true]
end
if t[1] == "!=" then
return ["!=", 3, true]
end
if t[1] == "<" then
return ["<", 3, true]
end
if t[1] == "<=" then
return ["<=", 3, true]
end
if t[1] == ">" then
return [">", 3, true]
end
if t[1] == ">=" then
return [">=", 3, true]
end
return ["", 0, false]
else
if t[0] == "IDENT" then
if t[1] == "shl" then
return ["shl", 6, true]
end
if t[1] == "shr" then
return ["shr", 6, true]
end
if t[1] == "band" then
return ["band", 4, true]
end
if t[1] == "bxor" then
return ["bxor", 4, true]
end
if t[1] == "bor" then
return ["bor", 4, true]
end
if t[1] == "and" then
return ["and", 2, false]
end
if t[1] == "or" then
return ["or", 1, false]
end
return ["", 0, false]
else
return ["", 0, false]
end
end
end
make a function called parse_binary takes toks, pos, min_prec returns r
let r1 = parse_unary(toks, pos)
let node = r1[0]
let p = r1[1]
let looping = true
while looping do
# Same-line operator first (the common case, no newline involved
# at all); only if that's absent do we look PAST any newlines to
# see whether a continuation-eligible operator starts the next
# line -- this order is what keeps `let x = 5\n-3` two separate
# things: a leading `-` past a newline IS eligible in principle,
# but only actually taken if `binop_info`'s continues flag says so
# (true for `-`, matching the native frontend's own choice; this
# is an intentional, documented tradeoff shared with every C-like
# language that allows optional statement separators, not a bug).
let info = binop_info(vec_get(toks, p))
let oppos = p
if info[0] == "" then
let p2 = skip_nl(toks, p)
let info2 = binop_info(vec_get(toks, p2))
if info2[2] then
let info = info2
let oppos = p2
end
end
if (info[0] != "") and (info[1] >= min_prec) then
let rhs_start = skip_nl(toks, oppos + 1)
let r2 = parse_binary(toks, rhs_start, info[1] + 1)
let node = ["Bin", info[0], node, r2[0]]
let p = r2[1]
else
let looping = false
end
end
return [node, p]
end
make a function called parse_expr takes toks, pos returns r
# GitHub #28: an expression may legitimately START on the line AFTER
# whatever introduces it (`let x =\n "..."`, `return\n expr`, an
# `if`/`while` condition on its own line, etc.) -- skip any leading
# newlines here ONCE, at parse_expr's own single entry point, so
# every one of its call sites gets this for free, matching the
# native Rust frontend's own parse_expression (skips newlines before
# parsing its first primary, unconditionally, before the Pratt loop
# even starts).
return parse_binary(toks, skip_nl(toks, pos), 1)
end
# ---- statements ----
make a function called at_block_stop takes toks, pos returns r
let t = vec_get(toks, pos)
if t[0] == "EOF" then
return true
else
if (t[0] == "IDENT") and ((t[1] == "end") or (t[1] == "else") or (t[1] == "elif")) then
return true
else
if (t[0] == "OP") and (t[1] == "}") then
return true
else
return false
end
end
end
end
make a function called parse_stmts_until takes toks, pos returns r
# Collect statements until 'end' / 'else' / EOF (stopper not consumed).
let stmts = []
let p = skip_nl(toks, pos)
let looping = true
while looping do
if at_block_stop(toks, p) then
let looping = false
else
let s = parse_stmt(toks, p)
let stmts = list_push(stmts, s[0])
let p = skip_nl(toks, s[1])
end
end
return [stmts, p]
end
make a function called parse_if_then_tail takes toks, cond, pos, line returns r
# pos points just after 'then'. Handles [elif cond then ...]* [else ...] end.
# `line` is the line of the 'if'/'elif' keyword that introduced THIS
# clause (each elif in a chain desugars to its own nested If node with
# its own line, not the original 'if''s).
let thenPart = parse_stmts_until(toks, pos)
let p2 = thenPart[1]
let t2 = vec_get(toks, p2)
if tok_is(t2, "IDENT", "else") then
let elsePart = parse_stmts_until(toks, p2 + 1)
let p3 = elsePart[1]
let t3 = vec_get(toks, p3)
if tok_is(t3, "IDENT", "end") then
return [["If", cond, thenPart[0], elsePart[0], line], p3 + 1]
else
return [["Err", "expected end after else block", vec_get(toks, p3)[2]], p3]
end
else
if tok_is(t2, "IDENT", "elif") then
let elif_line = t2[2]
let c2 = parse_expr(toks, p2 + 1)
let cond2 = c2[0]
let p3 = c2[1]
let t3 = vec_get(toks, p3)
if tok_is(t3, "IDENT", "then") then
let nested = parse_if_then_tail(toks, cond2, p3 + 1, elif_line)
return [["If", cond, thenPart[0], [nested[0]], line], nested[1]]
else
return [["Err", "expected then after elif condition", vec_get(toks, p3)[2]], p3]
end
else
if tok_is(t2, "IDENT", "end") then
return [["If", cond, thenPart[0], [], line], p2 + 1]
else
return [["Err", "expected else, elif, or end after if block", vec_get(toks, p2)[2]], p2]
end
end
end
end
make a function called parse_if_brace_tail takes toks, cond, pos, line returns r
# pos points just after '{'. Handles [elif cond { ... }]* [else { ... }] '}'.
let thenPart = parse_stmts_until(toks, pos)
let p2 = thenPart[1]
let t2 = vec_get(toks, p2)
if tok_is(t2, "OP", "}") then
let p3 = p2 + 1
let t3 = vec_get(toks, p3)
if tok_is(t3, "IDENT", "else") then
let t4 = vec_get(toks, p3 + 1)
if tok_is(t4, "OP", "{") then
let elsePart = parse_stmts_until(toks, p3 + 2)
let p5 = elsePart[1]
let t5 = vec_get(toks, p5)
if tok_is(t5, "OP", "}") then
return [["If", cond, thenPart[0], elsePart[0], line], p5 + 1]
else
return [["Err", "expected } after else block", vec_get(toks, p5)[2]], p5]
end
else
return [["Err", "expected { after else", vec_get(toks, p3 + 1)[2]], p3 + 1]
end
else
if tok_is(t3, "IDENT", "elif") then
let elif_line = t3[2]
let c2 = parse_expr(toks, p3 + 1)
let cond2 = c2[0]
let p4 = c2[1]
let t4 = vec_get(toks, p4)
if tok_is(t4, "OP", "{") then
let nested = parse_if_brace_tail(toks, cond2, p4 + 1, elif_line)
return [["If", cond, thenPart[0], [nested[0]], line], nested[1]]
else
return [["Err", "expected { after elif condition", vec_get(toks, p4)[2]], p4]
end
else
return [["If", cond, thenPart[0], [], line], p3]
end
end
else
return [["Err", "expected } after if block", vec_get(toks, p2)[2]], p2]
end
end
make a function called parse_if takes toks, pos returns r
# pos points at 'if'. Supports both `if cond then ... end` and
# `if cond { ... }`, with `elif` accepted in either form.
# Line-tracking note: tokens already carry [type, text, line]
# (lexer.patlang) but If/While AST nodes previously dropped it. The
# 'if'/'elif'/'while' keyword's own line is threaded through as the
# LAST element of the node (append-only, so every existing consumer
# that reads node[1]/node[2]/node[3] by fixed index is unaffected) --
# added for self_hosting/lib/coverage.patlang's branch catalog and
# source-line instrumentation, see that file's header for why.
let if_line = vec_get(toks, pos)[2]
let c = parse_expr(toks, pos + 1)
let cond = c[0]
let p = c[1]
let t = vec_get(toks, p)
if tok_is(t, "IDENT", "then") then
return parse_if_then_tail(toks, cond, p + 1, if_line)
else
if tok_is(t, "OP", "{") then
return parse_if_brace_tail(toks, cond, p + 1, if_line)
else
return [["Err", "expected then or { after if condition", vec_get(toks, p)[2]], p]
end
end
end
make a function called parse_while takes toks, pos returns r
# pos points at 'while'. Supports both `while cond do ... end` and
# `while cond { ... }`. See parse_if's comment for why the keyword's
# line is appended as the node's last element.
let while_line = vec_get(toks, pos)[2]
let c = parse_expr(toks, pos + 1)
let cond = c[0]
let p = c[1]
let t = vec_get(toks, p)
if tok_is(t, "IDENT", "do") or tok_is(t, "IDENT", "begin") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
let t2 = vec_get(toks, p2)
if tok_is(t2, "IDENT", "end") then
return [["While", cond, body[0], while_line], p2 + 1]
else
return [["Err", "expected end after while body", vec_get(toks, p2)[2]], p2]
end
else
if tok_is(t, "OP", "{") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
let t2 = vec_get(toks, p2)
if tok_is(t2, "OP", "}") then
return [["While", cond, body[0], while_line], p2 + 1]
else
return [["Err", "expected } after while body", vec_get(toks, p2)[2]], p2]
end
else
return [["Err", "expected do or { after while condition", vec_get(toks, p)[2]], p]
end
end
end
# [message, line] -- message is "" if `nameTok` is fine to use as a
# function name, else the reserved-name error text. Shared by both
# function-definition surface syntaxes (`make a function called` and
# `fn`) since the native-codegen collision this guards against applies
# regardless of which one defined the function.
make a function called pf_check_main_reserved takes nameTok returns r
if nameTok[1] == "main" then
# "main" collides with the native codegen backend's own Rust `fn
# main()` entry point -- a user function literally named this
# compiled successfully via patc1.exe with no error, but the
# emitted call recursed into itself infinitely at runtime instead
# of dispatching to the real program entry, overflowing the stack.
#
# This is genuinely ONLY a native-compilation problem (--patc /
# patc1.exe) -- the tree-walking interpreter (--ir-run) has no Rust
# entry point to collide with at all, `main` there is just an
# ordinary function name. But the parser has no idea, at parse
# time, which backend the resulting AST will end up running on --
# a single parsed program can be hand to either. Rejecting `main`
# unconditionally here, for BOTH backends, is a deliberate, known
# over-restriction chosen for simplicity (one check, one place,
# catchable with a clear message at parse time) over a more
# precise but architecturally heavier fix (deferring the check to
# codegen time, backend-conditional). Worth relaxing later if it
# ever becomes a real nuisance for --ir-run-only code, but the
# honest long-term fix is making native codegen emit a Rust entry
# point name that genuinely cannot collide with any PatLang
# identifier, removing the need for this restriction altogether.
return ["'main' is a reserved function name", nameTok[2]]
end
return ["", 0]
end
make a function called parse_function_def takes toks, pos returns r
# pos points at 'make'; expect: make a function called NAME
# [takes a, b] [returns r] NL body end
let p = pos + 1
if tok_is(vec_get(toks, p), "IDENT", "a") then
let p = p + 1
end
if tok_is(vec_get(toks, p), "IDENT", "function") then
let p = p + 1
else
return [["Err", "expected 'function' after make", vec_get(toks, p)[2]], p]
end
if tok_is(vec_get(toks, p), "IDENT", "called") then
let p = p + 1
end
let nameTok = vec_get(toks, p)
if nameTok[0] == "IDENT" then
let name = nameTok[1]
let mainErr = pf_check_main_reserved(nameTok)
if mainErr[0] != "" then
return [["Err", mainErr[0], mainErr[1]], p]
end
let p = p + 1
let params = []
if tok_is(vec_get(toks, p), "IDENT", "takes") then
let p = p + 1
let looping = true
while looping do
let t = vec_get(toks, p)
if t[0] == "IDENT" then
if (t[1] == "returns") then
let looping = false
else
let params = list_push(params, t[1])
let p = p + 1
end
else
if tok_is(t, "OP", ",") then
let p = p + 1
else
let looping = false
end
end
end
end
# Named-return hint (GitHub issue #5): `returns r` gives r real
# fall-through semantics via lower.patlang's Func-lowering, matching
# Stage 0's rust-runtime/src/parser.rs:781-797 -- previously parsed
# and unconditionally discarded here (Stage 1 always required an
# explicit return). "" means no hint, matching the convention no
# other Func-node consumer treats an empty string specially.
let return_hint = ""
if tok_is(vec_get(toks, p), "IDENT", "returns") then
let hintTok = vec_get(toks, p + 1)
if hintTok[0] == "IDENT" then
let return_hint = hintTok[1]
end
let p = p + 2
end
# Body may be `... end` (word form) or `{ ... }` (brace form).
if tok_is(vec_get(toks, p), "OP", "{") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
if tok_is(vec_get(toks, p2), "OP", "}") then
return [["Func", name, params, body[0], return_hint], p2 + 1]
else
return [["Err", "expected } after function body", vec_get(toks, p2)[2]], p2]
end
else
let body = parse_stmts_until(toks, p)
let p2 = body[1]
if tok_is(vec_get(toks, p2), "IDENT", "end") then
return [["Func", name, params, body[0], return_hint], p2 + 1]
else
return [["Err", "expected end after function body", vec_get(toks, p2)[2]], p2]
end
end
else
return [["Err", "expected function name", vec_get(toks, p)[2]], p]
end
end
# `fn` form: fn NAME ( param, param, ... ) { body } -- shorter surface
# syntax for the same ["Func", name, params, body] node
# parse_function_def produces. No takes/returns keywords, and (unlike
# parse_function_def, which accepts either `{ }` or `... end`) the body
# is brace-only here -- matches native's parse_function
# (rust-runtime/src/parser.rs:706), which unconditionally expects '{'
# right after the parameter list, with no `end`-delimited alternative.
make a function called parse_fn_def takes toks, pos returns r
# pos points at 'fn'
let p = pos + 1
let nameTok = vec_get(toks, p)
if nameTok[0] != "IDENT" then
return [["Err", "expected function name", nameTok[2]], p]
end
let name = nameTok[1]
let mainErr = pf_check_main_reserved(nameTok)
if mainErr[0] != "" then
return [["Err", mainErr[0], mainErr[1]], p]
end
let p = p + 1
if tok_is(vec_get(toks, p), "OP", "(") == false then
return [["Err", "expected ( after function name", vec_get(toks, p)[2]], p]
end
let p = p + 1
let params = []
if tok_is(vec_get(toks, p), "OP", ")") == false then
let looping = true
while looping do
let t = vec_get(toks, p)
if t[0] == "IDENT" then
let params = list_push(params, t[1])
let p = p + 1
let t2 = vec_get(toks, p)
if tok_is(t2, "OP", ",") then
let p = p + 1
else
let looping = false
end
else
return [["Err", "expected parameter name", t[2]], p]
end
end
end
if tok_is(vec_get(toks, p), "OP", ")") == false then
return [["Err", "expected ) to close parameter list", vec_get(toks, p)[2]], p]
end
let p = p + 1
if tok_is(vec_get(toks, p), "OP", "{") == false then
return [["Err", "expected { to start function body", vec_get(toks, p)[2]], p]
end
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
if tok_is(vec_get(toks, p2), "OP", "}") then
return [["Func", name, params, body[0]], p2 + 1]
else
return [["Err", "expected } after function body", vec_get(toks, p2)[2]], p2]
end
end
# ---- match/case pattern matching (issue #44) ----
#
# Grammar:
# match EXPR do
# case PATTERN [when GUARD] then STMTS
# ...
# case _ then STMTS
# end
#
# Pattern AST shapes (never seen outside the parser/lowerer -- lower.patlang's
# compile_pattern is the only consumer):
# ["PWild"] _
# ["PBind", name] bare lowercase identifier
# ["PLit", litNode] Num/Str/Bool literal
# ["PCmp", op, exprNode] > < >= <= == != EXPR (a guard wearing pattern
# clothing -- see lower.patlang's compile_pattern)
# ["PGlob", globStr] *fred*, fred*, etc (unquoted, translated to a
# regex and matched via regex.patlang's glob_match)
# ["PList", [subpatterns]] [p1, p2, ...] -- recursive tagged-list pattern
#
# `match` is pure syntactic sugar (see lower.patlang's lower_match): the
# ["Match", scrutinee, arms, line] AST node it produces here carries its own
# line as the last element, same append-only convention as If/While/When.
make a function called at_case_stop takes toks, pos returns r
# Case-arm bodies stop at the next 'case', 'end', or EOF (never 'else'/
# 'elif' -- those belong to a nested if/while inside the arm body, not to
# match's own grammar, so at_block_stop's stop-set doesn't apply here).
let t = vec_get(toks, pos)
if t[0] == "EOF" then
return true
else
if (t[0] == "IDENT") and ((t[1] == "end") or (t[1] == "case")) then
return true
else
return false
end
end
end
make a function called parse_case_stmts_until takes toks, pos returns r
let stmts = []
let p = skip_nl(toks, pos)
let looping = true
while looping do
if at_case_stop(toks, p) then
let looping = false
else
let s = parse_stmt(toks, p)
let stmts = list_push(stmts, s[0])
let p = skip_nl(toks, s[1])
end
end
return [stmts, p]
end
# Collects a run of IDENT/NUM/OP("*")/OP("?") token text into a single glob
# string, stopping before 'then'/'when' (or NL/EOF). Handles both leading-
# wildcard globs (`*fred*`, pos already at the leading '*') and trailing-
# wildcard globs (`fred*`, pos at the leading identifier).
make a function called parse_glob_pattern takes toks, pos returns r
let glob_b = sb_new()
let p = pos
let looping = true
while looping do
let t = vec_get(toks, p)
if (t[0] == "IDENT") and ((t[1] == "then") or (t[1] == "when")) then
let looping = false
else
if (t[0] == "NL") or (t[0] == "EOF") then
let looping = false
else
if (t[0] == "OP") and ((t[1] == "*") or (t[1] == "?") or (t[1] == ".")) then
sb_push(glob_b, t[1])
let p = p + 1
else
if (t[0] == "IDENT") or (t[0] == "NUM") then
sb_push(glob_b, t[1])
let p = p + 1
else
let looping = false
end
end
end
end
end
return [["PGlob", sb_str(glob_b)], p]
end
make a function called parse_pattern takes toks, pos returns r
let t = vec_get(toks, pos)
if tok_is(t, "IDENT", "_") then
return [["PWild"], pos + 1]
else
if tok_is(t, "OP", "[") then
let p = skip_nl(toks, pos + 1)
if tok_is(vec_get(toks, p), "OP", "]") then
return [["PList", []], p + 1]
else
return parse_pattern_list_tail(toks, p, [])
end
else
if (t[0] == "OP") and ((t[1] == ">") or (t[1] == "<") or (t[1] == ">=") or (t[1] == "<=") or (t[1] == "==") or (t[1] == "!=")) then
let e = parse_expr(toks, pos + 1)
return [["PCmp", t[1], e[0]], e[1]]
else
if tok_is(t, "OP", "*") then
return parse_glob_pattern(toks, pos)
else
if t[0] == "NUM" then
return [["PLit", ["Num", t[1]]], pos + 1]
else
if t[0] == "STR" then
return [["PLit", ["Str", t[1]]], pos + 1]
else
if tok_is(t, "IDENT", "true") or tok_is(t, "IDENT", "false") then
return [["PLit", ["Bool", t[1]]], pos + 1]
else
if t[0] == "IDENT" then
let nxt = vec_get(toks, pos + 1)
if (nxt[0] == "OP") and ((nxt[1] == "*") or (nxt[1] == "?")) then
return parse_glob_pattern(toks, pos)
else
return [["PBind", t[1]], pos + 1]
end
else
return [["Err", "invalid pattern", t[2]], pos + 1]
end
end
end
end
end
end
end
end
end
make a function called parse_pattern_list_tail takes toks, pos, acc returns r
let pr = parse_pattern(toks, pos)
let acc2 = list_push(acc, pr[0])
let p = skip_nl(toks, pr[1])
let t = vec_get(toks, p)
if tok_is(t, "OP", ",") then
return parse_pattern_list_tail(toks, skip_nl(toks, p + 1), acc2)
else
if tok_is(t, "OP", "]") then
return [["PList", acc2], p + 1]
else
return [["Err", "expected ',' or ']' in list pattern", t[2]], p]
end
end
end
make a function called parse_case_arm takes toks, pos returns r
# pos points at 'case'. Returns [[pattern, guard_or_false, body], next_pos].
let pr = parse_pattern(toks, pos + 1)
let pattern = pr[0]
let p = skip_nl(toks, pr[1])
let t = vec_get(toks, p)
if tok_is(t, "IDENT", "when") then
let g = parse_expr(toks, p + 1)
let guard = g[0]
let p2 = skip_nl(toks, g[1])
let t2 = vec_get(toks, p2)
if tok_is(t2, "IDENT", "then") then
let body = parse_case_stmts_until(toks, p2 + 1)
return [[pattern, guard, body[0]], body[1]]
else
return [["Err", "expected then after when guard", t2[2]], p2]
end
else
if tok_is(t, "IDENT", "then") then
let body = parse_case_stmts_until(toks, p + 1)
return [[pattern, false, body[0]], body[1]]
else
return [["Err", "expected then (or when GUARD then) after case pattern", t[2]], p]
end
end
end
make a function called parse_match_arms takes toks, pos, acc returns r
let p = skip_nl(toks, pos)
let t = vec_get(toks, p)
if tok_is(t, "IDENT", "end") then
return [acc, p + 1]
else
if tok_is(t, "IDENT", "case") then
let arm = parse_case_arm(toks, p)
if arm[0][0] == "Err" then
return [arm[0], arm[1]]
else
return parse_match_arms(toks, skip_nl(toks, arm[1]), list_push(acc, arm[0]))
end
else
return [["Err", "expected case or end in match block", t[2]], p]
end
end
end
make a function called parse_match takes toks, pos returns r
# pos points at 'match'.
let match_line = vec_get(toks, pos)[2]
let se = parse_expr(toks, pos + 1)
let scrutinee = se[0]
let p = se[1]
let t = vec_get(toks, p)
if tok_is(t, "IDENT", "do") then
let arms = parse_match_arms(toks, p + 1, [])
if arms[0][0] == "Err" then
return [arms[0], arms[1]]
else
return [["Match", scrutinee, arms[0], match_line], arms[1]]
end
else
return [["Err", "expected do after match expression", t[2]], p]
end
end
make a function called parse_when takes toks, pos returns r
# pos points at 'when'; expect: when EVENT do body end, or when EVENT { body }
# `when_line` is appended as the node's LAST element (same append-only
# convention parse_if/parse_while already use for their own line
# numbers) so a later structural check (parse_check_when_placement)
# can report exactly where a misplaced `when` was written.
let when_line = vec_get(toks, pos)[2]
let nameTok = vec_get(toks, pos + 1)
if nameTok[0] == "IDENT" then
let ev = nameTok[1]
let p = pos + 2
if tok_is(vec_get(toks, p), "IDENT", "do") or tok_is(vec_get(toks, p), "IDENT", "begin") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
if tok_is(vec_get(toks, p2), "IDENT", "end") then
return [["When", ev, body[0], when_line], p2 + 1]
else
return [["Err", "expected end after when body", vec_get(toks, p2)[2]], p2]
end
else
if tok_is(vec_get(toks, p), "OP", "{") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
if tok_is(vec_get(toks, p2), "OP", "}") then
return [["When", ev, body[0], when_line], p2 + 1]
else
return [["Err", "expected } after when body", vec_get(toks, p2)[2]], p2]
end
else
return [["Err", "expected do or { after when event", vec_get(toks, p)[2]], p]
end
end
else
return [["Err", "expected event name after when", vec_get(toks, pos + 1)[2]], pos + 1]
end
end
# ---- rule declarations: `rule Head(args) :- Body1, Body2.` / `rule Head(args).` ----
make a function called parse_rule_goal takes toks, pos returns r
# parses `name(args)` -> [[pred, [arg_expr_nodes]], next_pos]
let nameTok = vec_get(toks, pos)
let name = nameTok[1]
let lp = vec_get(toks, pos + 1)
if tok_is(lp, "OP", "(") then
let a = parse_args(toks, pos + 2)
return [[name, a[0]], a[1]]
else
return [["Err", "expected ( after rule goal name '" + name + "'", nameTok[2]], pos + 1]
end
end
make a function called parse_rule_decl takes toks, pos returns r
# pos points at the rule head's name (the 'rule' keyword itself was
# already consumed by the caller). Returns
# [["RuleDecl", head_pred, [head_args], [[pred,[args]], ...]], next_pos].
let head = parse_rule_goal(toks, pos)
let head_pred = head[0][0]
let head_args = head[0][1]
let p = head[1]
let t = vec_get(toks, p)
if tok_is(t, "OP", ":-") then
let p = p + 1
let body = []
let looping = true
while looping do
let p = skip_nl(toks, p)
let g = parse_rule_goal(toks, p)
let body = list_push(body, g[0])
let p = g[1]
let p = skip_nl(toks, p)
let t2 = vec_get(toks, p)
if tok_is(t2, "OP", ",") then
let p = p + 1
else
if tok_is(t2, "OP", ".") then
let p = p + 1
let looping = false
else
let looping = false
end
end
end
return [["RuleDecl", head_pred, head_args, body], p]
else
if tok_is(t, "OP", ".") then
return [["RuleDecl", head_pred, head_args, []], p + 1]
else
return [["Err", "expected ':-' or '.' after rule head", vec_get(toks, p)[2]], p]
end
end
end
# ---- goal declarations: `goal NAME { dep1(args), dep2(args) }` ----
make a function called parse_goal_decl takes toks, pos returns r
# pos points at the goal's name (the 'goal' keyword itself already
# consumed by the caller). Returns [["GoalDecl", name, deps], next_pos]
# where deps is [[pred, [arg_expr_nodes]], ...] -- each dependency term
# is parsed via parse_rule_goal, the exact same fact-term shape a rule
# body's goals use.
let nameTok = vec_get(toks, pos)
let name = nameTok[1]
let p = pos + 1
let openTok = vec_get(toks, p)
if tok_is(openTok, "OP", "{") == false then
return [["Err", "expected { after goal name", openTok[2]], p]
else
let p = skip_nl(toks, p + 1)
let t = vec_get(toks, p)
if tok_is(t, "OP", "}") then
return [["GoalDecl", name, []], p + 1]
else
let deps = []
let looping = true
while looping do
let p = skip_nl(toks, p)
let g = parse_rule_goal(toks, p)
let deps = list_push(deps, g[0])
let p = skip_nl(toks, g[1])
let t2 = vec_get(toks, p)
if tok_is(t2, "OP", ",") then
let p = skip_nl(toks, p + 1)
else
let looping = false
end
end
let t3 = vec_get(toks, p)
if tok_is(t3, "OP", "}") then
return [["GoalDecl", name, deps], p + 1]
else
return [["Err", "expected } to close goal block", vec_get(toks, p)[2]], p]
end
end
end
end
# Slice 1+2+3 of the classes/traits/inheritance feature (see the
# "synchronous-questing-metcalfe" plan) -- mirrors native parser.rs's
# Token::Class handling exactly. pos points at the class's name (the
# 'class' keyword itself already consumed by the caller). Returns
# [["ClassDecl", name, parent_or_empty_string, fields, methods, traits],
# next_pos] where fields is [[field_name, default_expr_node], ...],
# methods (Slice 2) is [[method_name, params, body], ...] -- a method is
# just an ordinary `make a function called NAME ... end`/`{ }` body
# declared inside the class block, reusing parse_function_def exactly --
# and traits (Slice 3) is [trait_name, ...] from a `traits A, B` line.
make a function called parse_class_decl takes toks, pos returns r
let nameTok = vec_get(toks, pos)
let name = nameTok[1]
let p = pos + 1
let parent = ""
let nt = vec_get(toks, p)
if (nt[0] == "IDENT") and (nt[1] == "inherits") then
let p = p + 1
let parentTok = vec_get(toks, p)
if parentTok[0] != "IDENT" then
return [["Err", "expected parent class name after 'inherits'", parentTok[2]], p]
else
let parent = parentTok[1]
let p = p + 1
end
end
let openTok = vec_get(toks, p)
# Slice 5: `{ ... }` or `do ... end` (`begin` accepted as a synonym for
# `do`, matching every other dual-form block in this parser -- see
# parse_while/parse_when above). `word_form` remembers which delimiter
# opened the block so the loop below knows whether to stop at `}` or at
# the word `end`.
if (tok_is(openTok, "OP", "{") == false) and (tok_is(openTok, "IDENT", "do") == false) and (tok_is(openTok, "IDENT", "begin") == false) then
return [["Err", "expected '{' or 'do'/'begin' after class header", openTok[2]], p]
else
let word_form = tok_is(openTok, "OP", "{") == false
let p = skip_nl(toks, p + 1)
let fields = []
let methods = []
let traits = []
let looping = true
while looping do
let t = vec_get(toks, p)
let at_block_end = false
if word_form then
if tok_is(t, "IDENT", "end") then
let at_block_end = true
end
else
if tok_is(t, "OP", "}") then
let at_block_end = true
end
end
if at_block_end then
let looping = false
else
if (t[0] == "IDENT") and (t[1] == "traits") then
let p = p + 1
let tlooping = true
while tlooping do
let tt = vec_get(toks, p)
if tt[0] != "IDENT" then
return [["Err", "expected trait name", tt[2]], p]
else
let traits = list_push(traits, tt[1])
let p = p + 1
if tok_is(vec_get(toks, p), "OP", ",") then
let p = p + 1
else
let tlooping = false
end
end
end
let p = skip_nl(toks, p)
else
if (t[0] == "IDENT") and (t[1] == "field") then
let fnameTok = vec_get(toks, p + 1)
if fnameTok[0] != "IDENT" then
return [["Err", "expected field name after 'field'", fnameTok[2]], p + 1]
else
let eqTok = vec_get(toks, p + 2)
if tok_is(eqTok, "OP", "=") == false then
return [["Err", "expected '=' after field name", eqTok[2]], p + 2]
else
let e = parse_expr(toks, p + 3)
let fields = list_push(fields, [fnameTok[1], e[0]])
let p = skip_nl(toks, e[1])
end
end
else
if (t[0] == "IDENT") and (t[1] == "make") then
let fr = parse_function_def(toks, p)
let fnode = fr[0]
if fnode[0] == "Err" then
return [fnode, fr[1]]
else
let methods = list_push(methods, [fnode[1], fnode[2], fnode[3]])
let p = skip_nl(toks, fr[1])
end
else
return [["Err", "expected 'field NAME = EXPR', 'make a function called NAME ... end', or 'traits A, B' inside a class block", t[2]], p]
end
end
end
end
end
return [["ClassDecl", name, parent, fields, methods, traits], p + 1]
end
end
make a function called parse_stmt_inner takes toks, pos returns r
let t = vec_get(toks, pos)
let ty = t[0]
let tx = t[1]
if tok_is(t, "OP", "{") then
# A bare `{ ... }` is only meaningful as a VALUE (parse_primary turns
# it into a zero-param closure literal, e.g. `let f = { ... }` -- a
# real function literal, not a leftover tolerance) -- never as its
# own free-standing statement. Rejecting it here is what makes
# dropping the statement-separator requirement safe: a bare block
# sitting right after something that could take a trailing-closure
# argument would otherwise be genuinely ambiguous between "two
# statements" and "one statement with sugar." Mirrors native
# parser.rs's parse_statement Token::BlockStart arm exactly.
return [["Err", "a bare '{ ... }' isn't a statement on its own -- it's a function literal as a VALUE (e.g. `let f = { ... }`, then call it with `f()`), not something to write standalone", t[2]], pos + 1]
else
if ty == "IDENT" then
if tx == "let" then
let mutTok = vec_get(toks, pos + 1)
let isMut = tok_is(mutTok, "IDENT", "mut")
let nameIdx = pos + 1
if isMut then
let nameIdx = pos + 2
end
let nameTok = vec_get(toks, nameIdx)
let name = nameTok[1]
let eqTok = vec_get(toks, nameIdx + 1)
if tok_is(eqTok, "OP", "=") then
let e = parse_expr(toks, nameIdx + 2)
return [["Let", name, e[0], false, isMut], e[1]]
else
return [["Err", "expected = after let name", vec_get(toks, nameIdx)[2]], nameIdx]
end
else
if tx == "if" then
return parse_if(toks, pos)
else
if tx == "while" then
return parse_while(toks, pos)
else
if tx == "match" then
return parse_match(toks, pos)
else
if (tx == "require") or (tx == "ensure") or (tx == "assert") then
let e = parse_expr(toks, pos + 1)
return [["Assert", tx, e[0]], e[1]]
else
if tx == "return" then
let e = parse_expr(toks, pos + 1)
return [["Return", e[0]], e[1]]
else
if tx == "make" then
return parse_function_def(toks, pos)
else
if tx == "fn" then
return parse_fn_def(toks, pos)
else
if tx == "when" then
return parse_when(toks, pos)
else
if (tx == "rule") and (tok_is(vec_get(toks, pos + 1), "OP", "(") == false) then
# Real declarative syntax: `rule Head(args) :- Body1, Body2.`
# (a rule) or `rule Head(args).` (a fact -- a rule with an
# empty body). The already-working call form `rule(...)`
# falls through to the bare-identifier-call path below,
# matching the native parser's own `if peek==LParen` branch.
return parse_rule_decl(toks, pos + 1)
else
if (tx == "goal") and (tok_is(vec_get(toks, pos + 1), "OP", "(") == false) then
# Real declarative syntax: `goal NAME { dep1(args), ... }`,
# mirroring native parser.rs's Token::Goal branch exactly.
return parse_goal_decl(toks, pos + 1)
else
if (tx == "class") and (tok_is(vec_get(toks, pos + 1), "OP", "(") == false) then
# Slice 1 of the classes/traits/inheritance feature (see
# the "synchronous-questing-metcalfe" plan): `class NAME
# [inherits PARENT] { field NAME = EXPR ... }`,
# mirroring native parser.rs's Token::Class branch
# exactly.
return parse_class_decl(toks, pos + 1)
else
if (tx == "pursue") or (tx == "activate") then
# Both are expressions (see parse_primary), routed through
# parse_expr/parse_primary at the statement level too --
# same pattern budgeted(...) uses just below.
let e = parse_expr(toks, pos)
return [["Expr", e[0]], e[1]]
else
if (tx == "budgeted") and tok_is(vec_get(toks, pos + 1), "OP", "(") then
# Bare statement usage (result discarded): route through
# parse_expr/parse_primary, which is where budgeted(...)
# is actually parsed -- parse_stmt's own bare-call fallback
# below has no block-body awareness and would otherwise
# mis-parse `budgeted(ms) { ... }`/`do ... end` as a
# malformed ordinary call.
let e = parse_expr(toks, pos)
return [["Expr", e[0]], e[1]]
else
# General fallback: parse a full expression (this already
# covers bare calls NAME(args), member/index chains
# obj.prop / obj.prop.sub / list[i], and full binary
# expressions like classify(x) == "adult" via the same
# parse_expr/parse_postfix/parse_primary path used
# everywhere else -- unlike native parser.rs, this
# self-hosted lexer/parser never treats a bare '=' as an
# equality operator (only '==' is recognized as a
# comparison, see the OP-token check just above this
# function), so parse_expr always stops cleanly BEFORE a
# pending assignment '=', with no risk of the ambiguity
# the native side has to guard against with its
# stop_trailing_block_for_condition flag. Previously this
# branch hand-rolled only 3 narrow shapes (bare call,
# bare reassignment, obj.prop[ = value]) and errored on
# anything else -- found via `pair[0]` (a bare index
# expression) and `classify(x) == "adult"` (a bare call
# followed by a comparison) both failing to parse as a
# closure body's implicit-return statement.
let e = parse_expr(toks, pos)
let node = e[0]
let p2 = e[1]
let eqTok = vec_get(toks, p2)
if tok_is(eqTok, "OP", "=") then
if node[0] == "Member" then
let vr = parse_expr(toks, p2 + 1)
return [["MemberAssign", node[1], node[2], vr[0]], vr[1]]
else
if node[0] == "Var" then
let vr = parse_expr(toks, p2 + 1)
return [["Let", node[1], vr[0], true, false], vr[1]]
else
return [["Err", "cannot assign to this expression", eqTok[2]], p2]
end
end
else
return [["Expr", node], p2]
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
else
return [["Err", "unexpected " + ty + " '" + tx + "'", vec_get(toks, pos)[2]], pos + 1]
end
end
end
# Thin line-tagging wrapper around parse_stmt_inner: the debugger's
# breakpoint/step machinery (self_hosting/lib/interp.patlang) needs a
# source line on Let/Expr/Return/Assert/MemberAssign nodes, which were
# never given one (If/While/When/Err already carry their own trailing
# line field). Deliberately an INCLUSION list, not "append to everything
# except the types that already have a line": Func nodes use
# `s.length > 4` in lower_program to detect an optional trailing
# return-hint element, so unconditionally appending a line there would
# silently corrupt that check on every ordinary function (s[4] would
# sometimes be the return hint, sometimes a line number). Any other node
# shape gains a shape-dependent meaning for a new trailing slot exactly
# the same way, so only touch the shapes actually needed here.
make a function called parse_stmt takes toks, pos returns r
let line = vec_get(toks, pos)[2]
let inner = parse_stmt_inner(toks, pos)
let node = inner[0]
let ty = node[0]
if (ty == "Let") or (ty == "Expr") or (ty == "Return") or (ty == "Assert") or (ty == "MemberAssign") then
return [list_push(node, line), inner[1]]
else
return inner
end
end
make a function called parse_program takes toks returns ast
let stmts = []
let pos = skip_nl(toks, 0)
let looping = true
while looping do
let t = vec_get(toks, pos)
if t[0] == "EOF" then
let looping = false
else
let r = parse_stmt(toks, pos)
let stmts = list_push(stmts, r[0])
let pos = skip_nl(toks, r[1])
end
end
return ["Program", stmts]
end
# ---- AST pretty printer ----
make a function called ast_list_to_str takes nodes returns s
# sb_new/sb_push/sb_str -- low real-world risk (this is a debug/error-
# message pretty printer, not part of the ordinary compile path), but
# fixed for consistency once the anti-pattern was found and a compiler
# warning added to catch it elsewhere.
let b = sb_new()
let i = 0
while i < nodes.length do
if i > 0 then
sb_push(b, "; ")
end
sb_push(b, ast_to_str(nodes[i]))
let i = i + 1
end
return sb_str(b)
end
# Joins a list of plain strings (e.g. closure param names), not AST nodes
make a function called ast_list_to_str_plain takes items returns s
let b = sb_new()
let i = 0
while i < items.length do
if i > 0 then
sb_push(b, ", ")
end
sb_push(b, items[i])
let i = i + 1
end
return sb_str(b)
end
# Renders a RuleDecl body: a list of [pred, arg_expr_nodes] pairs (not
# ast nodes with their own type tag, so this can't just call ast_to_str
# on each element the way ast_list_to_str does).
make a function called rule_body_to_str takes body returns s
let sb = sb_new()
sb_push(sb, "[")
let i = 0
let n = to_num(list_len(body))
while i < n do
if i > 0 then
sb_push(sb, ", ")
end
let g = body[i]
sb_push(sb, g[0] + "(" + ast_list_to_str(g[1]) + ")")
let i = i + 1
end
sb_push(sb, "]")
return sb_str(sb)
end
make a function called ast_to_str takes node returns s
let ty = node[0]
if ty == "Num" then
return "Num(" + node[1] + ")"
else
if ty == "Str" then
return "Str('" + node[1] + "')"
else
if ty == "Bool" then
return "Bool(" + node[1] + ")"
else
if ty == "Var" then
return "Var(" + node[1] + ")"
else
if ty == "Bin" then
return "(" + ast_to_str(node[2]) + " " + node[1] + " " + ast_to_str(node[3]) + ")"
else
if ty == "Un" then
return "(" + node[1] + " " + ast_to_str(node[2]) + ")"
else
if ty == "Call" then
return node[1] + "(" + ast_list_to_str(node[2]) + ")"
else
if ty == "Closure" then
return "|" + ast_list_to_str_plain(node[1]) + "| do " + ast_list_to_str(node[2]) + " end"
else
if ty == "List" then
return "[" + ast_list_to_str(node[1]) + "]"
else
if ty == "Index" then
return ast_to_str(node[1]) + "[" + ast_to_str(node[2]) + "]"
else
if ty == "Member" then
return ast_to_str(node[1]) + "." + node[2]
else
if ty == "Let" then
return "Let " + node[1] + " = " + ast_to_str(node[2])
else
if ty == "MemberAssign" then
return ast_to_str(node[1]) + "." + node[2] + " = " + ast_to_str(node[3])
else
if ty == "Expr" then
return ast_to_str(node[1])
else
if ty == "If" then
return "If " + ast_to_str(node[1]) + " Then {" + ast_list_to_str(node[2]) + "} Else {" + ast_list_to_str(node[3]) + "}"
else
if ty == "While" then
return "While " + ast_to_str(node[1]) + " {" + ast_list_to_str(node[2]) + "}"
else
if ty == "Func" then
return "Func " + node[1] + " {" + ast_list_to_str(node[3]) + "}"
else
if ty == "Return" then
return "Return " + ast_to_str(node[1])
else
if ty == "When" then
return "When " + node[1] + " {" + ast_list_to_str(node[2]) + "}"
else
if ty == "RuleDecl" then
return "RuleDecl(" + node[1] + ", " + ast_list_to_str(node[2]) + ", " + rule_body_to_str(node[3]) + ")"
else
if ty == "GoalDecl" then
return "GoalDecl(" + node[1] + ", " + rule_body_to_str(node[2]) + ")"
else
if ty == "Assert" then
return node[1] + " " + ast_to_str(node[2])
else
if ty == "Budgeted" then
return "budgeted(" + ast_to_str(node[1]) + ", " + ast_to_str(node[2]) + ") {" + ast_list_to_str(node[3]) + "}"
else
if ty == "Err" then
# Markers, not a single "@" -- the message can itself
# echo arbitrary token text (e.g. "unexpected UNK '@'"
# for a literal '@' in the source), so a single-char
# delimiter is not safe to search for. "@@ENDERR@@"
# gives parse_find_error an unambiguous message-end
# boundary regardless of what the message contains.
#
# Checking `ty == "Err"` explicitly here (not just "any
# unrecognized tag with >=3 elements", which is what this
# used to do) matters for a real reason, not just
# precision for its own sake: the OLD version produced a
# genuine false positive the moment this file's own
# source was compiled by patc1.exe and handed to itself
# as input (e.g. any selftest that `include`s lib/
# parser.patlang) -- this very file's source literally
# CONTAINS the string "@@ERR@@" as a string-literal value
# (right here), so a generic "any unknown 3+-field tag"
# fallback would render THIS Str node's own content and
# parse_find_error would mistake it for a real error, on
# perfectly valid, successfully-parsed source. Confirmed
# directly: self_hosting/regex_dsl_selftest.patlang
# (pre-existing, not new) failed to compile via patc1.exe
# with exactly this symptom before this fix.
return "@@ERR@@" + ("" + node[2]) + "@@" + node[1] + "@@ENDERR@@"
else
# A genuinely unrecognized tag (not "Err") -- this
# shouldn't normally happen given the catalog above, but
# stays a safety net for a future missing case (the way
# "RuleDecl" once was, before it got its own real case).
# Deliberately does NOT contain "@@ERR@@"/"@@ENDERR@@" or
# any other text parse_find_error searches for, so an
# unrecognized tag renders as inert, harmless text instead
# of being silently misidentified as a parse error.
return "Unrecognized(" + ty + ")"
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
# ---- compiler error reasoning: report *and* suggest a fix, not just
# fail silently. This is what closes the long-standing gap where
# patc1_main.patlang (the actual self-hosted compiler driver) had NO
# error handling at all -- a syntax error just silently produced an
# ["Err", ...] node buried somewhere in the AST, which lower_program
# either choked on unpredictably or silently dropped (the "silent
# statement dropping" Stage 0 gotcha), with zero indication to the user
# that anything went wrong. All PatLang-level: no rustc/Rust involved
# anywhere in this reasoning, and none ever should be -- the project's
# own direction is to remove that dependency entirely, not build more
# tooling around its error output. ----
# The first Err node found anywhere in a parsed program's statement
# list -- top-level, or nested inside a statement's own expressions
# (e.g. `let z = (((` produces a top-level Let statement whose VALUE is
# the malformed node, not a top-level Err itself).
#
# This is a genuine typed walk over the real AST (checking each node's
# OWN tag field directly), not text-rendering-and-scanning. An earlier
# version rendered each statement with ast_to_str and searched the
# rendered text for an "@@ERR@@...@@ENDERR@@" marker pair -- reusing
# self_hosting/lib/repl_core.patlang's original technique for the
# message alone, extended to also recover the line number. That
# approach had a real, confirmed bug: ast_to_str renders a plain Str
# node's OWN VALUE verbatim inline, so a string literal whose CONTENT
# happened to equal the marker text produced a false positive --
# exactly what happens the moment THIS FILE's own source (which
# legitimately contains "@@ERR@@" as a string literal, since that's the
# code that used to construct the marker) is compiled by patc1.exe and
# handed to itself as input, e.g. any selftest that `include`s this
# library. Confirmed directly: self_hosting/regex_dsl_selftest.patlang
# (pre-existing, unrelated to this session) failed to compile via
# patc1.exe with exactly this symptom. A typed walk checking `node[0] ==
# "Err"` directly is immune to this whole class of bug by construction
# -- it can never confuse a node's DATA (a string's contents) with its
# STRUCTURE (its tag), because it never turns the tree into text at all.
#
# Mirrors ast_to_str's own per-tag field layout exactly (read directly
# from that function, not guessed) so this stays a real inverse of it,
# not an approximation. Returns ["", -1] if no Err node is found
# anywhere.
make a function called parse_find_error takes stmts returns err
let placement = parse_check_when_placement(stmts, 0)
if placement[0] != "" then
return placement
end
return pf_walk_list(stmts)
end
make a function called pf_walk_list takes nodes returns err
let i = 0
let n = to_num(list_len(nodes))
while i < n do
let r = pf_walk_node(nodes[i])
if r[0] != "" then
return r
end
let i = i + 1
end
return ["", -1]
end
make a function called pf_walk_pattern takes pattern returns err
let tag = pattern[0]
if tag == "Err" then
return [pattern[1], pattern[2]]
end
if tag == "PCmp" then
return pf_walk_node(pattern[2])
end
if tag == "PList" then
let subs = pattern[1]
let i = 0
let n = to_num(list_len(subs))
while i < n do
let r = pf_walk_pattern(subs[i])
if r[0] != "" then return r end
let i = i + 1
end
return ["", -1]
end
# PWild/PBind/PLit/PGlob carry no sub-expressions that could hold an Err.
return ["", -1]
end
make a function called pf_walk_node takes node returns err
let ty = node[0]
if ty == "Err" then
return [node[1], node[2]]
end
if (ty == "Num") or (ty == "Str") or (ty == "Bool") or (ty == "Var") then
return ["", -1]
end
if ty == "Bin" then
let r = pf_walk_node(node[2])
if r[0] != "" then return r end
return pf_walk_node(node[3])
end
if ty == "Un" then
return pf_walk_node(node[2])
end
if (ty == "Call") or (ty == "Closure") then
return pf_walk_list(node[2])
end
if ty == "List" then
return pf_walk_list(node[1])
end
if ty == "Index" then
let r = pf_walk_node(node[1])
if r[0] != "" then return r end
return pf_walk_node(node[2])
end
if ty == "Member" then
return pf_walk_node(node[1])
end
if ty == "Let" then
return pf_walk_node(node[2])
end
if ty == "MemberAssign" then
let r = pf_walk_node(node[1])
if r[0] != "" then return r end
return pf_walk_node(node[3])
end
if (ty == "Expr") or (ty == "Return") then
return pf_walk_node(node[1])
end
if ty == "If" then
let r = pf_walk_node(node[1])
if r[0] != "" then return r end
let r2 = pf_walk_list(node[2])
if r2[0] != "" then return r2 end
return pf_walk_list(node[3])
end
if ty == "While" then
let r = pf_walk_node(node[1])
if r[0] != "" then return r end
return pf_walk_list(node[2])
end
if ty == "Match" then
let r = pf_walk_node(node[1])
if r[0] != "" then return r end
let arms = node[2]
let i = 0
let n = to_num(list_len(arms))
while i < n do
let arm = arms[i]
let pr = pf_walk_pattern(arm[0])
if pr[0] != "" then return pr end
if arm[1] != false then
let gr = pf_walk_node(arm[1])
if gr[0] != "" then return gr end
end
let br = pf_walk_list(arm[2])
if br[0] != "" then return br end
let i = i + 1
end
return ["", -1]
end
if (ty == "Func") or (ty == "When") then
return pf_walk_list(node[2])
end
if ty == "Assert" then
return pf_walk_node(node[2])
end
if ty == "Budgeted" then
let r = pf_walk_node(node[1])
if r[0] != "" then return r end
let r2 = pf_walk_node(node[2])
if r2[0] != "" then return r2 end
return pf_walk_list(node[3])
end
if ty == "RuleDecl" then
let r = pf_walk_list(node[2])
if r[0] != "" then return r end
let body = node[3]
let i = 0
let n = to_num(list_len(body))
while i < n do
let r2 = pf_walk_list(body[i][1])
if r2[0] != "" then return r2 end
let i = i + 1
end
return ["", -1]
end
if ty == "GoalDecl" then
let deps = node[2]
let i = 0
let n = to_num(list_len(deps))
while i < n do
let r = pf_walk_list(deps[i][1])
if r[0] != "" then return r end
let i = i + 1
end
return ["", -1]
end
# Unrecognized tag, or a leaf shape with nothing further to walk (e.g.
# "Bool"/"Num"/"Var" already handled above) -- nothing more to check.
return ["", -1]
end
# `when` is a genuine top-level-only declaration -- handlers are
# collected once, statically, when the program is lowered (rust-runtime/
# src/ir/lowering.rs's own top-level scan for Stmt::When; the self-
# hosted lower.patlang mirrors this). A `when` written anywhere else
# (nested inside if/while/a function body) used to compile with NO
# error at all and simply never fire, ever, silently -- confirmed
# directly while building self_hosting/examples/microservices_demo.
# patlang: `if role == "greeter" then when greet do ... end end`
# produced a worker that received every signal and answered NONE of
# them, no diagnostic anywhere pointing at why.
#
# This is deliberate, not a limitation to work around: PatLang's
# capability-discovery system (signal_discovery.patlang) treats a
# service's *advertised* actions as a fixed description of what it
# responds to for the life of the process. If handler registration
# could vary at runtime, an honest self-description could go stale on
# its own without a new announcement ever being made -- see that
# module's own header for the existing, separate "self-reported, not
# verified" trust caveat, which this would have quietly compounded.
# Global-only handlers were chosen deliberately over allowing
# conditional registration for that reason, not just because it's
# simpler to implement (though it is) -- so this is enforced as a real
# parse error now, not merely documented as a gotcha to remember.
#
# Walks the full statement tree (top-level, then recursively into every
# If/While/Func body, each one level deeper) looking for a "When" node
# at depth > 0. Returns [message, line] on the FIRST one found, or
# ["", -1] if every `when` in the program is genuinely at top level.
make a function called parse_check_when_placement takes stmts, depth returns err
let i = 0
let n = to_num(list_len(stmts))
while i < n do
let stmt = stmts[i]
let tag = stmt[0]
if tag == "When" then
if depth > 0 then
return ["'when' can only be declared at the top level of a program -- it cannot be nested inside if/while blocks or function bodies, since handlers are registered once, statically, not conditionally at runtime", stmt[3]]
end
end
if tag == "If" then
let r1 = parse_check_when_placement(stmt[2], depth + 1)
if r1[0] != "" then
return r1
end
let r2 = parse_check_when_placement(stmt[3], depth + 1)
if r2[0] != "" then
return r2
end
end
if tag == "While" then
let r = parse_check_when_placement(stmt[2], depth + 1)
if r[0] != "" then
return r
end
end
if tag == "Match" then
let arms = stmt[2]
let ai = 0
let an = to_num(list_len(arms))
while ai < an do
let r = parse_check_when_placement(arms[ai][2], depth + 1)
if r[0] != "" then
return r
end
let ai = ai + 1
end
end
if tag == "Func" then
let r = parse_check_when_placement(stmt[3], depth + 1)
if r[0] != "" then
return r
end
end
let i = i + 1
end
return ["", -1]
end
make a function called pf_find_substr takes hay, needle returns idx
let hn = hay.length
let nn = needle.length
if nn == 0 then
return 0
end
let i = 0
while i <= hn - nn do
if substr(hay, i, nn) == needle then
return i
end
let i = i + 1
end
return -1
end
# The text of source line `line` (1-indexed), with no trailing newline.
make a function called source_line_text takes source, line returns text
let n = source.length
let cur_line = 1
let start = 0
let i = 0
while (i < n) and (cur_line < line) do
if char_code(source, i) == 10 then
let cur_line = cur_line + 1
let start = i + 1
end
let i = i + 1
end
let end_pos = start
while (end_pos < n) and (char_code(source, end_pos) != 10) do
let end_pos = end_pos + 1
end
return substr(source, start, end_pos - start)
end
# Reasoning over the CLOSED, known set of parser error messages (see the
# ~30 return sites throughout this file) -- a direct, tailored suggestion
# per message shape, not a generic catch-all, matching the same "specific
# beats generic" discipline self_hosting/lib/synthesis_lgg.patlang's
# synth5_format_diagnosis already established for induction-engine
# diagnoses. Unrecognized messages (a future error site added without a
# matching case here) fall through to a still-useful generic line.
make a function called pf_contains takes hay, needle returns found
return pf_find_substr(hay, needle) >= 0
end
make a function called suggest_parse_fix takes message returns suggestion
if pf_contains(message, "'when' can only be declared at the top level") then
return "Move this 'when' block out to the top level of the program. If you need different behavior for different cases, register the handler unconditionally and put the conditional logic INSIDE its body instead (check whatever condition you need as the handler's first statement)."
end
if pf_contains(message, "unexpected") then
return "This token wasn't expected here -- look for a missing keyword, operator, or unbalanced bracket/parenthesis just before it."
end
if pf_contains(message, "expected end after") then
return "This block needs a matching 'end' -- check that every 'do'/'then'/block-opening keyword earlier has one, and that none was consumed by a nested block closing early."
end
if pf_contains(message, "else, elif, or end") then
return "An 'if' block needs to close with 'end', or continue with 'elif <cond> then'/'else' -- check what follows the last statement in this if-block."
end
if pf_contains(message, "} after") then
return "This block needs a matching '}' -- check for a missing closing brace, or an extra '{' earlier that consumed the one meant for this block."
end
if pf_contains(message, "expected )") then
return "A '(' opened earlier is missing its matching ')' -- check argument lists and parenthesized expressions for a missing closing paren, or a stray extra argument."
end
if pf_contains(message, "expected ]") then
return "A '[' opened earlier is missing its matching ']' -- check list literals and index expressions for a missing closing bracket."
end
if pf_contains(message, "expected = after let name") then
return "A 'let' declaration needs '= <expression>' right after the variable name."
end
if pf_contains(message, "'main' is a reserved") then
return "Rename this function -- 'main' collides with the native-compiled backend's own program entry point. Pick any other name and call it explicitly from top-level code instead."
end
if pf_contains(message, "function name") or pf_contains(message, "'function' after make") then
return "Function declarations need the exact form 'make a function called NAME takes ... returns ... ... end' -- check the keywords right after 'make'."
end
if pf_contains(message, "event name") or pf_contains(message, "after when event") then
return "'when' needs an event name identifier right after it, e.g. 'when some_event do ... end'."
end
if pf_contains(message, ":-") then
return "A rule head needs either ':-' followed by its body, or a '.' for a fact with no body."
end
if pf_contains(message, "name after .") then
return "Member access ('.') needs a plain identifier right after it, e.g. 'obj.field'."
end
return "Check the syntax immediately around this point against the construct being parsed."
end
# The full human-readable diagnostic: message, source location with the
# actual offending line shown, and a suggested fix -- exactly the "report
# on it, but also suggest potential fixes" the compiler-error-reasoning
# backlog item asked for, scoped to PatLang's own errors only.
make a function called format_parse_error takes message, line, source returns text
let line_text = source_line_text(source, line)
if line_text == "" then
let line_text = "(reached end of file -- nothing follows on this line)"
end
let suggestion = suggest_parse_fix(message)
return "Parse error at line " + ("" + line) + ":\n " + line_text + "\n " + message + "\n Suggestion: " + suggestion
end
# =============================================================================
# Stage 1 self-hosted lowerer (Stage 0 compilable subset).
# Walks the list-shaped AST from lib/parser.patlang and emits list-shaped IR
# instructions. The host's compile_ir only decodes this IR and runs codegen —
# lexing, parsing, lowering, AND (via lib/codegen.patlang) code generation all
# happen in PatLang.
#
# IR shape:
# ["ProgramIR", entry, [functions], [events]]
# ["FuncIR", name, [params], [instrs], [lines]]
# ["EventIR", event, handler_name]
#
# lines: a SPARSE, statement-granularity debug table, one [pc, source_line]
# pair per Let/Expr/Return/Assert/MemberAssign statement lowered into this
# function's own instrs (not every instruction -- see parser.patlang's
# parse_stmt wrapper for which node shapes carry a line at all, and why
# Func/If/While/When/Err were deliberately left out of that set). Built for
# self_hosting/lib/interp.patlang's debugger: enough to map a paused pc back
# to "which statement", not full per-instruction provenance.
# Instructions:
# ["Const", "num"|"str"|"bool", text] ["Load", name] ["Store", name]
# ["Bin", op] ["Un", op] ["Jump", n] ["JumpIfFalse", n]
# ["CallHost", name, argc] ["Call", name, argc]
# ["MakeClosure", func_name, [captured_names]] ["CallValue", argc]
# ["BuildList", n] ["Return"]
#
# Closures: |params| do body end (Stage 1 has no brace-delimited blocks
# anywhere, so closures use the same do/end convention as while-loops rather
# than Stage 0's |params| { body }). A closure captures its ENTIRE enclosing
# locals list (over-capture, not precise free-variable analysis) as leading
# hidden parameters of a synthesized function — simpler than exact capture
# and still correct: the flat per-call locals map means a closure's own
# `let` of a same-named variable just overwrites the pre-bound captured
# value, exactly matching intended shadowing.
# =============================================================================
make a function called contains_str takes xs, s returns r
let i = 0
while i < xs.length do
if xs[i] == s then
return true
end
let i = i + 1
end
return false
end
make a function called vec_contains takes v, s returns r
let i = 0
let n = vec_len(v)
while i < n do
if vec_get(v, i) == s then
return true
end
let i = i + 1
end
return false
end
# Finds the mutability flag for the MOST RECENT declaration of `name` in
# `locals`/`mutables` (index-parallel vecs) -- a re-`let` shadows, so the
# latest declaration's flag is the one that governs a later reassignment.
make a function called find_mut_flag takes locals, mutables, name returns r
let n = vec_len(locals)
let i = n - 1
let found = false
let result = false
while (i >= 0) and (not found) do
if vec_get(locals, i) == name then
let found = true
let result = vec_get(mutables, i)
end
let i = i - 1
end
return result
end
make a function called next_closure_name returns name
let n = get("__vars", "__closure_seq")
if not n then
let n = 0
end
set_var("__closure_seq", n + 1)
return "__closure_" + n
end
make a function called next_budgeted_name returns name
let n = get("__vars", "__budgeted_seq")
if not n then
let n = 0
end
set_var("__budgeted_seq", n + 1)
return "__budgeted_" + n
end
make a function called next_match_name returns name
let n = get("__vars", "__match_seq")
if not n then
let n = 0
end
set_var("__match_seq", n + 1)
return "__match_" + n
end
make a function called next_activate_name returns name
let n = get("__vars", "__activate_seq")
if not n then
let n = 0
end
set_var("__activate_seq", n + 1)
return "__act_" + n
end
# Nesting-depth counter (not a bool, so a budgeted block nested inside
# another still counts as "inside") tracking whether the statement/body
# currently being lowered is lexically inside a budgeted(...) { ... } block
# -- while-loop back-edges check this to decide whether to inject a
# budget_check() call. Stored via the same object-store convention as
# __closure_seq/__budgeted_seq since lower_*'s functions are plain
# functions, not methods on a stateful struct (unlike Stage 0's Lowerer).
make a function called budgeted_depth_get returns n
let n = get("__vars", "__budgeted_depth")
if not n then
return 0
end
return n
end
make a function called budgeted_depth_inc returns done
set_var("__budgeted_depth", budgeted_depth_get() + 1)
return true
end
make a function called budgeted_depth_dec returns done
set_var("__budgeted_depth", budgeted_depth_get() - 1)
return true
end
# lower_expr(node, code, fns, locals, pending, lines) -> code
# fns: list of top-level function names (static call vs host call)
# locals: vec of names currently bound in the enclosing scope (for
# disambiguating a call through a closure-valued variable, and
# for over-capturing into new closures)
# pending: vec of synthesized ["FuncIR", ...] nodes from closure literals,
# merged into the program's function list once lowering finishes
make a function called lower_expr takes node, code, fns, locals, pending, lines returns out
let ty = node[0]
if ty == "Num" then
vec_push(code, ["Const", "num", node[1]])
return code
else
if ty == "Str" then
vec_push(code, ["Const", "str", node[1]])
return code
else
if ty == "Bool" then
vec_push(code, ["Const", "bool", node[1]])
return code
else
if ty == "Var" then
# `unit` is the language's own literal for the Unit/void value
# (a program can write it directly as an expression, e.g.
# `return unit` -- see self_hosting/lib/interp.patlang's
# interp_const), so a bare `unit` that ISN'T a bound variable
# lowers to a Unit constant rather than a variable load. Found
# during the all-self-hosted fixpoint self-compile: without
# this, codegen_x64.patlang's own undefined-variable check
# correctly rejected the resulting unresolved `Load "unit"`.
#
# The `vec_contains(locals, ...)` guard is load-bearing, and
# NOT what rust-runtime/src/ir/lowering.rs's own Expr::
# Identifier arm does -- it special-cases the name
# unconditionally. Matching that unconditionally here is a
# real regression, because `unit` is also just an ordinary,
# legal variable name that this very codebase uses: x64_
# compile_unit.patlang's own x64_make_compile_unit declares
# `returns unit`, does `let unit = new("CompileUnit", ...)`,
# and ends `return unit`. Lowering that final read as a Unit
# constant makes the function silently return void instead of
# the object it just built -- every per-function compile unit
# then came back empty, and every --x64 build died with a
# confusing "LINK ERROR: undefined symbol '<first function>'"
# far downstream of the actual cause. A bound name always wins
# over the literal.
if (node[1] == "unit") and (not vec_contains(locals, "unit")) then
vec_push(code, ["Const", "unit", ""])
return code
end
vec_push(code, ["Load", node[1]])
return code
else
if ty == "Bin" then
if node[1] == "and" then
# short-circuit: false when lhs falsey, else truthiness of rhs
let code = lower_expr(node[2], code, fns, locals, pending, lines)
vec_push(code, ["Un", "not"])
let jif = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
vec_push(code, ["Const", "bool", "false"])
let jmp = vec_len(code)
vec_push(code, ["Jump", 0])
vec_set(code, jif, ["JumpIfFalse", vec_len(code)])
let code = lower_expr(node[3], code, fns, locals, pending, lines)
vec_push(code, ["Un", "not"])
vec_push(code, ["Un", "not"])
vec_set(code, jmp, ["Jump", vec_len(code)])
return code
else
if node[1] == "or" then
# short-circuit: true when lhs truthy, else truthiness of rhs
let code = lower_expr(node[2], code, fns, locals, pending, lines)
vec_push(code, ["Un", "not"])
let jif = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
let code = lower_expr(node[3], code, fns, locals, pending, lines)
vec_push(code, ["Un", "not"])
vec_push(code, ["Un", "not"])
let jmp = vec_len(code)
vec_push(code, ["Jump", 0])
vec_set(code, jif, ["JumpIfFalse", vec_len(code)])
vec_push(code, ["Const", "bool", "true"])
vec_set(code, jmp, ["Jump", vec_len(code)])
return code
else
let code = lower_expr(node[2], code, fns, locals, pending, lines)
let code = lower_expr(node[3], code, fns, locals, pending, lines)
vec_push(code, ["Bin", node[1]])
return code
end
end
else
if ty == "Un" then
let code = lower_expr(node[2], code, fns, locals, pending, lines)
vec_push(code, ["Un", node[1]])
return code
else
if ty == "Call" then
let callee = node[1]
let args = node[2]
if contains_str(fns, callee) then
let i = 0
while i < args.length do
let code = lower_expr(args[i], code, fns, locals, pending, lines)
let i = i + 1
end
vec_push(code, ["Call", callee, args.length])
return code
else
if vec_contains(locals, callee) then
# dynamic call through a local variable holding a closure
vec_push(code, ["Load", callee])
let i = 0
while i < args.length do
let code = lower_expr(args[i], code, fns, locals, pending, lines)
let i = i + 1
end
vec_push(code, ["CallValue", args.length])
return code
else
let i = 0
while i < args.length do
let code = lower_expr(args[i], code, fns, locals, pending, lines)
let i = i + 1
end
vec_push(code, ["CallHost", callee, args.length])
return code
end
end
else
if ty == "List" then
let items = node[1]
let i = 0
while i < items.length do
let code = lower_expr(items[i], code, fns, locals, pending, lines)
let i = i + 1
end
vec_push(code, ["BuildList", items.length])
return code
else
if ty == "Index" then
let code = lower_expr(node[1], code, fns, locals, pending, lines)
let code = lower_expr(node[2], code, fns, locals, pending, lines)
vec_push(code, ["CallHost", "list_get", 2])
return code
else
if ty == "Member" then
let code = lower_expr(node[1], code, fns, locals, pending, lines)
if (node[2] == "length") or (node[2] == "len") then
vec_push(code, ["CallHost", "len", 1])
return code
else
vec_push(code, ["Const", "str", node[2]])
vec_push(code, ["CallHost", "get", 2])
return code
end
else
if ty == "MethodCall" then
# obj.method(args) -- send(obj, "method", ...args),
# the same host call obj.prop = value already uses
# on the write side (see Stmt::MemberAssign below).
let object = node[1]
let method = node[2]
let args = node[3]
let code = lower_expr(object, code, fns, locals, pending, lines)
vec_push(code, ["Const", "str", method])
let i = 0
while i < args.length do
let code = lower_expr(args[i], code, fns, locals, pending, lines)
let i = i + 1
end
vec_push(code, ["CallHost", "send", 2 + args.length])
return code
else
if ty == "Closure" then
return lower_closure_literal(node[1], node[2], code, fns, locals, pending, lines)
else
if ty == "Budgeted" then
return lower_budgeted_literal(node[1], node[2], node[3], code, fns, locals, pending, lines)
else
# unknown expression: lower as unit-ish empty string
vec_push(code, ["Const", "str", ""])
return code
end
end
end
end
end
end
end
end
end
end
end
end
end
end
# Lowers a closure literal: synthesizes a Function (params = the enclosing
# scope's current locals ++ the closure's own params) queued in `pending`,
# and emits the Load.../MakeClosure sequence at the creation site into `code`.
make a function called lower_closure_literal takes params, body, code, fns, locals, pending, lines returns out
let captured = vec_to_list(locals)
let func_name = next_closure_name()
let all_params = []
let i = 0
while i < captured.length do
let all_params = list_push(all_params, captured[i])
let i = i + 1
end
let i = 0
while i < params.length do
let all_params = list_push(all_params, params[i])
let i = i + 1
end
let inner_locals = vec_new()
let inner_mutables = vec_new()
let i = 0
while i < all_params.length do
vec_push(inner_locals, all_params[i])
vec_push(inner_mutables, true)
let i = i + 1
end
# A closure's body is its own function with its own pc-space, so it gets
# its own fresh lines table -- the threaded-in `lines` param belongs to
# the OUTER code this closure literal is embedded in, and recording the
# inner body's statements into it would attach the wrong function's pcs
# to those source lines.
let inner_lines = vec_new()
let inner_code = lower_block(body, vec_new(), fns, func_name, inner_locals, inner_mutables, pending, inner_lines, true)
vec_push(inner_code, ["Return"])
vec_push(pending, ["FuncIR", func_name, all_params, vec_to_list(inner_code), vec_to_list(inner_lines)])
# At the creation site: push captured values in the SAME order used for
# all_params's leading section, then bundle them.
let i = 0
while i < captured.length do
vec_push(code, ["Load", captured[i]])
let i = i + 1
end
vec_push(code, ["MakeClosure", func_name, captured])
return code
end
# Lowers budgeted(ms[, existing]) { body }: synthesizes a function (like a
# closure) run via an implicit fiber. Unlike closures, captured locals are
# bundled into a SINGLE List argument rather than passed as leading params --
# fiber_new only ever passes one initial argument to the spawned function --
# and the synthesized function's own prologue unpacks them again via
# list_get/Store. `existing` defaults to Bool(false) at parse time when
# omitted from source (see parser.patlang), which budgeted_run treats the
# same as Unit: "no handle yet, start a new fiber".
make a function called lower_budgeted_literal takes msNode, existingNode, body, code, fns, locals, pending, lines returns out
let captured = vec_to_list(locals)
let func_name = next_budgeted_name()
let inner_locals = vec_new()
let inner_mutables = vec_new()
vec_push(inner_locals, "__captured")
vec_push(inner_mutables, true)
let i = 0
while i < captured.length do
vec_push(inner_locals, captured[i])
vec_push(inner_mutables, true)
let i = i + 1
end
let inner_code = vec_new()
let i = 0
while i < captured.length do
vec_push(inner_code, ["Load", "__captured"])
vec_push(inner_code, ["Const", "num", "" + i])
vec_push(inner_code, ["CallHost", "list_get", 2])
vec_push(inner_code, ["Store", captured[i]])
let i = i + 1
end
# Same reasoning as lower_closure_literal: this is a separate function
# with its own pc-space, so it needs its own fresh lines table rather
# than the caller's.
let inner_lines = vec_new()
budgeted_depth_inc()
let inner_code = lower_block(body, inner_code, fns, func_name, inner_locals, inner_mutables, pending, inner_lines, true)
budgeted_depth_dec()
vec_push(inner_code, ["Return"])
vec_push(pending, ["FuncIR", func_name, ["__captured"], vec_to_list(inner_code), vec_to_list(inner_lines)])
# budgeted_run(ms, func_name, captured_list, existing)
let code = lower_expr(msNode, code, fns, locals, pending, lines)
vec_push(code, ["Const", "str", func_name])
let i = 0
while i < captured.length do
vec_push(code, ["Load", captured[i]])
let i = i + 1
end
vec_push(code, ["BuildList", captured.length])
let code = lower_expr(existingNode, code, fns, locals, pending, lines)
vec_push(code, ["CallHost", "budgeted_run", 4])
return code
end
# Lowers `activate(PLAN)` (the desugared form of `activate PLAN`, see
# parser.patlang's parse_primary) -- mirrors ir/lowering.rs's
# lower_activate exactly: host functions can't call back into the
# interpreter to invoke a closure, so this is synthesized as ordinary
# control-flow AST (Let/While/If/Call) fed through the normal
# lower_block/lower_stmt path, rather than a new IR instruction. Each
# plan step's bound closure receives its bound argument values as a
# single List (not positionally splatted -- a plan step's real arity is
# only known once the planner has run), and the loop stops immediately,
# reporting failure, the moment one returns false.
make a function called lower_activate_stmt takes plan_node, code, fns, fname, locals, mutables, pending, lines returns out
let n = next_activate_name()
let plan_var = "__act_plan_" + n
let i_var = "__act_i_" + n
let ok_var = "__act_ok_" + n
let label_var = "__act_label_" + n
let fn_var = "__act_fn_" + n
let args_var = "__act_args_" + n
let r_var = "__act_r_" + n
let synth = [
["Let", plan_var, plan_node, false, false],
["Let", i_var, ["Num", "0"], false, true],
["Let", ok_var, ["Bool", "true"], false, true],
["While",
["Bin", "and",
["Var", ok_var],
["Bin", "<", ["Var", i_var], ["Call", "to_num", [["Call", "list_len", [["Var", plan_var]]]]]]
],
[
["Let", label_var, ["Index", ["Var", plan_var], ["Var", i_var]], false, false],
["Let", fn_var, ["Call", "action_lookup", [["Call", "action_base_name", [["Var", label_var]]]]], false, false],
["Let", args_var, ["Call", "action_label_args", [["Var", label_var]]], false, false],
["Let", r_var, ["Call", fn_var, [["Var", args_var]]], false, false],
["If", ["Bin", "==", ["Var", r_var], ["Bool", "false"]],
[["Let", ok_var, ["Bool", "false"], true, false]],
[]
],
["Let", i_var, ["Bin", "+", ["Var", i_var], ["Num", "1"]], true, false]
]
]
]
let code = lower_block(synth, code, fns, fname, locals, mutables, pending, lines, false)
return lower_expr(["Var", ok_var], code, fns, locals, pending, lines)
end
# Render a rule head/body arg as the compile-time string token rule_add
# expects: a bare identifier/number renders as its own text, but a string
# literal renders as its raw content, not source-quoted -- mirrors
# ir/lowering.rs's own rule_arg_text.
make a function called rule_arg_str takes node returns s
return node[1]
end
# is_side_effect_free_node(node) -> mirrors ir/lowering.rs's own
# is_side_effect_free_expr exactly, on this file's list-shaped AST instead
# of the native Expr enum: true for a shape that structurally CANNOT have a
# side effect of its own (a literal, arithmetic/comparison over such, a
# bare variable read, a list literal of such) -- everything that dispatches
# through a host/function call, a message send, or a closure creation is
# false, since those already speak for themselves via whatever they do; see
# lower_stmt's "Expr" arm below for why this distinction matters (auto-
# printing a Call's own incidental return value, e.g. print(x)'s Unit,
# would be noise, not the intended feature).
make a function called is_side_effect_free_node takes node returns r
let ty = node[0]
if (ty == "Num") or (ty == "Str") or (ty == "Bool") or (ty == "Var") then
return true
end
if ty == "Un" then
return is_side_effect_free_node(node[2])
end
if ty == "Bin" then
return is_side_effect_free_node(node[2]) and is_side_effect_free_node(node[3])
end
if ty == "List" then
let items = node[1]
let i = 0
let all_free = true
while i < items.length do
if is_side_effect_free_node(items[i]) == false then
let all_free = false
end
let i = i + 1
end
return all_free
end
return false
end
# wants_value: mirrors ir/lowering.rs's own `wants_value` thread exactly --
# true when this statement is in TAIL POSITION of a block/function that
# itself wants a value (the "blocks and functions return their last value"
# feature), meaning its own computed value should be left on the stack for
# the caller instead of discarded/auto-printed. See lower_block below for
# how a statement LIST decides which single statement (if any) gets
# wants_value=true.
# =============================================================================
# match/case (issue #44). Load-bearing design decision: `match` is PURE
# SYNTACTIC SUGAR -- compile_pattern below only ever produces ordinary AST
# nodes (Bin/Call/Index/Member/Var/Str/Num) that lower_expr/lower_stmt
# already know how to lower, and lower_match itself only ever emits the
# same Const/Load/Store/Jump/JumpIfFalse instructions an if/elif chain
# already uses (see lower_stmt's existing "If" arm just below, which this
# mirrors exactly). No new Instr variant -- see the plan's own design
# rationale (issue #44 follow-up) for why: every execution path
# (interp.patlang, codegen.patlang/codegen_x64.patlang) gets `match` for
# free since they never see anything but instructions they already handle.
#
# compile_pattern(pattern, scrutineeNode) -> [testExprNode, [letStmtNodes]]
# testExprNode: an ordinary boolean-valued AST expr node.
# letStmtNodes: ["Let", name, valueNode, false, false] nodes to run BEFORE
# the arm body, once the test has passed (a PBind's binding,
# or nested bindings from a PList's sub-patterns).
# =============================================================================
make a function called compile_pattern takes pattern, scrutineeNode returns r
let tag = pattern[0]
if tag == "PWild" then
return [["Bool", "true"], []]
else
if tag == "PBind" then
return [["Bool", "true"], [["Let", pattern[1], scrutineeNode, false, false]]]
else
if tag == "PLit" then
return [["Bin", "==", scrutineeNode, pattern[1]], []]
else
if tag == "PCmp" then
return [["Bin", pattern[1], scrutineeNode, pattern[2]], []]
else
if tag == "PGlob" then
return [["Call", "glob_match", [scrutineeNode, ["Str", pattern[1]]]], []]
else
if tag == "PList" then
return compile_list_pattern(pattern[1], scrutineeNode)
else
# Unreachable for a successfully-parsed pattern; a stray
# ["Err", ...] pattern (already surfaced separately by
# parser.patlang's pf_walk_pattern) falls back to an
# always-false test rather than crashing the lowerer.
return [["Bool", "false"], []]
end
end
end
end
end
end
end
make a function called compile_list_pattern takes subs, scrutineeNode returns r
let n = to_num(list_len(subs))
let test = ["Bin", "and",
["Bin", "==", ["Call", "type_of", [scrutineeNode]], ["Str", "list"]],
["Bin", "==", ["Member", scrutineeNode, "length"], ["Num", "" + n]]]
let lets = []
let i = 0
while i < n do
let elemNode = ["Index", scrutineeNode, ["Num", "" + i]]
let sub = compile_pattern(subs[i], elemNode)
let test = ["Bin", "and", test, sub[0]]
let j = 0
while j < sub[1].length do
let lets = list_push(lets, sub[1][j])
let j = j + 1
end
let i = i + 1
end
return [test, lets]
end
# Mirrors lower_stmt's "If" arm exactly, generalized to N arms with a
# guaranteed-fail tail when no arm's pattern (+ optional guard) matches --
# a runtime error naming the failure (via contract_check, the same
# design-by-contract host primitive already used for other "this should be
# unreachable" cases in this file, e.g. immutable-reassignment above), NOT
# a silent no-op: PatLang has no static type system to check exhaustiveness
# against, so an unmatched scrutinee with no `_` arm is a genuine bug at
# runtime, not a case to swallow quietly (decided explicitly in the issue
# #44 follow-up plan, not left implicit).
make a function called lower_match takes node, code, fns, fname, locals, mutables, pending, lines, wants_value returns out
let scrutineeExpr = node[1]
let arms = node[2]
# Store the scrutinee ONCE into a fresh synthesized local -- same
# reasoning as this file's other __closure_N/__budgeted_N synthesized
# names: re-evaluating an expression with side effects once per arm
# (e.g. `match next_event() do ...`) would be a real correctness bug,
# not just wasted work.
let tmp = next_match_name()
let code = lower_expr(scrutineeExpr, code, fns, locals, pending, lines)
vec_push(code, ["Store", tmp])
vec_push(locals, tmp)
vec_push(mutables, false)
let scrutineeVar = ["Var", tmp]
let endJumps = vec_new()
let i = 0
let n = to_num(list_len(arms))
while i < n do
let arm = arms[i]
let pattern = arm[0]
let guard = arm[1]
let body = arm[2]
let comp = compile_pattern(pattern, scrutineeVar)
let testNode = comp[0]
let letStmts = comp[1]
let code = lower_expr(testNode, code, fns, locals, pending, lines)
let jifPattern = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
# Bindings the pattern introduced (e.g. PBind's `let v = scrutinee`)
# must be emitted BEFORE the guard is evaluated, not folded into one
# `pattern_test and guard` expression -- a `when` guard is documented
# to reference bindings its own pattern just introduced (e.g. `case n
# when n > 100`), and `n` doesn't exist as a real local until this
# Store actually runs. Evaluating the AND as a single expression would
# load an unbound `n` while still building the LEFT side's truth value,
# silently reading Unit/stale-load instead of the real scrutinee no
# matter what the guard says. Found via a real failing run (guarded(200)
# returning "small:200" instead of "big:200"), not by inspection.
let j = 0
while j < letStmts.length do
let code = lower_stmt(letStmts[j], code, fns, fname, locals, mutables, pending, lines, false)
let j = j + 1
end
let jifGuard = -1
if guard != false then
let code = lower_expr(guard, code, fns, locals, pending, lines)
let jifGuard = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
end
let code = lower_block(body, code, fns, fname, locals, mutables, pending, lines, wants_value)
let jmp = vec_len(code)
vec_push(code, ["Jump", 0])
vec_push(endJumps, jmp)
let next_arm_pc = vec_len(code)
vec_set(code, jifPattern, ["JumpIfFalse", next_arm_pc])
if jifGuard >= 0 then
vec_set(code, jifGuard, ["JumpIfFalse", next_arm_pc])
end
let i = i + 1
end
# No arm matched: fatal, mirrors PatLang's existing "no try/catch, host
# errors are fatal" philosophy (contract_check's ok=false arg always
# errors -- see its use for immutable reassignment above -- so nothing
# after this point ever actually runs; the Store/Const that follow it
# exist only to keep the instruction stream's stack shape consistent
# with the matched-arm paths for any backend that reasons about it
# statically, matching the same dead-but-shape-correct pattern the
# immutable-reassignment path above already relies on).
vec_push(code, ["Const", "str", fname])
vec_push(code, ["Const", "str", "assert"])
vec_push(code, ["Const", "str", "match: no case matched the scrutinee value"])
vec_push(code, ["Const", "bool", "false"])
vec_push(code, ["CallHost", "contract_check", 4])
vec_push(code, ["Store", "__discard"])
if wants_value then
vec_push(code, ["Const", "unit", ""])
end
let end_pc = vec_len(code)
let i = 0
while i < vec_len(endJumps) do
vec_set(code, vec_get(endJumps, i), ["Jump", end_pc])
let i = i + 1
end
return code
end
make a function called lower_stmt takes node, code, fns, fname, locals, mutables, pending, lines, wants_value returns out
let ty = node[0]
# Record this statement's source line BEFORE lowering it, at the pc it's
# about to start emitting at -- only Let/Expr/Return/Assert/MemberAssign
# carry one (see parser.patlang's parse_stmt wrapper); node[node.length-1]
# is that line regardless of the node's own arity, since it was always
# appended as the trailing element there.
if (ty == "Let") or (ty == "Expr") or (ty == "Return") or (ty == "Assert") or (ty == "MemberAssign") then
vec_push(lines, [vec_len(code), node[node.length - 1]])
end
if ty == "Let" then
let valNode = node[2]
if (valNode[0] == "Call") and (valNode[1] == "activate") and (to_num(valNode[2].length) == 1) then
let code = lower_activate_stmt(valNode[2][0], code, fns, fname, locals, mutables, pending, lines)
else
let code = lower_expr(node[2], code, fns, locals, pending, lines)
end
let isReassign = node[3]
let isMut = node[4]
if isReassign then
if vec_contains(locals, node[1]) then
let flag = find_mut_flag(locals, mutables, node[1])
if not flag then
# Statically-known immutable reassignment: emit a contract_check
# that always fails at this point (mirrors the native Stage 0
# lowerer's approach -- lower_program is infallible here too, so
# this is enforced as a guaranteed-fail assertion at the
# reassignment site rather than a hard compile error).
vec_push(code, ["Const", "str", fname])
vec_push(code, ["Const", "str", "assert"])
vec_push(code, ["Const", "str", "cannot assign twice to immutable variable `" + node[1] + "` (declare it `let mut " + node[1] + "` to allow reassignment)"])
vec_push(code, ["Const", "bool", "false"])
vec_push(code, ["CallHost", "contract_check", 4])
end
vec_push(code, ["Store", node[1]])
else
# Not seen in this scope (e.g. a captured/outer-scope name this flat
# tracker can't see) -- fall back to the old permissive behaviour.
vec_push(code, ["Store", node[1]])
vec_push(locals, node[1])
vec_push(mutables, true)
end
else
# `let` / `let mut`: always allowed to introduce or shadow.
vec_push(code, ["Store", node[1]])
vec_push(locals, node[1])
vec_push(mutables, isMut)
end
# Ruby-style assignment-as-expression (mirrors ir/lowering.rs's own
# Stmt::Let arm): Store already consumed the value off the stack, so
# when this `let` is in tail position of a block that wants a value,
# reload it -- the assigned value IS the block's value.
if wants_value then
vec_push(code, ["Load", node[1]])
end
return code
else
if ty == "MemberAssign" then
# obj.prop = value -- lowers to send(obj, "set", prop, value), the
# same host call the object system already uses for explicit
# send("obj","set","prop",value) calls (matches ir/lowering.rs's
# native Stmt::MemberAssign arm). Previously called a nonexistent
# host fn "set" (3 args) instead -- never caught because
# MemberAssign was itself dead code in the native parser.rs until
# a real parser bug (bare '=' always consumed as equality before
# this ever got a chance) was fixed; nothing had ever exercised
# this lowering path before.
let code = lower_expr(node[1], code, fns, locals, pending, lines)
vec_push(code, ["Const", "str", "set"])
vec_push(code, ["Const", "str", node[2]])
let code = lower_expr(node[3], code, fns, locals, pending, lines)
vec_push(code, ["CallHost", "send", 4])
if wants_value == false then
vec_push(code, ["Store", "__discard"])
end
return code
else
if ty == "Expr" then
let exprNode = node[1]
if (exprNode[0] == "Call") and (exprNode[1] == "activate") and (to_num(exprNode[2].length) == 1) then
let code = lower_activate_stmt(exprNode[2][0], code, fns, fname, locals, mutables, pending, lines)
else
let code = lower_expr(node[1], code, fns, locals, pending, lines)
end
# GitHub #31 fix (mirrors ir/lowering.rs's Stmt::ExprStmt): every
# function body ends with an explicit Return, so a bare expression
# statement's pushed value is never actually consumed UNLESS this
# statement is in tail position of a block/function that wants it
# (wants_value) -- the interpreter's heap Vec-based operand stack
# tolerates leaving it regardless, but the x64 backend's real
# rsp-based stack overflows once a large loop repeats an unassigned
# call statement enough times (traced via WinDbg to
# expand_includes_at_depth's ~30,000-line loop). When unconsumed:
# auto-print (the "echoed only if not consumed" feature) ONLY for a
# structurally side-effect-free shape -- a Call (e.g. `activate PLAN`
# itself, or an ordinary print(x)) already speaks for itself via its
# own side effect, so it's silently discarded instead, exactly as
# before this feature existed (mirrors ir/lowering.rs's own
# discard_or_use / is_side_effect_free_expr split).
if wants_value then
return code
end
if is_side_effect_free_node(exprNode) then
vec_push(code, ["CallHost", "print", 1])
return code
end
vec_push(code, ["Store", "__discard"])
return code
else
if ty == "Print" then
let code = lower_expr(node[1], code, fns, locals, pending, lines)
vec_push(code, ["CallHost", "print", 1])
if wants_value == false then
vec_push(code, ["Store", "__discard"])
end
return code
else
if ty == "Return" then
let code = lower_expr(node[1], code, fns, locals, pending, lines)
vec_push(code, ["Return"])
return code
else
if ty == "Match" then
return lower_match(node, code, fns, fname, locals, mutables, pending, lines, wants_value)
else
if ty == "If" then
let code = lower_expr(node[1], code, fns, locals, pending, lines)
let jif = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
let code = lower_block(node[2], code, fns, fname, locals, mutables, pending, lines, wants_value)
let jmp = vec_len(code)
vec_push(code, ["Jump", 0])
vec_set(code, jif, ["JumpIfFalse", vec_len(code)])
# node[3] (the else branch) is always a list, `[]` when the
# source had no `else` -- lower_block's own empty-list handling
# already pushes Unit when wants_value, so the "no else but the
# false path needs a value too" case falls out for free.
let code = lower_block(node[3], code, fns, fname, locals, mutables, pending, lines, wants_value)
vec_set(code, jmp, ["Jump", vec_len(code)])
return code
else
if ty == "While" then
if wants_value then
# A while loop's value is Unit on zero iterations, otherwise
# its last-run iteration's tail value -- mirrors
# ir/lowering.rs's While arm exactly: push the zero-
# iteration default up front, then each iteration that
# actually runs pops the previous placeholder/prior value
# before pushing its own, so exactly one "current result"
# slot exists on the stack the whole time.
vec_push(code, ["Const", "unit", ""])
let start = vec_len(code)
let code = lower_expr(node[1], code, fns, locals, pending, lines)
let jif = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
vec_push(code, ["Store", "__discard"])
let code = lower_block(node[2], code, fns, fname, locals, mutables, pending, lines, true)
if budgeted_depth_get() > 0 then
# REAL BUG FOUND AND FIXED (caught by a real stack
# overflow, not review -- see budgeted_run's own
# x64 verification notes): CallHost always pushes
# exactly one result, but budget_check's own return
# value is never used here, and nothing was popping
# it -- on this wants_value=true path specifically it
# also broke the "exactly one current-result slot"
# invariant this loop's own header comment describes,
# stacking a second, permanent value on top of it
# every single iteration. A tight loop with thousands
# of iterations per budgeted() timeslice turns that
# into megabytes of unreclaimed stack in minutes.
vec_push(code, ["CallHost", "budget_check", 0])
vec_push(code, ["Store", "__discard"])
end
vec_push(code, ["Jump", start])
vec_set(code, jif, ["JumpIfFalse", vec_len(code)])
return code
else
let start = vec_len(code)
let code = lower_expr(node[1], code, fns, locals, pending, lines)
let jif = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
let code = lower_block(node[2], code, fns, fname, locals, mutables, pending, lines, false)
# Lexically inside a budgeted(ms) { ... } block: check the time
# budget just before looping back (mirrors lowering.rs's
# equivalent Stage 0 instrumentation). Store "__discard"
# pops budget_check's own unused return value -- see the
# wants_value=true branch just above for the real bug
# this fixes (a real stack overflow, not a review catch).
if budgeted_depth_get() > 0 then
vec_push(code, ["CallHost", "budget_check", 0])
vec_push(code, ["Store", "__discard"])
end
vec_push(code, ["Jump", start])
vec_set(code, jif, ["JumpIfFalse", vec_len(code)])
return code
end
else
if ty == "Assert" then
# contract_check(func_name, kind, text, ok) — args pushed in
# order, ok (the evaluated condition) last
vec_push(code, ["Const", "str", fname])
vec_push(code, ["Const", "str", node[1]])
vec_push(code, ["Const", "str", ast_to_str(node[2])])
let code = lower_expr(node[2], code, fns, locals, pending, lines)
vec_push(code, ["CallHost", "contract_check", 4])
if wants_value == false then
vec_push(code, ["Store", "__discard"])
end
return code
else
if ty == "RuleDecl" then
# Sugar: lowers to exactly the Instr sequence a hand-written
# rule_add(head_pred, [head_args...], [[pred,[args...]], ...])
# call already produces -- mirrors ir/lowering.rs's own
# Stmt::RuleDecl arm. Args are compile-time string TOKENS
# (a bare rule-head `X` is a logic-variable name, not a
# local-variable reference to evaluate).
let head_pred = node[1]
let head_args = node[2]
let body = node[3]
vec_push(code, ["Const", "str", head_pred])
let i = 0
while i < head_args.length do
vec_push(code, ["Const", "str", rule_arg_str(head_args[i])])
let i = i + 1
end
vec_push(code, ["BuildList", head_args.length])
let j = 0
while j < body.length do
let bodygoal = body[j]
let pred = bodygoal[0]
let args = bodygoal[1]
vec_push(code, ["Const", "str", pred])
let k = 0
while k < args.length do
vec_push(code, ["Const", "str", rule_arg_str(args[k])])
let k = k + 1
end
vec_push(code, ["BuildList", args.length])
vec_push(code, ["BuildList", 2])
let j = j + 1
end
vec_push(code, ["BuildList", body.length])
vec_push(code, ["CallHost", "rule_add", 3])
if wants_value == false then
vec_push(code, ["Store", "__discard"])
end
return code
else
if ty == "GoalDecl" then
# Sugar: lowers to exactly the Instr sequence a hand-
# written goal_def(NAME, [[pred,[args...]], ...]) call
# already produces -- mirrors ir/lowering.rs's
# Stmt::GoalDecl arm and the RuleDecl arm just above
# (dep args are compile-time string tokens too, same
# convention as a rule body's goal args).
let gname = node[1]
let deps = node[2]
vec_push(code, ["Const", "str", gname])
let j = 0
while j < deps.length do
let depgoal = deps[j]
let pred = depgoal[0]
let args = depgoal[1]
vec_push(code, ["Const", "str", pred])
let k = 0
while k < args.length do
vec_push(code, ["Const", "str", rule_arg_str(args[k])])
let k = k + 1
end
vec_push(code, ["BuildList", args.length])
vec_push(code, ["BuildList", 2])
let j = j + 1
end
vec_push(code, ["BuildList", deps.length])
vec_push(code, ["CallHost", "goal_def", 2])
if wants_value == false then
vec_push(code, ["Store", "__discard"])
end
return code
else
if ty == "ClassDecl" then
# Slice 1+2+3 of the classes/traits/inheritance
# feature (see the "synchronous-questing-metcalfe"
# plan) -- mirrors ir/lowering.rs's Stmt::ClassDecl
# arm exactly. Field defaults are real expressions,
# evaluated via the normal lower_expr path. Methods
# (Slice 2) become genuine closures via
# lower_closure_literal, each with an implicit
# leading "self" param, same shape as `when` blocks.
# Trait names (Slice 3) are compile-time string
# tokens, same convention as RuleDecl/GoalDecl dep
# args -- a trait reference is a registry lookup
# key, not an expression to evaluate.
let cname = node[1]
let cparent = node[2]
let cfields = node[3]
let cmethods = node[4]
let ctraits = node[5]
vec_push(code, ["Const", "str", cname])
vec_push(code, ["Const", "str", cparent])
let j = 0
while j < cfields.length do
let fld = cfields[j]
vec_push(code, ["Const", "str", fld[0]])
let code = lower_expr(fld[1], code, fns, locals, pending, lines)
vec_push(code, ["BuildList", 2])
let j = j + 1
end
vec_push(code, ["BuildList", cfields.length])
let j = 0
while j < cmethods.length do
let mth = cmethods[j]
let mname = mth[0]
let mparams = mth[1]
let mbody = mth[2]
vec_push(code, ["Const", "str", mname])
let full_params = ["self"]
let k = 0
while k < mparams.length do
let full_params = list_push(full_params, mparams[k])
let k = k + 1
end
let code = lower_closure_literal(full_params, mbody, code, fns, locals, pending, lines)
vec_push(code, ["BuildList", 2])
let j = j + 1
end
vec_push(code, ["BuildList", cmethods.length])
let j = 0
while j < ctraits.length do
vec_push(code, ["Const", "str", ctraits[j]])
let j = j + 1
end
vec_push(code, ["BuildList", ctraits.length])
vec_push(code, ["CallHost", "class_def", 5])
if wants_value == false then
vec_push(code, ["Store", "__discard"])
end
return code
else
# unknown statement: no code -- but a tail position
# that wants a value still needs something on the
# stack (mirrors ir/lowering.rs's own catch-all arm).
if wants_value then
vec_push(code, ["Const", "unit", ""])
end
return code
end
end
end
end
end
end
end
end
end
end
end
end
end
# Mirrors ir/lowering.rs's own lower_stmt_list exactly: lowers a statement
# LIST as a unit. Every statement but the last gets wants_value=false; the
# last inherits the list's own wants_value, so a value genuinely flows out
# of the block only from its tail position. An empty list that wants a
# value produces Unit (the block's value when it has no statements at all).
make a function called lower_block takes stmts, code, fns, fname, locals, mutables, pending, lines, wants_value returns out
if stmts.length == 0 then
if wants_value then
vec_push(code, ["Const", "unit", ""])
end
return code
end
let last = stmts.length - 1
let i = 0
while i < stmts.length do
let stmt_wants_value = (i == last) and wants_value
let code = lower_stmt(stmts[i], code, fns, fname, locals, mutables, pending, lines, stmt_wants_value)
let i = i + 1
end
return code
end
make a function called collect_fns takes stmts returns fns
let fns = []
let i = 0
while i < stmts.length do
let s = stmts[i]
if s[0] == "Func" then
let fns = list_push(fns, s[1])
end
let i = i + 1
end
return fns
end
# body_assigns_name(stmts, name) -> mirrors rust-runtime/src/parser.rs's
# own body_assigns_name exactly, on this file's list-shaped statement AST:
# true if `name` is `let`-assigned anywhere in `stmts`, at any nesting
# depth inside if/while branches (a shallow top-level-only scan would miss
# the common `if cond then let r = X else let r = Y end` shape). Used below
# to decide whether the `returns NAME` hint's synthesized trailing return
# should fire -- see that call site's own comment for why unconditionally
# synthesizing it would silently defeat implicit-last-value-return for any
# function that never touches NAME at all.
make a function called body_assigns_name takes stmts, name returns r
let i = 0
while i < stmts.length do
let s = stmts[i]
if (s[0] == "Let") and (s[1] == name) then
return true
end
if s[0] == "If" then
if body_assigns_name(s[2], name) then
return true
end
if body_assigns_name(s[3], name) then
return true
end
end
if s[0] == "While" then
if body_assigns_name(s[2], name) then
return true
end
end
if s[0] == "Match" then
let arms = s[2]
let ai = 0
while ai < arms.length do
if body_assigns_name(arms[ai][2], name) then
return true
end
let ai = ai + 1
end
end
let i = i + 1
end
return false
end
# Index of the LAST non-"Func" top-level statement, or -1 if there are
# none -- mirrors ir/lowering.rs's own `top_level`/`last_idx` computation
# in lower_program_basic: main's own tail statement wants a value too (the
# whole program's own implicit last-value return, restoring the pre-
# 86e4fef patc1_main echo behaviour as a consequence of this design rather
# than a separate code path), so its position needs to be known before the
# main loop below decides each statement's own wants_value.
make a function called last_nonfunc_index takes stmts returns idx
let last = -1
let i = 0
while i < stmts.length do
if stmts[i][0] != "Func" then
let last = i
end
let i = i + 1
end
return last
end
make a function called lower_program takes ast returns ir
set_var("__closure_seq", 0)
let stmts = ast[1]
let fns = collect_fns(stmts)
let funcs = []
let events = []
let mainCode = vec_new()
let mainLocals = vec_new()
let mainMutables = vec_new()
let mainLines = vec_new()
let pending = vec_new()
let handlerCount = 0
let last_top_level = last_nonfunc_index(stmts)
let i = 0
while i < stmts.length do
let s = stmts[i]
if s[0] == "Func" then
let flocals = vec_new()
let fmutables = vec_new()
let k = 0
while k < s[2].length do
vec_push(flocals, s[2][k])
vec_push(fmutables, true)
let k = k + 1
end
# Named-return hint (GitHub issue #5): if this Func node carries a
# 5th element (the `returns NAME` hint, non-empty), append a
# synthesized `["Return", ["Var", NAME]]` to the AST-level body
# BEFORE lowering -- matching Stage 0's rust-runtime/src/parser.rs
# (which appends the same synthesized return at parse time, not
# lowering time), so both backends' synthesis point stays
# structurally analogous. Reached only via genuine fall-through:
# any earlier explicit `return` already exits before this point.
# Only synthesized when the body actually assigns the hint name
# somewhere (see body_assigns_name above) -- unconditionally
# appending this for EVERY `returns NAME` declaration would
# silently steal the tail position from a function relying on
# implicit-last-value-return instead, returning an unassigned NAME.
let fbody = s[3]
if s.length > 4 then
if s[4] != "" then
if body_assigns_name(s[3], s[4]) then
let fbody = list_push(s[3], ["Return", ["Var", s[4]]])
end
end
end
let flines = vec_new()
let code = lower_block(fbody, vec_new(), fns, s[1], flocals, fmutables, pending, flines, true)
vec_push(code, ["Return"])
let funcs = list_push(funcs, ["FuncIR", s[1], s[2], vec_to_list(code), vec_to_list(flines)])
else
if s[0] == "When" then
# `when EVENT { ... }` now lowers to a genuine closure (reusing
# lower_closure_literal exactly, just with fixed auto-bound
# params ["event_name","event_data"] instead of user-supplied
# ones), registered at RUNTIME via register_event_handler --
# mirroring native lowering.rs's lower_when. Previously
# synthesized as an ISOLATED standalone function (no access to
# anything outer-scope) registered in a compile-time EventIR
# list, exactly the same gotcha found and fixed natively: a
# handler could never see an enclosing `let` (worked around by
# re-declaring the same name fresh inside every handler body).
# `mainLocals` (the CURRENT accumulated set at this exact point
# in program order, since this "When" case is handled inline in
# the SAME single sequential pass as everything else, not a
# separate pre-pass) is what lower_closure_literal captures --
# so this genuinely sees whatever's been declared before it.
vec_push(mainCode, ["Const", "str", s[1]])
let mainCode = lower_closure_literal(["event_name", "event_data"], s[2], mainCode, fns, mainLocals, pending, mainLines)
vec_push(mainCode, ["CallHost", "register_event_handler", 2])
if i != last_top_level then
vec_push(mainCode, ["Store", "__discard"])
end
else
let mainCode = lower_stmt(s, mainCode, fns, "main", mainLocals, mainMutables, pending, mainLines, i == last_top_level)
end
end
let i = i + 1
end
if last_top_level == -1 then
vec_push(mainCode, ["Const", "unit", ""])
end
vec_push(mainCode, ["Return"])
let funcs = list_push(funcs, ["FuncIR", "main", [], vec_to_list(mainCode), vec_to_list(mainLines)])
let pendingList = vec_to_list(pending)
let i = 0
while i < pendingList.length do
let funcs = list_push(funcs, pendingList[i])
let i = i + 1
end
return ["ProgramIR", "main", funcs, events]
end
# =============================================================================
# General-purpose regex engine, Stage 1 self-hosted dialect (concatenate
# after nothing else; only uses host functions available to the self-hosted
# compiler pipeline: char_code/substr/chr/list_push/list_get/list_len/to_num).
#
# This is the self-hosted-dialect twin of the Stage 0 (`fn`/`{}`) engine at
# rust-runtime's self_hosting/lib/regex.patlang, used by the Rust CLI's
# `syntax_dsl.rs` preprocessor. That version cannot be `include`d into
# anything compiled by the self-hosted pipeline (lib/lexer.patlang +
# lib/parser.patlang don't lex `{`/`}` as anything meaningful — this dialect
# is do/end-delimited throughout), so the logic is duplicated here in the
# dialect lib/lexer.patlang and lib/parser.patlang actually understand. Keep
# both in sync if the regex feature set changes.
#
# Supported syntax subset: literals, \d \w \s \b \n \t escapes, '.' (any),
# [abc] / [a-z0-9] / [^abc] classes, (...) grouping, '|' alternation,
# '*' '+' '?' quantifiers, '^' '$' anchors. No capture-group extraction —
# whole-match length only.
#
# AST nodes are tagged lists: ["lit", ch], ["any"], ["class", negate, items],
# ["seq", nodes], ["alt", branches], ["star", node], ["plus", node],
# ["opt", node], ["bol"], ["eol"], ["wordb"], ["group", node].
# =============================================================================
make a function called is_digit_char takes ch returns r
if ch == "0" then
return true
else
if ch == "1" then
return true
else
if ch == "2" then
return true
else
if ch == "3" then
return true
else
if ch == "4" then
return true
else
if ch == "5" then
return true
else
if ch == "6" then
return true
else
if ch == "7" then
return true
else
if ch == "8" then
return true
else
return ch == "9"
end
end
end
end
end
end
end
end
end
end
make a function called is_word_char takes ch returns r
if ch == "_" then
return true
else
if is_digit_char(ch) then
return true
else
let code = char_code(ch, 0)
if (code >= 65) and (code <= 90) then
return true
else
if (code >= 97) and (code <= 122) then
return true
else
return false
end
end
end
end
end
# ---------------------------------------------------------------------
# Parsing: pattern (string) -> AST node. Internal helpers return
# [node, next_pos] pairs; next_pos is -1 on failure.
# ---------------------------------------------------------------------
make a function called regex_atom_for_escape takes ch returns node
if ch == "d" then
return ["class", 0, [["range", "0", "9"]]]
else
if ch == "w" then
return ["class", 0, [["range", "a", "z"], ["range", "A", "Z"], ["range", "0", "9"], ["char", "_"]]]
else
if ch == "s" then
return ["class", 0, [["char", " "], ["char", "\t"], ["char", "\n"], ["char", "\r"]]]
else
if ch == "b" then
return ["wordb"]
else
if ch == "n" then
return ["lit", "\n"]
else
if ch == "t" then
return ["lit", "\t"]
else
return ["lit", ch]
end
end
end
end
end
end
end
make a function called regex_parse_class takes pattern, pos returns r
let plen = to_num(list_len(pattern))
let negate = 0
let p = pos
if (p < plen) and (substr(pattern, p, 1) == "^") then
let negate = 1
let p = p + 1
end
let items = []
let scanning = true
while (p < plen) and scanning do
if substr(pattern, p, 1) == "]" then
let scanning = false
else
let c = substr(pattern, p, 1)
if (c == "\\") and ((p + 1) < plen) then
let c = substr(pattern, p + 1, 1)
let p = p + 1
end
if ((p + 2) < plen) and (substr(pattern, p + 1, 1) == "-") and (substr(pattern, p + 2, 1) != "]") then
let hi = substr(pattern, p + 2, 1)
let items = list_push(items, ["range", c, hi])
let p = p + 3
else
let items = list_push(items, ["char", c])
let p = p + 1
end
end
end
if p >= plen then
return [["class", negate, items], -1]
else
return [["class", negate, items], p + 1]
end
end
make a function called regex_parse_atom takes pattern, pos returns r
let plen = to_num(list_len(pattern))
if pos >= plen then
return [["seq", []], -1]
else
let c = substr(pattern, pos, 1)
if c == "(" then
let inner = regex_parse_alt(pattern, pos + 1)
let node = inner[0]
let p = inner[1]
if p < 0 then
return [node, -1]
else
if (p >= plen) or (substr(pattern, p, 1) != ")") then
return [node, -1]
else
return [["group", node], p + 1]
end
end
else
if c == "." then
return [["any"], pos + 1]
else
if c == "^" then
return [["bol"], pos + 1]
else
if c == "$" then
return [["eol"], pos + 1]
else
if c == "\\" then
if (pos + 1) >= plen then
return [["lit", "\\"], pos + 1]
else
let esc = substr(pattern, pos + 1, 1)
return [regex_atom_for_escape(esc), pos + 2]
end
else
if c == "[" then
return regex_parse_class(pattern, pos + 1)
else
return [["lit", c], pos + 1]
end
end
end
end
end
end
end
end
make a function called regex_parse_quantified takes pattern, pos returns r
let r = regex_parse_atom(pattern, pos)
let node = r[0]
let p = r[1]
if p < 0 then
return [node, -1]
else
let plen = to_num(list_len(pattern))
if p < plen then
let c = substr(pattern, p, 1)
if c == "*" then
return [["star", node], p + 1]
else
if c == "+" then
return [["plus", node], p + 1]
else
if c == "?" then
return [["opt", node], p + 1]
else
return [node, p]
end
end
end
else
return [node, p]
end
end
end
make a function called regex_parse_seq takes pattern, pos returns r
let plen = to_num(list_len(pattern))
let nodes = []
let p = pos
let scanning = true
while (p < plen) and scanning do
if (substr(pattern, p, 1) == "|") or (substr(pattern, p, 1) == ")") then
let scanning = false
else
let r = regex_parse_quantified(pattern, p)
let node = r[0]
let p = r[1]
if p < 0 then
let scanning = false
else
let nodes = list_push(nodes, node)
end
end
end
return [["seq", nodes], p]
end
make a function called regex_parse_alt takes pattern, pos returns r
let plen = to_num(list_len(pattern))
let first = regex_parse_seq(pattern, pos)
let node = first[0]
let p = first[1]
if p < 0 then
return [node, p]
else
let branches = list_push([], node)
let scanning = true
while (p < plen) and scanning and (substr(pattern, p, 1) == "|") do
let nxt = regex_parse_seq(pattern, p + 1)
let branches = list_push(branches, nxt[0])
let p = nxt[1]
if p < 0 then
let scanning = false
end
end
if p < 0 then
return [["alt", branches], -1]
else
let n = to_num(list_len(branches))
if n == 1 then
return [branches[0], p]
else
return [["alt", branches], p]
end
end
end
end
make a function called regex_parse takes pattern returns node
let result = regex_parse_alt(pattern, 0)
return result[0]
end
# ---------------------------------------------------------------------
# Matching: attempts to match `node` in `text` starting at `pos`, calling
# the closure `k(new_pos)` with every position reachable after a successful
# match, until `k` returns a non-negative number (accepted) or every
# possibility is exhausted (-1). This continuation-passing style is what
# makes backtracking over alternation/quantifiers correct: `k` represents
# "the rest of the overall pattern", so a quantifier can retry with fewer
# repetitions if a later part of the pattern needs the characters back.
# ---------------------------------------------------------------------
make a function called regex_class_matches takes items, ch returns r
let n = to_num(list_len(items))
let i = 0
let found = false
while (i < n) and (not found) do
let it = items[i]
if it[0] == "char" then
if ch == it[1] then
let found = true
end
else
if (ch >= it[1]) and (ch <= it[2]) then
let found = true
end
end
let i = i + 1
end
return found
end
make a function called regex_match_seq takes nodes, idx, text, pos, k returns r
let n = to_num(list_len(nodes))
if idx >= n then
return k(pos)
else
let node = nodes[idx]
return regex_match_node(node, text, pos, |p2| do return regex_match_seq(nodes, idx + 1, text, p2, k) end)
end
end
make a function called regex_match_alt takes branches, idx, text, pos, k returns r
let n = to_num(list_len(branches))
if idx >= n then
return -1
else
let r = regex_match_node(branches[idx], text, pos, k)
if r >= 0 then
return r
else
return regex_match_alt(branches, idx + 1, text, pos, k)
end
end
end
# Backtracking repetition: tries the greatest number of repetitions first,
# then backs off, so whatever comes after (represented by `k`) still gets a
# chance to match — standard greedy-with-backtracking behavior.
make a function called regex_match_repeat takes node, text, pos, k, min_count returns r
let r = regex_match_node(node, text, pos, |p2| do
if p2 == pos then
return -1
else
return regex_match_repeat(node, text, p2, k, 0)
end
end)
if r >= 0 then
return r
else
if min_count <= 0 then
return k(pos)
else
return -1
end
end
end
make a function called regex_match_node takes node, text, pos, k returns r
let tag = node[0]
let tlen = to_num(list_len(text))
if tag == "lit" then
if (pos < tlen) and (substr(text, pos, 1) == node[1]) then
return k(pos + 1)
else
return -1
end
else
if tag == "any" then
if (pos < tlen) and (substr(text, pos, 1) != "\n") then
return k(pos + 1)
else
return -1
end
else
if tag == "class" then
if pos >= tlen then
return -1
else
let ch = substr(text, pos, 1)
let hit = regex_class_matches(node[2], ch)
if node[1] == 1 then
let hit = not hit
end
if hit then
return k(pos + 1)
else
return -1
end
end
else
if tag == "bol" then
if pos == 0 then
return k(pos)
else
if substr(text, pos - 1, 1) == "\n" then
return k(pos)
else
return -1
end
end
else
if tag == "eol" then
if pos == tlen then
return k(pos)
else
if substr(text, pos, 1) == "\n" then
return k(pos)
else
return -1
end
end
else
if tag == "wordb" then
let before = false
let after = false
if pos > 0 then
let before = is_word_char(substr(text, pos - 1, 1))
end
if pos < tlen then
let after = is_word_char(substr(text, pos, 1))
end
if before != after then
return k(pos)
else
return -1
end
else
if tag == "group" then
return regex_match_node(node[1], text, pos, k)
else
if tag == "seq" then
return regex_match_seq(node[1], 0, text, pos, k)
else
if tag == "alt" then
return regex_match_alt(node[1], 0, text, pos, k)
else
if tag == "star" then
return regex_match_repeat(node[1], text, pos, k, 0)
else
if tag == "plus" then
return regex_match_repeat(node[1], text, pos, k, 1)
else
if tag == "opt" then
let inner = node[1]
let r = regex_match_node(inner, text, pos, k)
if r >= 0 then
return r
else
return k(pos)
end
else
return -1
end
end
end
end
end
end
end
end
end
end
end
end
end
# Attempts to match `pattern_ast` (already parsed via regex_parse) against
# `text` starting exactly at `start`. Returns the end position of the
# leftmost-greedy match, or -1 if there is no match anchored at `start`.
#
# Note: closures in this dialect must use an explicit `return` for their
# body value to propagate — a bare trailing expression statement (no
# `return`) does not implicitly become the closure's return value.
make a function called regex_match_at takes pattern_ast, text, start returns r
return regex_match_node(pattern_ast, text, start, |p| do return p end)
end
# Convenience combined entry point for callers that only have the raw
# pattern string (no pre-parsed AST cached) — parses then matches.
make a function called regex_match_string_at takes pattern, text, start returns r
let ast = regex_parse(pattern)
return regex_match_at(ast, text, start)
end
# =============================================================================
# Self-hosted-dialect source-to-source preprocessor for user-defined
# `syntax NAME { ... }` DSL blocks (do/end dialect twin of the Rust CLI's
# rust-runtime/src/syntax_dsl.rs). Runs on raw source TEXT before the normal
# lexer/parser pipeline ever sees it, exactly like expand_includes does for
# `include "..."` lines -- so no change to lib/lexer.patlang or
# lib/parser.patlang is needed to support it.
#
# A PatLang program may declare a small grammar extension inline:
#
# syntax RouterDSL {
# trigger: Keyword("routes");
# tokens {
# HttpVerb(verb) = regex("\b(GET|POST|PUT|DELETE)\b", verb);
# UrlPath(path) = regex("/[a-zA-Z0-9_/:-]*", path);
# Arrow = "->";
# }
# rule RouteLine {
# let verb = expect HttpVerb;
# let path = expect UrlPath;
# expect Arrow;
# let controller = expect Identifier;
# expect Symbol(".");
# let action = expect Identifier;
# return AST.RegisterRoute(verb, path, controller, action);
# }
# }
#
# ...and then use it:
#
# routes {
# GET /users -> UserController.index
# }
#
# The `syntax { ... }` block itself is stripped. Every `routes { ... }` block
# is expanded line-by-line into plain PatLang statements built from the
# rule's `return` template, with `expect`-bound captures substituted in as
# string literals -- before the normal lexer/parser ever sees the file.
#
# Per-token `regex(...)` matching is done by the general-purpose regex engine
# at lib/regex_dsl.patlang -- this file assumes regex_match_string_at (and
# nothing else from that file) is already defined in the same program, i.e.
# a driver script should concatenate lib/regex_dsl.patlang's source BEFORE
# this file's source, the same way lib/lexer.patlang/parser.patlang/
# lower.patlang are concatenated in pipeline driver scripts.
#
# NOTE: `{`/`}` are used here only as characters being scanned inside plain
# strings (the *target* syntax being preprocessed) -- never as PatLang block
# syntax. This file's own code is ordinary do/end/if/then/while dialect.
#
# Data shapes (plain tagged lists -- there is no map/struct in this dialect):
# TokenDef = ["token", name, kind, pattern] kind: "regex" | "literal"
# RuleStep = ["step", kind, value, bind] kind: "named" | "identifier" | "symbol"
# RuleDef = ["rule", steps_list, return_template]
# SyntaxDef = ["syntax", trigger_keyword, tokens_list, rules_list]
# MatchResult = ["ok", bindings_list] | ["fail"] bindings_list: [[name, text], ...]
# =============================================================================
make a function called sdsl_is_ws takes ch returns r
if ch == " " then
return true
else
if ch == "\t" then
return true
else
if ch == "\n" then
return true
else
return ch == "\r"
end
end
end
end
make a function called sdsl_is_digit takes ch returns r
return (ch >= "0") and (ch <= "9")
end
make a function called sdsl_is_ident_start takes ch returns r
if ch == "_" then
return true
else
return ((ch >= "a") and (ch <= "z")) or ((ch >= "A") and (ch <= "Z"))
end
end
make a function called sdsl_is_ident_char takes ch returns r
if sdsl_is_ident_start(ch) then
return true
else
return sdsl_is_digit(ch)
end
end
make a function called sdsl_len takes s returns r
return to_num(list_len(s))
end
# NOTE on `h` params below (sdsl_starts_keyword_at/sdsl_skip_ws/
# sdsl_skip_ws_and_comments/sdsl_find_char/sdsl_take_identifier/
# sdsl_take_balanced_block): `h` is a HANDLE from str_intern(s), not a
# plain string -- accessed via sc_char/sc_len, not substr/sdsl_len.
# `substr` (and sdsl_len, which goes through list_len) calls the plain
# Rust `str::is_ascii()`/`.chars().count()` FRESH on every single call,
# an O(n) full-string scan -- calling that once per character in a
# whole-file scan is exactly what turned this preprocessor's O(n) scan
# into O(n^2) (confirmed: patc1.exe took 30+ minutes on its own
# ~700K-char combined source, which never even contains a real `syntax`
# block, just because "syntax " appears in doc comments and triggers
# the scan at all). `str_intern` caches the is-ascii check ONCE, at
# intern time, and sc_char/sc_len read that cached flag -- the
# established idiom this codebase already uses elsewhere for exactly
# this reason (see hosts.rs's ISTRINGS comment / patlang-ascii-fastpath-
# cache-fix). `sc_char` returns the same 1-char-string shape `substr(s,
# i,1)` did, so callers otherwise read identically to before.
#
# Whole-word keyword match: `keyword` occurs at `h[pos..]` and is not
# glued to an identifier character on either side.
make a function called sdsl_starts_keyword_at takes h, pos, keyword returns r
# Compares character-by-character directly against `keyword` (always a
# short literal like "syntax") instead of building a substring via
# sdsl_sc_substr first -- this function is called once per NON-comment
# character position across the whole file (see sdsl_extract_syntax_
# defs/sdsl_expand_trigger_blocks), so building a fresh sb_new/sb_push
# x N/sb_str-allocated string just to immediately discard it after one
# comparison was the real remaining cost after the O(n^2)->O(n)
# complexity fix: ~8 host calls per character for a ~700K-character
# file that's mostly non-comment code, none of which the comparison
# itself actually needs. `keyword` itself stays a plain substr() call
# (not interned) -- it's a handful of characters, so its own
# uncached is-ascii check is negligible, unlike scanning the whole file.
let slen = sc_len(h)
let klen = sdsl_len(keyword)
if (pos + klen) > slen then
return false
else
let matched = true
let k = 0
while (k < klen) and matched do
if sc_char(h, pos + k) != substr(keyword, k, 1) then
let matched = false
end
let k = k + 1
end
if not matched then
return false
else
let ok = true
if pos > 0 then
if sdsl_is_ident_char(sc_char(h, pos - 1)) then
let ok = false
end
end
let after = pos + klen
if after < slen then
if sdsl_is_ident_char(sc_char(h, after)) then
let ok = false
end
end
return ok
end
end
end
# sc_char/sc_len only give O(1) single-character access -- there's no
# sc_substr, so a short multi-character slice (keyword-length compares,
# almost always just a few chars) is built by concatenating individual
# sc_char results via a string builder rather than substr(h,...), which
# would need a plain string, not a handle.
make a function called sdsl_sc_substr takes h, pos, count returns r
let b = sb_new()
let k = 0
while k < count do
sb_push(b, sc_char(h, pos + k))
let k = k + 1
end
return sb_str(b)
end
make a function called sdsl_skip_ws takes h, pos returns r
let slen = sc_len(h)
let i = pos
let scanning = true
while (i < slen) and scanning do
if sdsl_is_ws(sc_char(h, i)) then
let i = i + 1
else
let scanning = false
end
end
return i
end
make a function called sdsl_skip_ws_and_comments takes h, pos returns r
let slen = sc_len(h)
let i = sdsl_skip_ws(h, pos)
let scanning = true
while scanning do
let scanning = false
if (i < slen) and (sc_char(h, i) == "#") then
while (i < slen) and (sc_char(h, i) != "\n") do
let i = i + 1
end
let i = sdsl_skip_ws(h, i)
let scanning = true
end
end
return i
end
make a function called sdsl_find_char takes h, from, target returns r
let slen = sc_len(h)
let i = from
let found = -1
let scanning = true
while (i < slen) and scanning do
if sc_char(h, i) == target then
let found = i
let scanning = false
else
let i = i + 1
end
end
return found
end
# Returns [identifier_text, next_pos], or ["", -1] if no identifier starts there.
make a function called sdsl_take_identifier takes h, pos returns r
let slen = sc_len(h)
if pos >= slen then
return ["", -1]
else
if not sdsl_is_ident_start(sc_char(h, pos)) then
return ["", -1]
else
let i = pos + 1
let scanning = true
while (i < slen) and scanning do
if sdsl_is_ident_char(sc_char(h, i)) then
let i = i + 1
else
let scanning = false
end
end
return [sdsl_sc_substr(h, pos, i - pos), i]
end
end
end
make a function called sdsl_trim takes s returns r
let slen = sdsl_len(s)
let start = 0
while (start < slen) and sdsl_is_ws(substr(s, start, 1)) do
let start = start + 1
end
let stop = slen
while (stop > start) and sdsl_is_ws(substr(s, stop - 1, 1)) do
let stop = stop - 1
end
return substr(s, start, stop - start)
end
# `"..."` -> `...` (no escape processing needed: the DSL block source is
# scanned as raw text, and quoted strings inside it only ever contain plain
# pattern/literal text in the examples this mechanism targets).
make a function called sdsl_unquote takes s returns r
let t = sdsl_trim(s)
let n = sdsl_len(t)
if (n >= 2) and (substr(t, 0, 1) == "\"") and (substr(t, n - 1, 1) == "\"") then
return substr(t, 1, n - 2)
else
return t
end
end
make a function called sdsl_starts_with takes s, prefix returns r
let n = sdsl_len(prefix)
if sdsl_len(s) < n then
return false
else
return substr(s, 0, n) == prefix
end
end
# `s[open_brace_pos]` must be "{". Returns [inner_body, index_just_past_matching_"}"],
# tracking string literals so braces inside "..." don't confuse the balance count.
make a function called sdsl_take_balanced_block takes h, open_brace_pos returns r
let slen = sc_len(h)
let depth = 0
let i = open_brace_pos
let in_string = false
let start_inner = open_brace_pos + 1
let scanning = true
let result_body = ""
let result_pos = -1
while (i < slen) and scanning do
let c = sc_char(h, i)
if in_string then
if c == "\\" then
let i = i + 2
else
if c == "\"" then
let in_string = false
end
let i = i + 1
end
else
if c == "\"" then
let in_string = true
let i = i + 1
else
if c == "{" then
let depth = depth + 1
let i = i + 1
else
if c == "}" then
let depth = depth - 1
if depth == 0 then
# a single sdsl_sc_substr call here (not per outer-loop-
# iteration), so its own internal O(body-length) builder
# loop is fine -- this only runs once per matched block.
let result_body = sdsl_sc_substr(h, start_inner, i - start_inner)
let result_pos = i + 1
let scanning = false
else
let i = i + 1
end
else
let i = i + 1
end
end
end
end
end
return [result_body, result_pos]
end
# Splits a `syntax`/`rule`/`tokens` body into ";"-terminated statements,
# respecting string literals and nested parens (so `regex("a;b", x)` isn't
# split in the middle).
make a function called sdsl_split_top_level_statements takes body returns r
# sb_new/sb_push/sb_str for `cur`, same O(n^2)-avoidance reason as
# sdsl_extract_syntax_defs/sdsl_expand_trigger_blocks -- lower stakes
# here (bounded by the longest single DSL statement, not the whole
# file) but fixed for consistency once the anti-pattern was found.
let n = sdsl_len(body)
let out = []
let cur = sb_new()
let depth = 0
let in_string = false
let i = 0
while i < n do
let c = substr(body, i, 1)
if in_string then
sb_push(cur, c)
if (c == "\\") and ((i + 1) < n) then
sb_push(cur, substr(body, i + 1, 1))
let i = i + 2
else
if c == "\"" then
let in_string = false
end
let i = i + 1
end
else
if c == "\"" then
let in_string = true
sb_push(cur, c)
let i = i + 1
else
if c == "(" then
let depth = depth + 1
sb_push(cur, c)
let i = i + 1
else
if c == ")" then
let depth = depth - 1
sb_push(cur, c)
let i = i + 1
else
if (c == ";") and (depth == 0) then
let out = list_push(out, sb_str(cur))
let cur = sb_new()
let i = i + 1
else
sb_push(cur, c)
let i = i + 1
end
end
end
end
end
end
if sdsl_len(sdsl_trim(sb_str(cur))) > 0 then
let out = list_push(out, sb_str(cur))
end
return out
end
# ---------------------------------------------------------------------
# Pass 1: find and remove `syntax NAME { ... }` blocks, building a list of
# SyntaxDefs keyed (by linear scan) on each definition's trigger keyword.
# ---------------------------------------------------------------------
make a function called sdsl_parse_trigger_stmt takes stmt returns r
let s = sdsl_trim(stmt)
if sdsl_starts_with(s, "trigger") then
let s = sdsl_trim(substr(s, 7, sdsl_len(s) - 7))
end
if sdsl_starts_with(s, ":") then
let s = sdsl_trim(substr(s, 1, sdsl_len(s) - 1))
end
if sdsl_starts_with(s, "Keyword") then
let s = sdsl_trim(substr(s, 7, sdsl_len(s) - 7))
end
let n = sdsl_len(s)
if (n >= 2) and (substr(s, 0, 1) == "(") and (substr(s, n - 1, 1) == ")") then
let s = substr(s, 1, n - 2)
end
return sdsl_unquote(s)
end
make a function called sdsl_parse_token_defs takes tbody returns r
let stmts = sdsl_split_top_level_statements(tbody)
let out = []
let i = 0
while i < stmts.length do
let stmt = sdsl_trim(stmts[i])
if sdsl_len(stmt) > 0 then
let eq = sdsl_find_char(str_intern(stmt), 0, "=")
let lhs = sdsl_trim(substr(stmt, 0, eq))
let rhs = sdsl_trim(substr(stmt, eq + 1, sdsl_len(stmt) - eq - 1))
let op = sdsl_find_char(str_intern(lhs), 0, "(")
let name = lhs
if op >= 0 then
let name = sdsl_trim(substr(lhs, 0, op))
end
if sdsl_starts_with(rhs, "regex") then
let inner = sdsl_trim(substr(rhs, 5, sdsl_len(rhs) - 5))
let ilen = sdsl_len(inner)
if (ilen >= 2) and (substr(inner, 0, 1) == "(") and (substr(inner, ilen - 1, 1) == ")") then
let inner = substr(inner, 1, ilen - 2)
end
let comma = sdsl_find_char(str_intern(inner), 0, ",")
let pat_str = inner
if comma >= 0 then
let pat_str = substr(inner, 0, comma)
end
let out = list_push(out, ["token", name, "regex", sdsl_unquote(pat_str)])
else
let out = list_push(out, ["token", name, "literal", sdsl_unquote(rhs)])
end
end
let i = i + 1
end
return out
end
make a function called sdsl_parse_rule_def takes rbody returns r
let stmts = sdsl_split_top_level_statements(rbody)
let steps = []
let return_template = ""
let i = 0
while i < stmts.length do
let stmt = sdsl_trim(stmts[i])
if sdsl_len(stmt) > 0 then
if sdsl_starts_with(stmt, "return") then
let return_template = sdsl_trim(substr(stmt, 6, sdsl_len(stmt) - 6))
else
let bind = ""
let expect_part = stmt
if sdsl_starts_with(stmt, "let") then
let rest = sdsl_trim(substr(stmt, 3, sdsl_len(stmt) - 3))
let eq = sdsl_find_char(str_intern(rest), 0, "=")
let bind = sdsl_trim(substr(rest, 0, eq))
let expect_part = sdsl_trim(substr(rest, eq + 1, sdsl_len(rest) - eq - 1))
end
if sdsl_starts_with(expect_part, "expect") then
let expect_part = sdsl_trim(substr(expect_part, 6, sdsl_len(expect_part) - 6))
end
if sdsl_starts_with(expect_part, "Symbol") then
let inner = sdsl_trim(substr(expect_part, 6, sdsl_len(expect_part) - 6))
let ilen = sdsl_len(inner)
if (ilen >= 2) and (substr(inner, 0, 1) == "(") and (substr(inner, ilen - 1, 1) == ")") then
let inner = substr(inner, 1, ilen - 2)
end
let steps = list_push(steps, ["step", "symbol", sdsl_unquote(inner), bind])
else
if expect_part == "Identifier" then
let steps = list_push(steps, ["step", "identifier", "", bind])
else
let steps = list_push(steps, ["step", "named", expect_part, bind])
end
end
end
end
let i = i + 1
end
return ["rule", steps, return_template]
end
make a function called sdsl_parse_syntax_def takes body returns r
# Interns `body` ONCE and threads the handle through the (now
# handle-based) scanning helpers -- see the note above
# sdsl_starts_keyword_at for why (substr/sdsl_len recompute an O(n)
# is-ascii/char-count check on every call; sc_char/sc_len read a
# cached flag from str_intern instead).
let h = str_intern(body)
let n = sc_len(h)
let i = sdsl_skip_ws_and_comments(h, 0)
let trigger_keyword = ""
let tokens = []
let rules = []
while i < n do
if sdsl_starts_keyword_at(h, i, "trigger") then
let semi = sdsl_find_char(h, i, ";")
let trigger_keyword = sdsl_parse_trigger_stmt(sdsl_sc_substr(h, i, semi - i))
let i = semi + 1
else
if sdsl_starts_keyword_at(h, i, "tokens") then
let j = sdsl_skip_ws(h, i + 6)
let br = sdsl_take_balanced_block(h, j)
let tokens = sdsl_parse_token_defs(br[0])
let i = br[1]
else
if sdsl_starts_keyword_at(h, i, "rule") then
let j = sdsl_skip_ws(h, i + 4)
let nameinfo = sdsl_take_identifier(h, j)
let j = sdsl_skip_ws(h, nameinfo[1])
let br = sdsl_take_balanced_block(h, j)
let rules = list_push(rules, sdsl_parse_rule_def(br[0]))
let i = br[1]
else
let i = i + 1
end
end
end
let i = sdsl_skip_ws_and_comments(h, i)
end
return ["syntax", trigger_keyword, tokens, rules]
end
make a function called sdsl_extract_syntax_defs takes src returns r
# Two separate O(n^2)-avoidance fixes, both needed:
# (1) Builds the stripped-of-real-defs output via sb_new/sb_push/
# sb_str, NOT `out = out + substr(...)` -- string concatenation in
# a loop copies the WHOLE accumulated string on every append
# (PatLang strings are immutable).
# (2) Reads `src` via an str_intern HANDLE (sc_char/sc_len), NOT plain
# substr/sdsl_len -- substr calls Rust's `str::is_ascii()` (an
# O(n) full-string scan) fresh on EVERY call, so calling it once
# per character in a whole-file scan was actually the DOMINANT
# cost here, not the string-concat alone: fixing (1) alone barely
# moved the needle (8x input still took ~8x longer, i.e. still
# O(n^2)) until this handle-based fix was added too.
# Found the hard way: this made `expand_syntax_dsls` (called
# unconditionally on EVERY source patc1.exe compiles, including
# compiling itself) take 30+ minutes on patc1_all.patlang's own
# ~700K-character combined source, even though it never contains a
# real `syntax NAME {` block -- confirmed via a direct scaling test
# (8x input length took ~64x longer, the signature of O(n^2)) before
# touching anything, exactly the same anti-pattern already fixed
# several times elsewhere in this codebase (see best-practices' sb_*
# and str_intern/sc_* guidance).
# NOTE: also tracks in_string, matching native syntax_dsl.rs's
# extract_syntax_defs exactly -- a real, separate, pre-existing
# correctness gap found while diagnosing performance on a real
# ~700K-character file: without this, a STRING LITERAL anywhere in the
# file that happens to contain text shaped like "syntax NAME {" (very
# plausible here -- self_hosting/lib/runtime_rs.patlang embeds large
# chunks of literal Rust source AS PatLang string literals, and Rust
# source plausibly mentions the word "syntax" somewhere, e.g. in an
# error message) gets misread as a real definition, corrupting parsing
# of everything downstream. Never manifested before because nothing
# had ever actually run this preprocessor's whole-file scan against a
# large real file with lots of string literals until now.
let h = str_intern(src)
let n = sc_len(h)
let out = sb_new()
let defs = []
let i = 0
let in_string = false
while i < n do
if in_string then
sb_push(out, sc_char(h, i))
if (sc_char(h, i) == "\\") and ((i + 1) < n) then
sb_push(out, sc_char(h, i + 1))
let i = i + 2
else
if sc_char(h, i) == "\"" then
let in_string = false
end
let i = i + 1
end
else
if sc_char(h, i) == "\"" then
let in_string = true
sb_push(out, sc_char(h, i))
let i = i + 1
else
if sc_char(h, i) == "#" then
# Copy the whole comment line through untouched -- a comment may
# itself mention "syntax" or "routes" (e.g. this file's own header,
# or a demo's usage-example doc comment) without being a real block.
let cstart = i
while (i < n) and (sc_char(h, i) != "\n") do
let i = i + 1
end
sb_push(out, sdsl_sc_substr(h, cstart, i - cstart))
else
if sdsl_starts_keyword_at(h, i, "syntax") then
let j = sdsl_skip_ws(h, i + 6)
let nameinfo = sdsl_take_identifier(h, j)
let j = sdsl_skip_ws(h, nameinfo[1])
let br = sdsl_take_balanced_block(h, j)
let def = sdsl_parse_syntax_def(br[0])
let defs = list_push(defs, def)
let i = br[1]
else
sb_push(out, sc_char(h, i))
let i = i + 1
end
end
end
end
end
return [sb_str(out), defs]
end
# ---------------------------------------------------------------------
# Pass 2: expand every `<trigger keyword> { ... }` block found in the
# (already syntax-def-stripped) source.
# ---------------------------------------------------------------------
make a function called sdsl_match_literal takes s, pos, lit returns r
let n = sdsl_len(lit)
if (pos + n) > sdsl_len(s) then
return -1
else
if substr(s, pos, n) == lit then
return pos + n
else
return -1
end
end
end
make a function called sdsl_match_identifier takes s, pos returns r
let n = sdsl_len(s)
if pos >= n then
return -1
else
if not sdsl_is_ident_start(substr(s, pos, 1)) then
return -1
else
let i = pos + 1
let scanning = true
while (i < n) and scanning do
if sdsl_is_ident_char(substr(s, i, 1)) then
let i = i + 1
else
let scanning = false
end
end
return i
end
end
end
# Finds `name` inside a TokenDef list; returns the TokenDef or ["token", "", "", ""] if absent.
make a function called sdsl_find_token_def takes tokens, name returns r
let n = tokens.length
let i = 0
let found = ["token", "", "", ""]
let scanning = true
while (i < n) and scanning do
let t = tokens[i]
if t[1] == name then
let found = t
let scanning = false
end
let i = i + 1
end
return found
end
# Finds `value` inside a [[name, value], ...] bindings list; returns "" if absent.
make a function called sdsl_bindings_lookup takes bindings, name returns r
let n = bindings.length
let i = 0
let found = ""
let scanning = true
while (i < n) and scanning do
let b = bindings[i]
if b[0] == name then
let found = b[1]
let scanning = false
end
let i = i + 1
end
return found
end
# Attempts to match every step of `rule` against `line` in order. Returns
# ["ok", bindings] on full success, ["fail"] if any step fails to match.
make a function called sdsl_try_match_rule takes def, the_rule, line returns r
# NOTE: param renamed from `rule` -- a bare reserved keyword used as a
# plain parameter name silently truncates the params list instead of
# erroring (the parser's "takes NAME, NAME, ..." list expects
# Token::Identifier, but `rule` lexes as its own dedicated Token::Rule) --
# found via a genuinely confusing silent crash: this function ended up
# with params=["def"] only and a corrupted 2-instruction body, which the
# interpreter then panicked on internally, and that panic was silently
# swallowed by main()'s `.join().unwrap_or(())` thread-join wrapper,
# producing a totally silent "successful" exit with zero output and no
# error message at all. Same class of bug as the earlier `fn`-as-a-
# param-name collision found in the paradigm gallery work.
let steps = the_rule[1]
let line_h = str_intern(line)
let pos = 0
let bindings = []
let n = steps.length
let i = 0
let ok = true
while (i < n) and ok do
let pos = sdsl_skip_ws(line_h, pos)
let step = steps[i]
let kind = step[1]
let matched = -1
if kind == "identifier" then
let matched = sdsl_match_identifier(line, pos)
else
if kind == "symbol" then
let matched = sdsl_match_literal(line, pos, step[2])
else
let tdef = sdsl_find_token_def(def[2], step[2])
if tdef[2] == "literal" then
let matched = sdsl_match_literal(line, pos, tdef[3])
else
if tdef[2] == "regex" then
let text = substr(line, pos, sdsl_len(line) - pos)
let end_off = regex_match_string_at(tdef[3], text, 0)
if end_off > 0 then
let matched = pos + end_off
end
end
end
end
end
if matched < 0 then
let ok = false
else
let bind = step[3]
if sdsl_len(bind) > 0 then
let bindings = list_push(bindings, [bind, substr(line, pos, matched - pos)])
end
let pos = matched
end
let i = i + 1
end
if ok then
return ["ok", bindings]
else
return ["fail"]
end
end
# Replaces every whole-word occurrence of a bound capture name in `template`
# with its matched text as a PatLang string literal.
make a function called sdsl_substitute_template takes template, bindings returns r
let n = sdsl_len(template)
let out = sb_new()
let i = 0
while i < n do
let c = substr(template, i, 1)
if sdsl_is_ident_start(c) then
let j = i + 1
let scanning = true
while (j < n) and scanning do
if sdsl_is_ident_char(substr(template, j, 1)) then
let j = j + 1
else
let scanning = false
end
end
let word = substr(template, i, j - i)
let val = sdsl_bindings_lookup(bindings, word)
if sdsl_len(val) > 0 then
sb_push(out, "\"" + val + "\"")
else
sb_push(out, word)
end
let i = j
else
sb_push(out, c)
let i = i + 1
end
end
return sb_str(out)
end
# Runs each of the DSL's rules (in declaration order) against `line`,
# returning the first one whose full `expect` sequence matches, with its
# return template's captures substituted in as PatLang string literals.
make a function called sdsl_expand_rule_line takes def, line returns r
let rules = def[3]
let n = rules.length
let i = 0
let result = ""
let found = false
while (i < n) and (not found) do
let m = sdsl_try_match_rule(def, rules[i], line)
if m[0] == "ok" then
# No trailing ";" -- PatLang has no statement-terminating semicolon
# anywhere in its grammar. A stray ";" tokenizes as its own UNK token
# and becomes a parse error; that error was invisible when this output
# only ever went through compile_native (lower_program silently drops
# unparseable Err statements), but is fatal for the playground/IDE
# path, which aborts entirely if any parse error is present.
let result = sdsl_substitute_template(rules[i][2], m[1])
let found = true
end
let i = i + 1
end
return result
end
make a function called sdsl_expand_trigger_blocks takes src, defs returns r
# sb_new/sb_push/sb_str + str_intern/sc_char/sc_len throughout, same
# two-part reason as sdsl_extract_syntax_defs (string-concat AND
# substr's per-call O(n) is-ascii check both turn this into O(n^2)
# otherwise). This function only actually runs when defs is non-empty
# (see expand_syntax_dsls_with's early-return), i.e. only when a real
# `syntax NAME {}` block was found -- lower-traffic than
# sdsl_extract_syntax_defs (called on every compile unconditionally)
# but fixed the same way for the same reason and for real syntax-DSL
# users on large files.
let h = str_intern(src)
let n = sc_len(h)
let out = sb_new()
let i = 0
while i < n do
if sc_char(h, i) == "#" then
# Same rationale as sdsl_extract_syntax_defs: a comment mentioning a
# trigger keyword (e.g. this file's own doc comments about `routes`)
# is not a real usage block, and must never be scanned into one.
let cstart = i
while (i < n) and (sc_char(h, i) != "\n") do
let i = i + 1
end
sb_push(out, sdsl_sc_substr(h, cstart, i - cstart))
else
let kw_idx = -1
let d = 0
while (d < defs.length) and (kw_idx < 0) do
if sdsl_starts_keyword_at(h, i, defs[d][1]) then
let kw_idx = d
end
let d = d + 1
end
if kw_idx >= 0 then
let def = defs[kw_idx]
let j = sdsl_skip_ws(h, i + sdsl_len(def[1]))
if (j < n) and (sc_char(h, j) == "{") then
let br = sdsl_take_balanced_block(h, j)
let body = br[0]
let bh = str_intern(body)
let blen = sc_len(bh)
let start = 0
while start <= blen do
let nl = sdsl_find_char(bh, start, "\n")
let line_end = nl
if nl < 0 then
let line_end = blen
end
let line = sdsl_trim(sdsl_sc_substr(bh, start, line_end - start))
if sdsl_len(line) > 0 then
sb_push(out, sdsl_expand_rule_line(def, line) + "\n")
end
if nl < 0 then
let start = blen + 1
else
let start = nl + 1
end
end
let i = br[1]
else
sb_push(out, sc_char(h, i))
let i = i + 1
end
else
sb_push(out, sc_char(h, i))
let i = i + 1
end
end
end
return sb_str(out)
end
# Public entry point, composable across multiple separate runtime calls:
# expands every `syntax NAME { ... }` definition and every triggered block
# in `src`, using both `src`'s own definitions AND `existing_defs` (defs
# already extracted from earlier, separate `expand_syntax_dsls_with` calls
# -- e.g. a REPL-style session where one snippet defines a syntax and a
# LATER, separately-submitted snippet uses it). Returns [stripped, all_defs]
# so a caller can thread `all_defs` into its next call and keep
# accumulating, rather than every dynamic snippet needing to redeclare (or
# be concatenated with) every syntax it wants to use.
make a function called expand_syntax_dsls_with takes src, existing_defs returns r
let extracted = sdsl_extract_syntax_defs(src)
let stripped = extracted[0]
let new_defs = extracted[1]
let all_defs = existing_defs
let i = 0
while i < new_defs.length do
let all_defs = list_push(all_defs, new_defs[i])
let i = i + 1
end
if all_defs.length == 0 then
return [stripped, all_defs]
else
return [sdsl_expand_trigger_blocks(stripped, all_defs), all_defs]
end
end
# Public entry point: expands every `syntax NAME { ... }` definition and
# every triggered block in `src`. Thin wrapper over `expand_syntax_dsls_with`
# with no prior defs -- every existing call site (playground_main.patlang,
# build_portfolio.patlang's dsl_driver) is unaffected by the composable
# variant above.
make a function called expand_syntax_dsls takes src returns r
let r = expand_syntax_dsls_with(src, [])
return r[0]
end
# Dynamic syntax demo: proves `syntax NAME { ... }` DSL definitions are a
# genuine RUNTIME capability, not just a compile-time convenience. Every
# other syntax-DSL example in this portfolio (router_dsl_demo.patlang) has
# its `syntax { ... }` block written as a source-code literal, expanded
# exactly once, before the ordinary compiler ever runs -- indistinguishable
# from a macro. This demo instead:
#
# 1. Computes the DSL's trigger keyword from runtime data (not a literal
# in this file) -- the grammar extension genuinely could not have been
# known until the program executed.
# 2. Defines the syntax via `expand_syntax_dsls_with(src, [])` in one call,
# keeping only the returned `defs` -- the definition text itself is
# discarded immediately afterward.
# 3. Uses that syntax in a SEPARATE, later `expand_syntax_dsls_with(usage,
# defs)` call, on a string that contains ONLY the usage line -- no
# syntax declaration anywhere in it. This is the composability case:
# the two calls could be arbitrarily far apart (different functions,
# different point in the program, even a different network request in
# a real system), carrying only `defs` -- a plain list value -- between
# them, exactly like passing any other piece of program state.
# 4. Concatenates both expanded fragments and runs the RESULT through the
# ordinary tokenize/parse_program/lower_program/run_ir pipeline --
# itself just plain PatLang function calls, no special "eval" host
# function required.
#
# NOTE on style: every string built here uses one `let` per concatenation
# step (`let s2 = s1 + "..."`), never a multi-line `+`-continuation
# expression (`"a" + chr(10)` then `+ "b"` starting the NEXT line). The
# self-hosted parser's parse_add/parse_mul (unlike the native Rust frontend,
# and unlike this same file's own list/argument parsing, which explicitly
# skip newlines) do not skip newlines before checking for a continuation
# operator, so a leading `+` on a new line silently truncates the
# expression right there -- a real, previously-latent gap in
# self_hosting/lib/parser.patlang, only surfaced now because this is the
# first self-hosted-pipeline-compiled program to build a string this way.
#
# Concatenate after lib/lexer.patlang + lib/parser.patlang + lib/lower.patlang
# + lib/regex_dsl.patlang + lib/syntax_dsl.patlang when compiling (matches
# playground_main.patlang's own concatenation -- this demo needs
# tokenize/parse_program/lower_program/expand_syntax_dsls_with in scope to
# call them on data it constructs itself, exactly like the live IDE does on
# whatever a user types).
# --- Step 0: pick the trigger keyword from runtime data, not a literal ---
let candidates = ["greet", "salute", "hail"]
let pick = (3 * 5) % candidates.length
let trigger_word = candidates[pick]
print("chosen trigger keyword at runtime: " + trigger_word)
# --- Step 1: define the syntax from a string built with that keyword ---
let d1 = "make a function called say_hello takes name returns done" + chr(10)
let d2 = d1 + " print(" + chr(34) + "Hello, " + chr(34) + " + name + " + chr(34) + "!" + chr(34) + ")" + chr(10)
let d3 = d2 + " return true" + chr(10)
let d4 = d3 + "end" + chr(10)
let d5 = d4 + chr(10)
let d6 = d5 + "syntax GreetDSL {" + chr(10)
let d7 = d6 + " trigger: Keyword(" + chr(34) + trigger_word + chr(34) + ");" + chr(10)
let d8 = d7 + " tokens {" + chr(10)
let d9 = d8 + " }" + chr(10)
let d10 = d9 + " rule GreetLine {" + chr(10)
let d11 = d10 + " let name = expect Identifier;" + chr(10)
let d12 = d11 + " return say_hello(name);" + chr(10)
let d13 = d12 + " }" + chr(10)
let def_src = d13 + "}" + chr(10)
let step1 = expand_syntax_dsls_with(def_src, [])
let program_prefix = step1[0]
let defs = step1[1]
print("syntax defined; " + defs.length + " grammar extension(s) known so far")
# --- Step 2: a LATER, separate call sees ONLY the usage text -- no syntax
# declaration in sight -- yet still expands correctly, because `defs`
# carried forward from step 1. ---
let u1 = trigger_word + " {" + chr(10)
let u2 = u1 + " World" + chr(10)
let usage_src = u2 + "}" + chr(10)
let step2 = expand_syntax_dsls_with(usage_src, defs)
let program_suffix = step2[0]
# --- Step 3: run the assembled program through the ordinary pipeline ---
let p1 = program_prefix + chr(10)
let p2 = p1 + program_suffix
let full_program = p2 + "print(" + chr(34) + "dynamic syntax demo complete." + chr(34) + ")" + chr(10)
let toks = tokenize(full_program)
let ast = parse_program(toks)
let ir = lower_program(ast)
run_ir(ir)
Native run on the build machine:
chosen trigger keyword at runtime: greet syntax defined; 1 grammar extension(s) known so far Hello, World! dynamic syntax demo complete.