lexer.ts · Source

lexer.tspackages/database-engine/src/formula/lexer.tsraw ← formula4,974 B← Back to the app
1/**
2 * @factory/database-engine/formula — tokenizer (TASK-0084 R1).
3 *
4 * Numbers (int/float/exponent `1e5`), strings (double-quoted with `\"`
5 * escapes; single-quoted accepted for user-friendness — both produce the
6 * same token, recorded), booleans, ISO date literals, identifiers
7 * (case-sensitive; the spec's `[a-z][a-z0-9_]*` pattern is recorded as
8 * insufficient — the fixed function set contains uppercase names like
9 * `dateAdd`, so identifiers accept `[A-Za-z][A-Za-z0-9_]*`), operators,
10 * and `//` line comments (stripped, recorded). Unknown char → PARSE error
11 * with position (the editor's caret-annotated highlighting consumes `pos`).
12 */
13 
14import { FormulaError } from "./types.js";
15 
16export type Token =
17  | { kind: "number"; value: number; pos: number }
18  | { kind: "string"; value: string; pos: number }
19  | { kind: "date"; value: string; pos: number }
20  | { kind: "bool"; value: boolean; pos: number }
21  | { kind: "null"; pos: number }
22  | { kind: "ident"; value: string; pos: number }
23  | { kind: "op"; value: string; pos: number }
24  | { kind: "eof"; pos: number };
25 
26// NOTE: `and` / `or` / `not` are FUNCTIONS in the formula spec (and(a,b),
27// not(x)) and are therefore NOT reserved keywords — they tokenize as
28// `ident` so the parser's ident→call dispatch handles them. `true`/`false`/
29// `null` are handled explicitly below. (The spec's infix `or`/`and` operators
30// are unused; the closed function set is the canonical form. Recorded.)
31export const KEYWORDS = new Set<string>([]);
32 
33const TWO_CHAR_OPS = ["==", "!=", "<=", ">="];
34const ONE_CHAR_OPS = ["+", "-", "*", "/", "%", "<", ">", "(", ")", ",", "?", ":", "[", "]"];
35 
36export function tokenize(src: string): Token[] {
37  const tokens: Token[] = [];
38  let i = 0;
39  const n = src.length;
40  while (i < n) {
41    const c = src[i]!;
42    if (c === " " || c === "\t" || c === "\n" || c === "\r") {
43      i++;
44      continue;
45    }
46    if (c === "/" && src[i + 1] === "/") {
47      while (i < n && src[i] !== "\n") i++;
48      continue;
49    }
50    // date literal — MUST come before the number rule: the year would
51    // otherwise be lexed as a plain number and the dashes as operators
52    // (YYYY-MM-DD starts with a digit, so the number rule shadows it).
53    const dateMatch =
54      /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?/.exec(src.slice(i));
55    if (dateMatch) {
56      const value = dateMatch[0];
57      tokens.push({ kind: "date", value, pos: i });
58      i += value.length;
59      continue;
60    }
61    // number
62    if (/[0-9]/.test(c) || (c === "." && i + 1 < n && /[0-9]/.test(src[i + 1]!))) {
63      const start = i;
64      while (i < n && /[0-9.]/.test(src[i]!)) i++;
65      if (i < n && (src[i] === "e" || src[i] === "E")) {
66        let j = i + 1;
67        if (j < n && (src[j] === "+" || src[j] === "-")) j++;
68        if (j < n && /[0-9]/.test(src[j]!)) {
69          i = j;
70          while (i < n && /[0-9]/.test(src[i]!)) i++;
71        }
72      }
73      const raw = src.slice(start, i);
74      const value = Number(raw);
75      if (!Number.isFinite(value))
76        throw new FormulaError("PARSE", `invalid number "${raw}"`, start);
77      tokens.push({ kind: "number", value, pos: start });
78      continue;
79    }
80    // string
81    if (c === '"' || c === "'") {
82      const quote = c;
83      let j = i + 1;
84      let out = "";
85      while (j < n) {
86        const ch = src[j]!;
87        if (ch === "\\" && j + 1 < n) {
88          const next = src[j + 1]!;
89          if (next === quote || next === "\\") {
90            out += next;
91            j += 2;
92            continue;
93          }
94          throw new FormulaError("PARSE", `invalid escape "\${next}" in string`, j);
95        }
96        if (ch === quote) break;
97        out += ch;
98        j++;
99      }
100      if (j >= n) throw new FormulaError("PARSE", "unterminated string", i);
101      tokens.push({ kind: "string", value: out, pos: i });
102      i = j + 1;
103      continue;
104    }
105    // identifier / keyword
106    if (/[A-Za-z_]/.test(c)) {
107      const start = i;
108      while (i < n && /[A-Za-z0-9_]/.test(src[i]!)) i++;
109      const value = src.slice(start, i);
110      if (value === "true") tokens.push({ kind: "bool", value: true, pos: start });
111      else if (value === "false") tokens.push({ kind: "bool", value: false, pos: start });
112      else if (value === "null") tokens.push({ kind: "null", pos: start });
113      else if (KEYWORDS.has(value)) tokens.push({ kind: "op", value, pos: start });
114      else tokens.push({ kind: "ident", value, pos: start });
115      continue;
116    }
117    // operators
118    const two = src.slice(i, i + 2);
119    if (TWO_CHAR_OPS.includes(two)) {
120      tokens.push({ kind: "op", value: two, pos: i });
121      i += 2;
122      continue;
123    }
124    if (ONE_CHAR_OPS.includes(c)) {
125      tokens.push({ kind: "op", value: c, pos: i });
126      i++;
127      continue;
128    }
129    throw new FormulaError("PARSE", `unexpected character "${c}"`, i);
130  }
131  tokens.push({ kind: "eof", pos: n });
132  return tokens;
133}
134