Skip to content

Commit f5420d9

Browse files
mtillermtiller
committed
More work, mainly on typing related aspects
One big priority was removing optional fields from AST nodes and tagging AST nodes with `global` properties (like `keepArray`, see below). I haven't changed the AST definitions at all. The `type` and `value` fields should have the same values they always had. However, in the types I'm using tagged unions for all the AST nodes. Generally, the `type` field is used to narrow the type. But in some cases, I further refine type by `value` (which you can really think of as a subtype for the AST nodes). This basically allows me to narrow the type further in the code and do more static checking during compilation to make sure that the proper fields are being read and/or written. One "breaking change" is with the parser recovery tests. These aren't really breaking in the sense that they would impact most users. They have to do with including some additional lexical information. This information was, generally speaking, already there. But there were cases where it wasn't included for some AST nodes. So the change is really just including it consistently. Another noticeable change is with error handling. Previously, the syntax errors were annotated on the root node of the AST. I've changed that so they don't impact the AST. I've also changed the error semantics very slightly. Now, if there are no errors, the `errors` field in an expression will return an empty array (previously, it returned `undefined`). One completely internal change was the elimination of the `keepArray` attribute on AST nodes. This was added whenever the `[]` was found in an expression. I've changed the handling so that instead of having the parser walk through the AST and find the head of the path that contains the `[]` operator and then set the `keepArray` attribute on it, I'm simply adding a special "decorator" AST node that is optimized away eventually. This keeps the parser focused just on building the initial AST and leaves the semantics of the `SingletonArrayDecorator` (the node that gets inserted as a result of an `[]` in a path) to be handled completely during `ast_optimize`. It is important to note that this special AST node is always optimized away. In `ast_optimize` I generate a new AST node as a result of optimization in each `case` of the `switch` statement. This is mainly to keep the typing clean (so we know precisely what we are working with at any given time).
1 parent 33d5430 commit f5420d9

16 files changed

+957
-637
lines changed

‎__tests__/parser-recovery.ts

Lines changed: 340 additions & 289 deletions
Large diffs are not rendered by default.

‎package.json

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,13 @@
5252
"request": "^2.81.0",
5353
"rollup": "^0.54.1",
5454
"ts-jest": "21.2.3",
55-
"uglify-es": "^3.0.20"
55+
"uglify-es": "^3.0.20",
56+
"@types/node": "^9.3.0",
57+
"typescript": "^2.6.2"
5658
},
5759
"engines": {
5860
"node": ">= 4"
5961
},
60-
"dependencies": {
61-
"@types/node": "^9.3.0",
62-
"typescript": "^2.6.2"
63-
},
6462
"jest": {
6563
"globals": {
6664
"ts-jest": {

‎src/ast.ts

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
import { Token } from "./tokenizer";
2+
import { Signature } from './signatures';
3+
4+
// Potential AST changes
5+
//
6+
// - Make predicate and group into AST nodes instead of optional fields on every node.
7+
// - Change unary operator "[" to a different type...?
8+
// - Get errors? off of BaseNode
9+
// - Rationalize unary nodes
10+
11+
export interface BaseNode {
12+
type: string;
13+
value: any;
14+
position: number;
15+
// This gets added to nodes to indicate how a value (assuming it is an object)
16+
// should be grouped.
17+
// TODO: Rename lhs...?
18+
group?: { lhs: ASTNode[][], position: number };
19+
// This gets added to nodes to specify a list of predicates to filter on.
20+
predicate?: ASTNode[];
21+
}
22+
23+
export interface WildcardNode extends BaseNode {
24+
type: "wildcard";
25+
}
26+
27+
export interface DescendantNode extends BaseNode {
28+
type: "descendant";
29+
}
30+
31+
export interface ErrorFields {
32+
code: string;
33+
position?: number;
34+
token?: string;
35+
stack?: string;
36+
}
37+
38+
export interface ErrorNode extends BaseNode {
39+
type: "error";
40+
error: ErrorFields;
41+
lhs: ASTNode;
42+
remaining: Token[];
43+
}
44+
45+
export interface VariableNode extends BaseNode {
46+
type: "variable";
47+
}
48+
49+
export interface NameNode extends BaseNode {
50+
type: "name";
51+
}
52+
export interface LiteralNode extends BaseNode {
53+
type: "literal";
54+
}
55+
56+
export interface RegexNode extends BaseNode {
57+
type: "regex";
58+
}
59+
60+
export interface OperatorNode extends BaseNode {
61+
type: "operator";
62+
}
63+
64+
export interface EndNode extends BaseNode {
65+
type: "end";
66+
value: string;
67+
}
68+
69+
export type TerminalNode = VariableNode | NameNode | LiteralNode | RegexNode | OperatorNode | EndNode;
70+
71+
export interface UnaryMinusNode extends BaseNode {
72+
type: "unary";
73+
value: "-";
74+
expression: ASTNode;
75+
}
76+
77+
export interface ArrayConstructorNode extends BaseNode {
78+
type: "unary";
79+
value: "[";
80+
expressions: ASTNode[];
81+
consarray: boolean;
82+
}
83+
84+
export interface UnaryObjectNode extends BaseNode {
85+
type: "unary";
86+
value: "{";
87+
lhs: ASTNode[][];
88+
}
89+
90+
export type UnaryNode = UnaryMinusNode | ArrayConstructorNode | UnaryObjectNode;
91+
92+
export interface BinaryOperationNode extends BaseNode {
93+
type: "binary";
94+
value: "+" | "-" | "*" | "/" | "[" | ".." | "." | "[" | ":=" | "~>"; // TODO: There must be more?!? (e.g., comparisons)
95+
lhs: ASTNode;
96+
rhs: ASTNode; // ASTNode if operator is "." | "[" | ":=" | "~>"
97+
}
98+
99+
export interface BinaryObjectNode extends BaseNode {
100+
type: "binary";
101+
value: "{";
102+
lhs: ASTNode;
103+
rhs: ASTNode[]; // ASTNode[] if operator is "{"
104+
}
105+
106+
export type BinaryNode = BinaryOperationNode | BinaryObjectNode;
107+
108+
export interface SortTerm {
109+
descending: boolean;
110+
expression: ASTNode;
111+
}
112+
113+
export interface SortNode extends BaseNode {
114+
type: "sort";
115+
lhs: ASTNode;
116+
rhs: SortTerm[];
117+
}
118+
119+
export interface TernaryNode extends BaseNode {
120+
type: "condition";
121+
condition: ASTNode;
122+
then: ASTNode;
123+
else: ASTNode;
124+
position: number;
125+
}
126+
127+
export interface BlockNode extends BaseNode {
128+
type: "block";
129+
expressions: ASTNode[];
130+
}
131+
132+
export interface TransformNode extends BaseNode {
133+
type: "transform";
134+
pattern: ASTNode;
135+
update: ASTNode;
136+
delete?: ASTNode;
137+
}
138+
139+
export interface FunctionInvocationNode extends BaseNode {
140+
type: "function" | "partial";
141+
procedure: ASTNode;
142+
arguments: ASTNode[];
143+
// This is added when creating PathNodes.
144+
nextFunction?: Function;
145+
}
146+
147+
export interface LambdaDefinitionNode extends BaseNode {
148+
type: "lambda";
149+
body: ASTNode;
150+
signature: Signature;
151+
arguments: ASTNode[];
152+
thunk: boolean;
153+
}
154+
155+
export interface SingletonArrayDecorator extends BaseNode {
156+
type: "singleton";
157+
next: ASTNode;
158+
}
159+
160+
// This type of node only appears after the AST is optimized
161+
export interface PathNode extends BaseNode {
162+
type: "path";
163+
steps: ASTNode[];
164+
keepSingletonArray: boolean,
165+
}
166+
167+
export interface BindNode extends BaseNode {
168+
type: "bind";
169+
lhs: ASTNode;
170+
rhs: ASTNode;
171+
}
172+
173+
export interface ApplyNode extends BaseNode {
174+
type: "apply";
175+
lhs: ASTNode;
176+
rhs: ASTNode;
177+
}
178+
179+
/**
180+
* These are the AST nodes that come directly out of the parser before
181+
* ast_optimize is called.
182+
*/
183+
export type ASTNode =
184+
| WildcardNode
185+
| DescendantNode
186+
| ErrorNode
187+
| LiteralNode
188+
| NameNode
189+
| VariableNode
190+
| RegexNode
191+
| OperatorNode
192+
| UnaryNode
193+
| BinaryNode
194+
| BinaryObjectNode
195+
| SortNode
196+
| TernaryNode
197+
| BlockNode
198+
| TransformNode
199+
| FunctionInvocationNode
200+
| LambdaDefinitionNode
201+
| PathNode
202+
| BindNode
203+
| ApplyNode
204+
| EndNode
205+
| SingletonArrayDecorator;
206+

‎src/evaluate.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@ import {
1111
toSequence,
1212
} from "./utils";
1313
import { defineFunction } from "./signatures";
14-
import { parser } from "./parser";
14+
import { parser } from './parser';
1515
import { functionBoolean, functionAppend, functionString, functionSort, createStandardFrame } from "./functions";
16+
// import * as ast from "./ast";
1617

1718
/**
1819
* Evaluate expression against input data

‎src/jsonata.ts

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { lookupMessage, createFrame } from './utils';
33
import { evaluate } from './evaluate';
44
import { defineFunction } from './signatures';
55
import { createStandardFrame } from './functions';
6+
import { ASTNode } from './ast';
67

78
export interface Options {
89
recover: boolean;
@@ -27,12 +28,10 @@ export type AST = any;
2728
* @returns {{evaluate: evaluate, assign: assign}} Evaluated expression
2829
*/
2930
export function jsonata(expr: string, options?: Partial<Options>): Expression {
30-
var ast;
31-
var errors;
31+
let ast: undefined | ASTNode = undefined;
32+
let errors: string[] = [];
3233
try {
33-
ast = parser(expr, options && options.recover);
34-
errors = ast.errors;
35-
delete ast.errors;
34+
ast = parser(expr, errors, options && options.recover);
3635
} catch (err) {
3736
// insert error message into structure
3837
err.message = lookupMessage(err);
@@ -59,7 +58,7 @@ export function jsonata(expr: string, options?: Partial<Options>): Expression {
5958
return {
6059
evaluate: function(input, bindings, callback) {
6160
// throw if the expression compiled with syntax errors
62-
if (typeof errors !== "undefined") {
61+
if (typeof errors !== "undefined" && errors.length>0) {
6362
var err: any = {
6463
code: "S0500",
6564
position: 0,

0 commit comments

Comments
 (0)