Skip to content

Commit 5b12227

Browse files
byCedricbyCedric
authored
feat: add greedy newline tokenizer (#25)
1 parent 2b679ff commit 5b12227

3 files changed

Lines changed: 602 additions & 88 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ in sync with the written specification on conventionalcommits.org.
6060
/* Any non-newline whitespace: */
6161
<whitespace> ::= <ZWNBSP> | <TAB> | <VT> | <FF> | <SP> | <NBSP> | <USP>
6262
63-
<message> ::= <summary>, <newline>+, <body>, <newline>*, <footer>+
64-
| <summary>, <newline>*, <footer>+
63+
<message> ::= <summary>, <newline>+, <body>, (<newline>+, <footer>)*
64+
| <summary>, (<newline>+, <footer>)*
6565
| <summary>, <newline>*
6666
6767
/* "!" should be added to the AST as a <breaking-change> node with the value "!" */
@@ -78,9 +78,9 @@ in sync with the written specification on conventionalcommits.org.
7878
* <text> tokens of <body-text> should be appended as children to <body> */
7979
<body-text> ::= [<breaking-change>, ":", <whitespace>*], text
8080
/* Note: <pre-footer> is used during parsing, but not returned in the AST. */
81-
<pre-footer> ::= <newline>*, <footer>+
81+
<pre-footer> ::= <newline>+, <footer>
8282
83-
<footer> ::= <token>, <separator>, <whitespace>*, <value>, [<newline>]
83+
<footer> ::= <token>, <separator>, <whitespace>*, <value>
8484
/* "!" should be added to the AST as a <breaking-change> node with the value "!" */
8585
<token> ::= <breaking-change>
8686
| <type>, "(" <scope> ")", ["!"]

‎lib/parser.js‎

Lines changed: 81 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@ const Scanner = require('./scanner')
22
const { isWhitespace, isNewline, isParens } = require('./type-checks')
33

44
/*
5-
* <message> ::= <summary>, <newline>*, <body>, <newline>*, <footer>+
6-
* | <summary>, <newline>*, <footer>+
5+
* <message> ::= <summary>, <newline>+, <body>, (<newline>+, <footer>)*
6+
* | <summary>, (<newline>+, <footer>)*
77
* | <summary>, <newline>*
88
*
99
*/
1010
function message (commitText) {
1111
const scanner = new Scanner(commitText.trim())
1212
const node = scanner.enter('message', [])
1313

14-
// <summary>
14+
// <summary> ...
1515
const s = summary(scanner)
1616
if (s instanceof Error) {
1717
throw s
@@ -22,27 +22,49 @@ function message (commitText) {
2222
return scanner.exit(node)
2323
}
2424

25-
// <summary> <newline>* <body>
26-
if (isNewline(scanner.peek())) {
27-
// TODO(@byCedric): include <newline>* in AST.
28-
while (isNewline(scanner.peek())) {
29-
scanner.next()
30-
}
25+
let nl
26+
let b
27+
// ... <newline>* <body> ...
28+
nl = newline(scanner)
29+
if (nl instanceof Error) {
30+
throw nl
3131
} else {
32-
throw scanner.abort(node)
32+
node.children.push(nl)
33+
b = body(scanner)
34+
if (b instanceof Error) {
35+
b = null
36+
} else {
37+
node.children.push(b)
38+
}
3339
}
34-
const b = body(scanner)
35-
if (!(b instanceof Error)) {
36-
node.children.push(b)
40+
if (scanner.eof()) {
41+
return scanner.exit(node)
3742
}
38-
// TODO(@byCedric): include <newline>* in AST.
39-
while (isNewline(scanner.peek())) {
40-
scanner.next()
43+
44+
// ... <newline>* <footer>+
45+
if (b) {
46+
nl = newline(scanner)
47+
if (nl instanceof Error) {
48+
throw nl
49+
} else {
50+
node.children.push(nl)
51+
}
4152
}
4253
while (!scanner.eof()) {
4354
const f = footer(scanner)
44-
if (!(f instanceof Error)) node.children.push(f)
55+
if (f instanceof Error) {
56+
break
57+
} else {
58+
node.children.push(f)
59+
}
60+
nl = newline(scanner)
61+
if (nl instanceof Error) {
62+
break
63+
} else {
64+
node.children.push(nl)
65+
}
4566
}
67+
4668
return scanner.exit(node)
4769
}
4870

@@ -189,10 +211,13 @@ function body (scanner) {
189211
const t = text(scanner)
190212
node.children.push(t)
191213
// <newline>, <body>*
192-
if (isNewline(scanner.peek())) {
193-
scanner.next()
214+
const nl = newline(scanner)
215+
if (!(nl instanceof Error)) {
194216
const b = body(scanner)
195-
if (!(b instanceof Error)) {
217+
if (b instanceof Error) {
218+
scanner.abort(nl)
219+
} else {
220+
node.children.push(nl)
196221
Array.prototype.push.apply(node.children, b.children)
197222
}
198223
}
@@ -204,19 +229,17 @@ function body (scanner) {
204229
*/
205230
function preFooter (scanner) {
206231
const node = scanner.enter('pre-footer', [])
207-
while (isNewline(scanner.peek())) {
208-
scanner.next()
209-
}
210232
let f
211233
while (!scanner.eof()) {
234+
newline(scanner)
212235
f = footer(scanner)
213236
if (f instanceof Error) return scanner.abort(node)
214237
}
215238
return scanner.exit(node)
216239
}
217240

218241
/*
219-
* <footer> ::= <token> <separator> <whitespace>* <value> <newline>?
242+
* <footer> ::= <token> <separator> <whitespace>* <value>
220243
*/
221244
function footer (scanner) {
222245
const node = scanner.enter('footer', [])
@@ -231,6 +254,7 @@ function footer (scanner) {
231254
// <separator>
232255
const s = separator(scanner)
233256
if (s instanceof Error) {
257+
scanner.abort(node)
234258
return s
235259
} else {
236260
node.children.push(s)
@@ -242,16 +266,15 @@ function footer (scanner) {
242266
node.children.push(ws)
243267
}
244268

245-
// <value> <newline>?
269+
// <value>
246270
const v = value(scanner)
247271
if (v instanceof Error) {
272+
scanner.abort(node)
248273
return v
249274
} else {
250275
node.children.push(v)
251276
}
252-
if (isNewline(scanner.peek())) {
253-
scanner.next()
254-
}
277+
255278
return scanner.exit(node)
256279
}
257280

@@ -330,18 +353,24 @@ function value (scanner) {
330353
*/
331354
function continuation (scanner) {
332355
const node = scanner.enter('continuation', [])
333-
if (isNewline(scanner.peek())) {
334-
scanner.next()
335-
const ws = whitespace(scanner)
336-
if (ws instanceof Error) {
337-
return ws
338-
} else {
339-
node.children.push(ws)
340-
node.children.push(text(scanner))
341-
}
356+
// <newline>
357+
const nl = newline(scanner)
358+
if (nl instanceof Error) {
359+
return nl
342360
} else {
343-
return scanner.abort(node)
361+
node.children.push(nl)
362+
}
363+
364+
// <whitespace> <text>
365+
const ws = whitespace(scanner)
366+
if (ws instanceof Error) {
367+
scanner.abort(node)
368+
return ws
369+
} else {
370+
node.children.push(ws)
371+
node.children.push(text(scanner))
344372
}
373+
345374
return scanner.exit(node)
346375
}
347376

@@ -385,4 +414,18 @@ function whitespace (scanner) {
385414
return scanner.exit(node)
386415
}
387416

417+
/*
418+
* <newline>+ ::= [<CR>], <LF>
419+
*/
420+
function newline (scanner) {
421+
const node = scanner.enter('newline', '')
422+
while (isNewline(scanner.peek())) {
423+
node.value += scanner.next()
424+
}
425+
if (node.value === '') {
426+
return scanner.abort(node, ['<CR><LF>', '<LF>'])
427+
}
428+
return scanner.exit(node)
429+
}
430+
388431
module.exports = message

0 commit comments

Comments
 (0)