rename Lexer and Token for both json and yaml modules

Signed-off-by: George Lemon <georgelemon@protonmail.com>

Author: georgelemon

src/openparser/json.nim

@@ -49,7 +49,7 @@ type
   # JSON JsonParser
   #
   JsonTokenKind* = enum
-    ## Token kinds for JSON parsing
+    ## JsonToken kinds for JSON parsing
     jtkEof = "<EOF>"
     jtkLBrace = "{"
     jtkRBrace = "}"
@@ -63,22 +63,22 @@ type
     jtkFalse = "<false>"
     jtkNull = "<null>"
 
-  Lexer = ref object
+  JsonLexer = ref object
     input: string
     data: ptr UncheckedArray[char]
     len: int
     pos: int
     line, col: int
     current: char
 
-  Token* = ref object
+  JsonToken* = ref object
     kind*: JsonTokenKind
     value*: string
     line*, col*, pos*: int
 
   JsonParser* = object
-    lexer: Lexer
-    prev*, curr*, next*: Token
+    lexer: JsonLexer
+    prev*, curr*, next*: JsonToken
     currentField*: Option[string] # for context-aware parseHooks
       ## The name of the current field being parsed, if applicable. This is set
       ## before calling parseHook for a field value, allowing parseHooks to have
@@ -97,11 +97,11 @@ const
   unexpectedTokenExpected = "Got `$1`, expected $2"
   unexpectedChar = "Unexpected character `$1`"
 
-proc charAt(l: Lexer, idx: int): char {.inline.} =
+proc charAt(l: JsonLexer, idx: int): char {.inline.} =
   if idx < 0 or idx >= l.len: return '\0'
   if l.data != nil: l.data[idx] else: l.input[idx]
 
-proc getContext(l: Lexer, posOverride: int = -1): string =
+proc getContext(l: JsonLexer, posOverride: int = -1): string =
   # Show the full current line and place caret at exact token position.
   let rawPos = if posOverride >= 0: posOverride else: l.pos
   let atPos = max(0, min(rawPos, l.len))
@@ -125,7 +125,7 @@ proc getContext(l: Lexer, posOverride: int = -1): string =
   let markerPos = max(0, min(snippet.len, atPos - lineStart))
   result = snippet & "\n" & " ".repeat(markerPos) & "^"
 
-proc error(l: var Lexer, msg: string) =
+proc error(l: var JsonLexer, msg: string) =
   # Raise a lexer error
   let context = getContext(l)
   raise newException(OpenParserJsonError, ("\n" & context & "\n" & "Error ($1:$2) " % [$l.line, $l.col]) & msg)
@@ -524,25 +524,25 @@ proc arrayToJson*(v, valImpl: NimNode, opts: JsonOptions = nil): NimNode =
       move(str) # return the JSON string
 
 #
-# Lexer API
+# JsonLexer API
 #
-proc `$`(tk: Token): string =
-  ## Convert Token to string
+proc `$`(tk: JsonToken): string =
+  ## Convert JsonToken to string
   result = "TOKEN<kind: " & $tk.kind & 
            (if tk.value.len > 0: ", value:" & tk.value else: "") & 
            ", line:" & $tk.line & ", col:" & $tk.col & ">"
 
-proc nextToken(parser: var JsonParser): Token {.discardable.}
+proc nextToken(parser: var JsonParser): JsonToken {.discardable.}
 
-proc newLexer(input: string): Lexer =
-  result = Lexer(input: input, data: nil, len: input.len, pos: 0, line: 1, col: 1)
+proc newJsonLexer(input: string): JsonLexer =
+  result = JsonLexer(input: input, data: nil, len: input.len, pos: 0, line: 1, col: 1)
   result.current = result.charAt(0)
 
-proc newLexer(mem: pointer, size: int): Lexer =
-  result = Lexer(data: cast[ptr UncheckedArray[char]](mem), len: size, pos: 0, line: 1, col: 1)
+proc newJsonLexer(mem: pointer, size: int): JsonLexer =
+  result = JsonLexer(data: cast[ptr UncheckedArray[char]](mem), len: size, pos: 0, line: 1, col: 1)
   result.current = result.charAt(0)
 
-proc advance(l: var Lexer) =
+proc advance(l: var JsonLexer) =
   if l.pos < l.len - 1:
     inc l.pos
     l.current = l.charAt(l.pos)
@@ -551,7 +551,7 @@ proc advance(l: var Lexer) =
     l.pos = l.len
     l.current = '\0'
 
-proc matchKeyword(l: var Lexer, kw: string): bool =
+proc matchKeyword(l: var JsonLexer, kw: string): bool =
   if l.pos + kw.len > l.len: return false
   for i, c in kw:
     if l.charAt(l.pos + i) != c:
@@ -560,7 +560,7 @@ proc matchKeyword(l: var Lexer, kw: string): bool =
     advance(l)
   result = true
 
-proc skipWhitespace(l: var Lexer) =
+proc skipWhitespace(l: var JsonLexer) =
   while true:
     case l.current
     of {' ', '\t', '\n', '\r'}:
@@ -570,7 +570,7 @@ proc skipWhitespace(l: var Lexer) =
       advance(l)
     else: break
 
-proc readString(l: var Lexer): string =
+proc readString(l: var JsonLexer): string =
   result = ""
   while true:
     case l.current
@@ -611,7 +611,7 @@ proc readString(l: var Lexer): string =
       result.add(l.current)
       advance(l)
 
-proc readNumber(l: var Lexer): string =
+proc readNumber(l: var JsonLexer): string =
   result = ""
   if l.current == '-':
     result.add('-')
@@ -639,10 +639,10 @@ proc readNumber(l: var Lexer): string =
 #
 # JSON Parsing implementation to Nim objects
 #
-proc nextToken(parser: var JsonParser): Token =
+proc nextToken(parser: var JsonParser): JsonToken =
   # Get the next token from the lexer
   skipWhitespace(parser.lexer)
-  result = Token(
+  result = JsonToken(
     line: parser.lexer.line,
     col: parser.lexer.col,
     pos: parser.lexer.pos
@@ -708,7 +708,7 @@ proc parseHook*[T: Integers](parser: var JsonParser, v: var T)
 proc parseHook*[T: tuple](parser: var JsonParser, v: var T)
 proc skipValue*(parser: var JsonParser)
 
-proc advance*(parser: var JsonParser): Token {.discardable.} =
+proc advance*(parser: var JsonParser): JsonToken {.discardable.} =
   # Advance to the next token and return it
   parser.prev = parser.curr
   parser.curr = parser.next
@@ -1121,7 +1121,7 @@ proc parseArray(parser: var JsonParser, arr: var JsonNode) =
       parser.error(unexpectedToken % [$token.kind])
   parser.advance() # consume the closing ']'
 
-proc initParser(lexer: Lexer): JsonParser =
+proc initParser(lexer: JsonLexer): JsonParser =
   result = JsonParser(lexer: lexer)
   result.curr = result.nextToken()
   result.next = result.nextToken()
@@ -1154,22 +1154,22 @@ proc parseAnyRootL(parser: var JsonParser): JsonNode =
 
 proc fromJson*(str: string): JsonNode =
   ## Parse a JSON from `str` and returns the standard `JsonNode`
-  var parser = initParser(newLexer(str))
+  var parser = initParser(newJsonLexer(str))
   result = parseAnyRoot(parser)
 
 proc fromJsonL*(str: string): JsonNode =
   ## Parse line-delimited JSON from `str` and returns a `JsonNode` array
-  var parser = initParser(newLexer(str))
+  var parser = initParser(newJsonLexer(str))
   result = parseAnyRootL(parser)
 
 proc fromJson*(mapped: MemFile): JsonNode =
   ## Parse JSON directly from mapped memory.
-  var parser = initParser(newLexer(mapped.mem, mapped.size))
+  var parser = initParser(newJsonLexer(mapped.mem, mapped.size))
   result = parseAnyRoot(parser)
 
 proc fromJsonL*(mapped: MemFile): JsonNode =
   ## Parse line-delimited JSON directly from mapped memory.
-  var parser = initParser(newLexer(mapped.mem, mapped.size))
+  var parser = initParser(newJsonLexer(mapped.mem, mapped.size))
   result = parseAnyRootL(parser)
 
 proc fromJsonFile*(filename: string): JsonNode =
@@ -1214,7 +1214,7 @@ macro fromJsonMacro(x: typed, str: typed): untyped =
   add blockStmtList, quote do:
     var
       tmp = `objIdent`()
-      parser = JsonParser(lexer: newLexer(`str`))
+      parser = JsonParser(lexer: newJsonLexer(`str`))
     parser.curr = parser.nextToken()
     parser.next = parser.nextToken()
     parser.parseJson(tmp)

src/openparser/yaml.nim

@@ -41,7 +41,7 @@ type
     ytkComment
     ytkUnknown
 
-  Token* = ref object
+  YamlToken* = ref object
     ## Represents a lexical token produced by the YAML lexer
     kind*: YamlTokenKind
     value*: string
@@ -51,7 +51,7 @@ type
     wsno*: int
     indent*: int
 
-  Lexer* = object
+  YamlLexer* = object
     ## Performs lexical analysis on a YAML input string,
     ## producing tokens for the parser
     input: string
@@ -61,21 +61,21 @@ type
     current: char
 
   YamlParser* = object
-    ## Parses a sequence of tokens from the Lexer to build a YAMLObject
-    lex: Lexer
-    prev, curr, next: Token
+    ## Parses a sequence of tokens from the YamlLexer to build a YAMLObject
+    lex: YamlLexer
+    prev, curr, next: YamlToken
 
   YAML* = string
     ## A simple alias for YAML strings
 
   OpenParserYamlError* = object of CatchableError
     ## Exception type for errors encountered during YAML parsing or dumping
 
-proc newLexer*(input: string): Lexer =
-  ## Create a new Lexer for the given input string
-  Lexer(input: input, len: input.len, line: 1, col: 1)
+proc newYamlLexer*(input: string): YamlLexer =
+  ## Create a new YamlLexer for the given input string
+  YamlLexer(input: input, len: input.len, line: 1, col: 1)
 
-proc charAt(l: Lexer, idx: int): char {.inline.} =
+proc charAt(l: YamlLexer, idx: int): char {.inline.} =
   if idx < 0 or idx >= l.len: return '\0'
   else: return l.input[idx]
 
@@ -87,7 +87,7 @@ const
   unexpectedChar = "Unexpected character `$1`"
 
 
-proc getContext(l: Lexer, posOverride: int = -1): string =
+proc getContext(l: YamlLexer, posOverride: int = -1): string =
   # Show the full current line and place caret at exact token position.
   let rawPos = if posOverride >= 0: posOverride else: l.pos
   let atPos = max(0, min(rawPos, l.len))
@@ -111,7 +111,7 @@ proc getContext(l: Lexer, posOverride: int = -1): string =
   let markerPos = max(0, min(snippet.len, atPos - lineStart))
   result = snippet & "\n" & " ".repeat(markerPos) & "^"
 
-proc error(l: var Lexer, msg: string) =
+proc error(l: var YamlLexer, msg: string) =
   # Raise a lexer error
   let context = getContext(l)
   raise newException(OpenParserYamlError, ("\n" & context & "\n" & "Error ($1:$2) " % [$l.line, $l.col]) & msg)
@@ -133,7 +133,7 @@ proc error(p: var YamlParser, msg: string) =
     ("\n" & context & "\n" & "Error ($1:$2) " % [$atLine, $atCol]) & msg
   )
 
-proc advance(l: var Lexer) =
+proc advance(l: var YamlLexer) =
   if l.pos < l.len - 1:
     inc l.pos
     l.current = l.charAt(l.pos)
@@ -142,7 +142,7 @@ proc advance(l: var Lexer) =
     l.pos = l.len
     l.current = '\0'
 
-proc lineIndentAt(l: Lexer, idx: int): int {.inline.} =
+proc lineIndentAt(l: YamlLexer, idx: int): int {.inline.} =
   ## Indent of the logical line containing idx (spaces/tabs at line start).
   if idx < 0 or idx >= l.len: return 0
 
@@ -162,7 +162,7 @@ proc lineIndentAt(l: Lexer, idx: int): int {.inline.} =
     else:
       break
 
-proc skipWhitespace(l: var Lexer, wsBeforeToken: var int): int =
+proc skipWhitespace(l: var YamlLexer, wsBeforeToken: var int): int =
   # Skip whitespace/newlines
   wsBeforeToken = 0
   while true:
@@ -187,20 +187,20 @@ proc skipWhitespace(l: var Lexer, wsBeforeToken: var int): int =
     return 0
   result = lineIndentAt(l, l.pos)
 
-proc readIdentifier(l: var Lexer): string =
+proc readIdentifier(l: var YamlLexer): string =
   # Read an unquoted identifier (e.g. for keys or unquoted values)
   while l.current in {'a'..'z', 'A'..'Z', '0'..'9', '_', '-'}:
     result.add(l.current)
     advance(l)
 
-proc readComment(l: var Lexer): string =
+proc readComment(l: var YamlLexer): string =
   # Read from '#' to end of line (excluding newline)
   advance(l) # skip '#'
   while l.current notin {'\0', '\n', '\r'}:
     result.add(l.current)
     advance(l)
 
-proc readString(l: var Lexer, quote: char): string =
+proc readString(l: var YamlLexer, quote: char): string =
   # read a quoted string, handling escape sequences for double quotes
   while true:
     if l.current == '\0':
@@ -222,7 +222,7 @@ proc readString(l: var Lexer, quote: char): string =
       result.add(l.current)
     advance(l)
 
-proc readNumber(l: var Lexer, kind: var YamlTokenKind): string =
+proc readNumber(l: var YamlLexer, kind: var YamlTokenKind): string =
   result = ""
   kind = ytkInteger
 
@@ -253,7 +253,7 @@ proc readNumber(l: var Lexer, kind: var YamlTokenKind): string =
       result.add(l.current)
       advance(l)
 
-proc tokenText(t: Token): string =
+proc tokenText(t: YamlToken): string =
   case t.kind
   of ytkIdentifier, ytkString, ytkFloat, ytkInteger: t.value
   else: $t.kind
@@ -270,12 +270,12 @@ let tokens = {
   '>': ytkGT,
 }.toTable
 
-proc nextToken*(p: var YamlParser): Token =
+proc nextToken*(p: var YamlParser): YamlToken =
   ## Lexical analysis to produce the next token from the input
   var wsBefore = 0
   let lineIndent = skipWhitespace(p.lex, wsBefore)
 
-  result = Token()
+  result = YamlToken()
   result.line = p.lex.line
   result.col = p.lex.col
   result.pos = p.lex.pos
@@ -352,7 +352,7 @@ proc advance(p: var YamlParser) {.inline.} =
     p.curr = p.next
     p.next = p.nextToken()
 
-proc getScalarValue(t: Token): YamlNode =
+proc getScalarValue(t: YamlToken): YamlNode =
   # Convert a scalar token to a YamlNode based on its kind
   case t.kind
   of ytkString:
@@ -609,7 +609,7 @@ proc `$`*(yamlObject: YAMLObject): string =
   toJson(yamlObject)
 
 proc parseYAML*(input: YAML): YAMLObject =
-  var p = YamlParser(lex: Lexer(input: input, len: input.len, line: 1, col: 1))
+  var p = YamlParser(lex: YamlLexer(input: input, len: input.len, line: 1, col: 1))
   p.lex.current = p.lex.charAt(0)
   p.curr = p.nextToken()
   p.next = p.nextToken()
@@ -696,7 +696,7 @@ macro parseYamlMacro(x: typed, str: typed): untyped =
   add blockStmtList, quote do:
     var
       tmp = `objIdent`()
-      parser = YamlParser(lex: newLexer(`str`))
+      parser = YamlParser(lex: newYamlLexer(`str`))
     parser.curr = parser.nextToken()
     parser.next = parser.nextToken()
     parser.parseYAML(tmp)