src/toktok/lexer

Search:
Group by:

Types

openArrayParams = openArray[tuple[k: string, t: NimNode]]
Settings = object
  lexerName*, lexerTuple*, lexerTokenKind*: string
  tkModifier*: TokenModifierCallback
  tkPrefix*: string
  keepUnknown*: bool         ## Whether to keep unknown tokens (default false).
                             ## This is useful when building a Markdown parser (for example)
  useDefaultIdent*: bool     ## Disable default string-based handler
  useDefaultInt*: bool = true ## Disable default `int` handler 
  ## Generate a static `lexer.nim` file (default false)
  ## Use `-d:toktokStatic` to (re)generate lexer file.
  ## Useful in case you don't want `toktok` package
  ## to run every time you build your program.
  ## 
  ## Also, due to toktok limitations, you may want to
  ## go deeper and extend the generated lexer manually.
  ## 
  ## **Important!** Running `-d:toktokStatic` will overwrite
  ## your existing `lexer.nim`. Be careful
  keepChar*: bool

TKNode = ref object
  ## Token identifier
  case
  of tString:
    
  of tChar:
    
  of tRange:
    
  of tVariant:
    
  of tSet:
    
  of tHandler:
    
  else:
    nil
  
TokenModifierCallback = proc (tkName, tkPrefix: string): string

Lets

defaultTokenModifier {.compileTime.} = proc (tkName, tkPrefix: string): string = result = tkPrefix &
    capitalizeAscii(tkName)

Consts

defaultSettings = (lexerName: "Lexer", lexerTuple: "TokenTuple",
                   lexerTokenKind: "TokenKind", tkModifier: (:anonymous, nil),
                   tkPrefix: "tk", keepUnknown: true, useDefaultIdent: true,
                   useDefaultInt: true, enableStaticGen: false, keepChar: false)

Procs

proc addField(tkEnum: var NimNode; enumIdent, enumType: NimNode) {....raises: [],
    tags: [], forbids: [].}
proc addField(tkEnum: var NimNode; enumType: NimNode) {....raises: [], tags: [],
    forbids: [].}
proc newCaseStmt(caseNode: NimNode;
                 branches: openArray[tuple[cond, body: NimNode]];
                 elseBranch = newEmptyNode()): NimNode {....raises: [], tags: [],
    forbids: [].}
proc newExceptionStmt(exception: NimNode; msg: NimNode): NimNode {....raises: [],
    tags: [], forbids: [].}
proc newExclude(id: string): NimNode {....raises: [], tags: [], forbids: [].}
proc newImport(id: string): NimNode {....raises: [], tags: [], forbids: [].}
proc newInclude(id: string): NimNode {....raises: [], tags: [], forbids: [].}
proc newObject(id: string; fields: openArrayParams = []; parent = "";
               public = false): NimNode {....raises: [], tags: [], forbids: [].}
Create a new object 
proc newProc(id: string; params: openArray[tuple[k, t: string, vt: bool]];
             public = false; returnType, body = newEmptyNode()): NimNode {.
    ...raises: [], tags: [], forbids: [].}
Create a new procedure 
proc newTemplate(id: string; params: openArray[tuple[k, t: string, vt: bool]];
                 public = false; body = newEmptyNode()): NimNode {....raises: [],
    tags: [], forbids: [].}
Create a template TODO support for generic parameters 
proc newTupleType(id: string; fields: openArrayParams; public = false): NimNode {.
    ...raises: [], tags: [], forbids: [].}
Creates a new tuple type 
proc newWhenStmt(whenBranch: tuple[cond, body: NimNode]): NimNode {....raises: [],
    tags: [], forbids: [].}
Constructor for when statements. 
proc newWhenStmt(whenBranch: tuple[cond, body: NimNode]; elseBranch: NimNode): NimNode {.
    ...raises: [], tags: [], forbids: [].}
Constructor for when statements.

Macros

macro handlers(custom: untyped)

Define your own handlers. For example:

 proc handleClass(lex: var Lexer, kind: `TokenKind`) =
   # your code

macro registerTokens(settings: static Settings; tokens: untyped)

Made with Nim. Generated: 2023-08-11 02:41:48 UTC