Lexer

Constructors

this

this(string input, string[] delims)

Undocumented in source.

Members

Functions

advance

void advance()

Undocumented in source. Be warned that the author may not have intended to support it.

consumeDelimiter

string consumeDelimiter()

Undocumented in source. Be warned that the author may not have intended to support it.

empty

bool empty()

Undocumented in source. Be warned that the author may not have intended to support it.

forwardCompare

bool forwardCompare(string str)

Undocumented in source. Be warned that the author may not have intended to support it.

forwardJump

void forwardJump(size_t numChars)

Undocumented in source. Be warned that the author may not have intended to support it.

front

string front()

Undocumented in source. Be warned that the author may not have intended to support it.

getLexeme

string getLexeme()

Undocumented in source. Be warned that the author may not have intended to support it.

isNewline

bool isNewline(dchar c)

Undocumented in source. Be warned that the author may not have intended to support it.

isWhitespace

bool isWhitespace(dchar c)

Undocumented in source. Be warned that the author may not have intended to support it.

moveFront

string moveFront()

Undocumented in source. Be warned that the author may not have intended to support it.

opApply

int opApply(int delegate(string) dg)

Undocumented in source. Be warned that the author may not have intended to support it.

opApply

int opApply(int delegate(size_t, string) dg)

Undocumented in source. Be warned that the author may not have intended to support it.

popFront

void popFront()

Undocumented in source. Be warned that the author may not have intended to support it.

pos

uint pos()

Undocumented in source. Be warned that the author may not have intended to support it.

position

uint position()

Undocumented in source. Be warned that the author may not have intended to support it.

Variables

current

dchar current;

Undocumented in source.

currentSize

uint currentSize;

Undocumented in source.

dec

UTF8Decoder dec;

Undocumented in source.

delims

string[] delims;

Undocumented in source.

ignoreNewlines

bool ignoreNewlines;

Undocumented in source.

ignoreWhitespaces

bool ignoreWhitespaces;

Undocumented in source.

index

uint index;

Undocumented in source.

input

string input;

Undocumented in source.

import std.array: array;
import std.range: iota;

string[] delims = ["(", ")", ";", " ", "{", "}", ".", "\n", "\r", "=", "++", "<"];
auto input = "for (int i=0; i<arr.length; ++i)\r\n{doThing();}\n";
auto lexer = new Lexer(input, delims);

string[] arr;
while(true) {
    auto lexeme = lexer.getLexeme();
    if(lexeme.length == 0) {
        break;
    }
    arr ~= lexeme;
}
auto reference = [
    "for", " ", "(", "int", " ", "i", "=", "0", ";", " ", "i", "<",
    "arr", ".", "length", ";", " ", "++", "i", ")", "\n", "{", "doThing",
    "(", ")", ";", "}", "\n" ];
assert(arr == reference);

lexer = new Lexer(input, delims);
arr = array(lexer);
assert(arr == reference);

input = "";
lexer = new Lexer(input, delims);
assert(lexer.getLexeme().length == 0);

• Lexer