diff --git a/.github/workflows/d.yml b/.github/workflows/d.yml index 8ac08696..e82fc4ca 100644 --- a/.github/workflows/d.yml +++ b/.github/workflows/d.yml @@ -6,9 +6,9 @@ name: D on: push: - branches: [ "vardec_varass_dependency" ] + branches: [ "**" ] pull_request: - branches: [ "vardec_varass_dependency" ] + branches: [ "**" ] workflow_dispatch: permissions: @@ -63,9 +63,23 @@ jobs: uses: dlang-community/setup-dlang@v1 with: compiler: ${{ matrix.dc }} + ref: ${{ github.head_ref }} + + - name: Install Doveralls (code coverage tool) + run: | + # wget -O doveralls "https://github.com/ColdenCullen/doveralls/releases/download/v1.1.5/doveralls_linux_travis" + # mv doveralls_linux_travis doveralls + # chmod +x doveralls + dub fetch doveralls + sudo apt update + sudo apt install libcurl4-openssl-dev - name: DUB unit tests with coverage run: dub test --coverage + + - name: Coverage upload + run: | + dub run doveralls -- -t ${{secrets.COVERALLS_REPO_TOKEN}} - uses: actions/upload-artifact@v3 with: @@ -416,7 +430,7 @@ jobs: run: | ./tlang compile source/tlang/testing/simple_function_decls.t ./tlang.out - - name: Simple function (only) decls + - name: Simple variables (only) decls run: | ./tlang compile source/tlang/testing/simple_variables_only_decs.t ./tlang.out diff --git a/README.md b/README.md index e4cf2f04..a28c63cc 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ tlang ===== -[![D](https://github.com/tbklang/tlang/actions/workflows/d.yml/badge.svg?branch=vardec_varass_dependency)](https://github.com/tbklang/tlang/actions/workflows/d.yml) +[![D](https://github.com/tbklang/tlang/actions/workflows/d.yml/badge.svg?branch=vardec_varass_dependency)](https://github.com/tbklang/tlang/actions/workflows/d.yml) [![Coverage Status](https://coveralls.io/repos/github/tbklang/tlang/badge.svg?branch=vardec_varass_dependency)](https://coveralls.io/github/tbklang/tlang?branch=vardec_varass_dependency) Official Tristan Language project compiler diff --git a/source/tlang/compiler/core.d b/source/tlang/compiler/core.d index 2288be51..be2bb2f8 100644 --- a/source/tlang/compiler/core.d +++ b/source/tlang/compiler/core.d @@ -13,7 +13,7 @@ import tlang.compiler.typecheck.exceptions; import core.stdc.stdlib; import tlang.compiler.codegen.emit.core; import tlang.compiler.codegen.emit.dgen; -import misc.exceptions : TError; +import misc.exceptions; import tlang.compiler.codegen.mapper.core : SymbolMapper; import tlang.compiler.codegen.mapper.hashmapper : HashMapper; import tlang.compiler.codegen.mapper.lebanese : LebaneseMapper; @@ -247,6 +247,27 @@ public class Compiler } } +/** + * Opens the source file at the given path, reads the data + * and returns it + * + * Params: + * sourceFile = the path to the file to open + * Returns: the source data + */ +private string gibFileData(string sourceFile) +{ + File sourceFileFile; + sourceFileFile.open(sourceFile); /* TODO: Error handling with ANY file I/O */ + ulong fileSize = sourceFileFile.size(); + byte[] fileBytes; + fileBytes.length = fileSize; + fileBytes = sourceFileFile.rawRead(fileBytes); + sourceFileFile.close(); + + return cast(string)fileBytes; +} + /** * Performs compilation of the provided module(s) * @@ -283,26 +304,220 @@ void beginCompilation(string[] sourceFiles) } } +/** + * Tests the following pipeline: + * + * 1. lexing -> parsing -> typecheck/codegen -> emit (DGen) + * + * Kinds of tests: + * + * 1. Positive tests (must pass) + */ unittest { - // TODO: Add tests here for our `simple_.t` tests or put them in DGen, I think here is better - // FIXME: Crashes and I think because too fast or actually bad state? Maybe something is not being - // cleared, I believe this may be what is happening - // ... see issue #88 - // ... UPDATE: It seems to be any unit test..... mhhhh. - // string[] testFiles = ["source/tlang/testing/simple_while.t" - // ]; - - // // "source/tlang/testing/simple_functions.t", - // // "source/tlang/testing/simple_while.t", - // // "source/tlang/testing/simple_for_loops.t", - // // "source/tlang/testing/simple_cast.t", - // // "source/tlang/testing/simple_conditionals.t", - // // "source/tlang/testing/nested_conditionals.t", - // // "source/tlang/testing/simple_discard.t" - // foreach(string testFile; testFiles) - // { - // beginCompilation([testFile]); - // } + // TODO: Ensure up to date with d.yml + string[] testFiles = [ + "source/tlang/testing/simple_functions.t", + "source/tlang/testing/simple_direct_func_call.t", + "source/tlang/testing/simple_function_recursion_factorial.t", + + "source/tlang/testing/simple_conditionals.t", + "source/tlang/testing/nested_conditionals.t", + "source/tlang/testing/simple_function_decls.t", + "source/tlang/testing/simple_variables_only_decs.t", + "source/tlang/testing/simple_variables_decls_ass.t", + "source/tlang/testing/simple_while.t", + + "source/tlang/testing/simple_for_loops.t", + "source/tlang/testing/simple_cast.t", + + "source/tlang/testing/simple_pointer.t", + "source/tlang/testing/simple_pointer_cast_le.t", + + "source/tlang/testing/simple_stack_arrays4.t", + "source/tlang/testing/simple_stack_array_coerce.t", + "source/tlang/testing/simple_stack_array_coerce_ptr_syntax.t", + "source/tlang/testing/complex_stack_array_coerce.t", + + + "source/tlang/testing/complex_stack_arrays1.t", + "source/tlang/testing/simple_arrays.t", + "source/tlang/testing/simple_arrays2.t", + "source/tlang/testing/simple_arrays4.t", + + + "source/tlang/testing/simple_pointer_array_syntax.t", + ]; + foreach(string testFile; testFiles) + { + beginCompilation([testFile]); + } } +/** + * Tests the following pipeline: + * + * 1. lexing -> parsing -> typecheck/codegen -> emit (DGen) + * + * Kinds of tests: + * + * 1. Negative tests (must fail) + */ +unittest +{ + // TODO: Be specific about the catches maybe + string[] failingTestFiles = [ + "source/tlang/testing/simple_function_return_type_check_bad.t" + ]; + + foreach(string testFile; failingTestFiles) + { + try + { + beginCompilation([testFile]); + assert(false); + } + catch(TError) + { + assert(true); + } + catch(Exception e) + { + assert(false); + } + } +} + +/** + * Tests the following pipeline: + * + * 1. lexing -> parsing -> typecheck/codegen + * + * Kinds of tests: + * + * 1. Positive tests (must pass) + * 2. Negative tests (must fail) + */ +unittest +{ + // TODO: Enesure we keep this up-to-date with the d.yml + string[] testFilesGood = [ + "source/tlang/testing/return/simple_return_expressionless.t", + "source/tlang/testing/return/simple_return_type.t", + "source/tlang/testing/typecheck/simple_function_call.t", + + "source/tlang/testing/simple_arrays.t", + "source/tlang/testing/simple_arrays2.t", + "source/tlang/testing/simple_arrays4.t", + + "source/tlang/testing/simple_stack_array_coerce.t", + "source/tlang/testing/complex_stack_arrays1.t", + + "source/tlang/testing/complex_stack_array_coerce_permutation_good.t", + "source/tlang/testing/simple1_module_positive.t", + "source/tlang/testing/simple2_name_recognition.t", + + "source/tlang/testing/simple_literals.t", + "source/tlang/testing/simple_literals3.t", + "source/tlang/testing/simple_literals5.t", + "source/tlang/testing/simple_literals6.t", + "source/tlang/testing/universal_coerce/simple_coerce_literal_good.t", + "source/tlang/testing/universal_coerce/simple_coerce_literal_good_stdalo.t", + "source/tlang/testing/simple_function_return_type_check_good.t" + ]; + + foreach(string testFileGood; testFilesGood) + { + string sourceText = gibFileData(testFileGood); + + try + { + File tmpFile; + tmpFile.open("/tmp/bruh", "wb"); + Compiler compiler = new Compiler(sourceText, tmpFile); + + // Lex + compiler.doLex(); + + // Parse + compiler.doParse(); + + // Dep gen/typecheck/codegen + compiler.doTypeCheck(); + + assert(true); + } + // On Error + catch(TError e) + { + assert(false); + } + // On Error + catch(Exception e) + { + gprintln("Yo, we should not be getting this but rather ONLY TErrors, this is a bug to be fixed", DebugType.ERROR); + assert(false); + } + } + + // TODO: ENsure we keep this up to date with the d.yml + string[] testFilesFail = [ + "source/tlang/testing/typecheck/simple_function_call_1.t", + + "source/tlang/testing/simple_stack_array_coerce_wrong.t", + + "source/tlang/testing/complex_stack_array_coerce_bad1.t", + "source/tlang/testing/complex_stack_array_coerce_bad2.t", + "source/tlang/testing/complex_stack_array_coerce_bad3.t", + + "source/tlang/testing/collide_container_module1.t", + "source/tlang/testing/collide_container_module2.t", + "source/tlang/testing/collide_container_non_module.t", + "source/tlang/testing/collide_container.t", + "source/tlang/testing/collide_member.t", + "source/tlang/testing/precedence_collision_test.t", + + "source/tlang/testing/else_if_without_if.pl", + + "source/tlang/testing/simple_literals2.t", + "source/tlang/testing/simple_literals4.t", + "source/tlang/testing/universal_coerce/simple_coerce_literal_bad.t", + "source/tlang/testing/universal_coerce/simple_coerce_literal_bad_stdalon.t", + "source/tlang/testing/simple_function_return_type_check_bad.t" + ]; + + foreach(string testFileFail; testFilesFail) + { + string sourceText = gibFileData(testFileFail); + + try + { + File tmpFile; + tmpFile.open("/tmp/bruh", "wb"); + Compiler compiler = new Compiler(sourceText, tmpFile); + + // Lex + compiler.doLex(); + + // Parse + compiler.doParse(); + + // Dep gen/typecheck/codegen + compiler.doTypeCheck(); + + // All of these checks should be failing + assert(false); + } + // On Error + catch(TError e) + { + assert(true); + } + // We should ONLY be getting TErrors + catch(Exception e) + { + gprintln("Got non TError, this is a bug that must be fixed", DebugType.ERROR); + assert(false); + } + } +} \ No newline at end of file diff --git a/source/tlang/compiler/lexer/core/lexer.d b/source/tlang/compiler/lexer/core/lexer.d index 0b0af864..258fa83f 100644 --- a/source/tlang/compiler/lexer/core/lexer.d +++ b/source/tlang/compiler/lexer/core/lexer.d @@ -4,6 +4,7 @@ module tlang.compiler.lexer.core.lexer; import tlang.compiler.lexer.core.tokens : Token; +import std.ascii : isDigit, isAlpha, isWhite; /** * Defines the interface a lexer must provide @@ -73,4 +74,163 @@ public interface LexerInterface * Returns: a `Token[]` containing all tokens */ public Token[] getTokens(); +} + +/** + * Human-readable names assigned + * to commonly used character + * constants + */ +public enum LexerSymbols : char +{ + L_PAREN = '(', + R_PAREN = ')', + SEMI_COLON = ';', + COMMA = ',', + L_BRACK = '[' , + R_BRACK = ']' , + PLUS = '+' , + MINUS = '-' , + FORWARD_SLASH = '/' , + PERCENT = '%' , + STAR = '*' , + AMPERSAND = '&' , + L_BRACE = '{' , + R_BRACE = '}' , + EQUALS = '=' , + SHEFFER_STROKE = '|' , + CARET = '^' , + EXCLAMATION = '!' , + TILDE = '~' , + DOT = '.' , + COLON = ':', + SPACE = ' ', + TAB = '\t', + NEWLINE = '\n', + DOUBLE_QUOTE = '"', + SINGLE_QUOTE = '\'' , + BACKSLASH = '\\' , + UNDERSCORE = '_' , + LESS_THAN = '<' , + BIGGER_THAN = '>' , + + ESC_NOTHING = '0' , + ESC_CARRIAGE_RETURN = 'r' , + ESC_TAB = 't' , + ESC_NEWLINE = 'n' , + ESC_BELL= 'a' , + + ENC_BYTE = 'B' , + ENC_INT = 'I' , + ENC_LONG = 'L' , + ENC_WORD = 'W' , + ENC_UNSIGNED = 'U' , + ENC_SIGNED = 'S' , +} + +/** + * Alias to `LexerSymbols` + */ +public alias LS = LexerSymbols; + +/** + * Checks if the provided character is an operator + * + * Params: + * c = the character to check + * Returns: `true` if it is a character, `false` + * otherwise + */ +public bool isOperator(char c) +{ + return c == LS.PLUS || c == LS.TILDE || c == LS.MINUS || + c == LS.STAR || c == LS.FORWARD_SLASH || c == LS.AMPERSAND || + c == LS.CARET || c == LS.EXCLAMATION || c == LS.SHEFFER_STROKE || + c == LS.LESS_THAN || c == LS.BIGGER_THAN; +} + +/** + * Checks if the provided character is a splitter + * + * Params: + * c = the character to check + * Returns: `true` if it is a splitter, `false` + * otherwise + */ +public bool isSplitter(char c) +{ + return c == LS.SEMI_COLON || c == LS.COMMA || c == LS.L_PAREN || + c == LS.R_PAREN || c == LS.L_BRACK || c == LS.R_BRACK || + c == LS.PERCENT || c == LS.L_BRACE || c == LS.R_BRACE || + c == LS.EQUALS || c == LS.DOT || c == LS.COLON || + isOperator(c) || isWhite(c); +} + +/** + * Checks if the provided character is a + * numerical size encoder + * + * Params: + * character = the character to check + * Returns: `true` if so, `false` otheriwse + */ +public bool isNumericalEncoder_Size(char character) +{ + return character == LS.ENC_BYTE || character == LS.ENC_WORD || + character == LS.ENC_INT || character == LS.ENC_LONG; +} + +/** + * Checks if the provided character is a + * numerical signage encoder + * + * Params: + * character = the character to check + * Returns: `true` if so, `false` otherwise + */ +public bool isNumericalEncoder_Signage(char character) +{ + return character == LS.ENC_SIGNED || character == LS.ENC_UNSIGNED; +} + +/** + * Checks if the provided character is + * either a numerical size encoder + * or signage encoder + * + * Params: + * character = the character to check + * Returns: `true` if so, `false` otherwise + */ +public bool isNumericalEncoder(char character) +{ + return isNumericalEncoder_Size(character) || + isNumericalEncoder_Signage(character); +} + +/** + * Checks if the given character is a valid + * escape character (something which would + * have followed a `\`) + * + * Params: + * character = the character to check + * Returns: `true` if so, `false` otherwise + */ +public bool isValidEscape_String(char character) +{ + return character == LS.BACKSLASH || character == LS.DOUBLE_QUOTE || character == LS.SINGLE_QUOTE || + character == LS.ESC_NOTHING || character == LS.ESC_NEWLINE || character == LS.ESC_CARRIAGE_RETURN || + character == LS.TAB || character == LS.ESC_BELL; +} + +/** + * Given a character return whether it is valid entry + * for preceding a '.'. + * + * Returns: `true` if so, otherwise `false` + */ +public bool isValidDotPrecede(char character) +{ + return character == LS.R_PAREN || character == LS.R_BRACK; // || isAlpha(character) || isDigit(character); } \ No newline at end of file diff --git a/source/tlang/compiler/lexer/kinds/arr.d b/source/tlang/compiler/lexer/kinds/arr.d new file mode 100644 index 00000000..719d7f3f --- /dev/null +++ b/source/tlang/compiler/lexer/kinds/arr.d @@ -0,0 +1,124 @@ +module tlang.compiler.lexer.kinds.arr; + +import tlang.compiler.lexer.core; + +/** + * An array-based tokenizer which takes a + * provided array of `Token[]`. useful + * for testing parser-only related things + * with concrete tokens + */ +public final class ArrLexer : LexerInterface +{ + /** + * The concrete token source + */ + private Token[] tokens; + + /** + * Position in the `tokens` array + */ + private ulong tokenPtr = 0; + + /** + * Constructs a new `ArrLexer` (dummy lexer) with + * the tokens already in concrete form in the + * provided array. + * + * Params: + * tokens = the `Token[]` + */ + this(Token[] tokens) + { + this.tokens = tokens; + } + + /** + * Returns the token at the current cursor + * position + * + * Returns: the `Token` + */ + public Token getCurrentToken() + { + return tokens[tokenPtr]; + } + + /** + * Moves the cursor one token forward + */ + public void nextToken() + { + tokenPtr++; + } + + /** + * Moves the cursor one token backwards + */ + public void previousToken() + { + tokenPtr--; + } + + /** + * Sets the position of the cursor + * + * Params: + * cursor = the new position + */ + public void setCursor(ulong cursor) + { + this.tokenPtr = cursor; + } + + /** + * Retrieves the cursor's current position + * + * Returns: the position + */ + public ulong getCursor() + { + return this.tokenPtr; + } + + /** + * Checks whether more tokens are available + * of not + * + * Returns: true if more tokens are available, false otherwise + */ + public bool hasTokens() + { + return tokenPtr < tokens.length; + } + + /** + * Get the line position of the lexer in the source text + * + * Returns: the position + */ + public ulong getLine() + { + return 0; // TODO: anything meaningful? + } + + /** + * Get the column position of the lexer in the source text + * + * Returns: the position + */ + public ulong getColumn() + { + return 0; // TODO: anything meaningful? + } + + /** + * Exhaustively provide a list of all tokens + * + * Returns: a `Token[]` containing all tokens + */ + public Token[] getTokens() + { + return tokens; + } +} \ No newline at end of file diff --git a/source/tlang/compiler/lexer/kinds/basic.d b/source/tlang/compiler/lexer/kinds/basic.d index ed48fd7c..3d0db441 100644 --- a/source/tlang/compiler/lexer/kinds/basic.d +++ b/source/tlang/compiler/lexer/kinds/basic.d @@ -4,11 +4,14 @@ module tlang.compiler.lexer.kinds.basic; import std.container.slist; +import std.string : replace; import gogga; import std.conv : to; -import std.ascii : isDigit; +import std.ascii : isDigit, isAlpha, isWhite; import tlang.compiler.lexer.core; +enum EMPTY = ""; + /** * Represents a basic lexer which performs the whole tokenization * process in one short via a call to `performLex()`, only after @@ -54,7 +57,7 @@ public final class BasicLexer : LexerInterface */ public override void previousToken() { - tokenPtr--; + tokenPtr--; } /** @@ -119,7 +122,6 @@ public final class BasicLexer : LexerInterface return tokens; } - /** * Lexer state data */ @@ -130,458 +132,243 @@ public final class BasicLexer : LexerInterface private string currentToken; /* Current token */ private ulong position; /* Current character position */ private char currentChar; /* Current character */ - private bool stringMode; /* Whether we are in a string "we are here" or not */ - private bool floatMode; /* Whether or not we are building a floating point constant */ - /* The tokens */ private Token[] tokens; + /** + * Constructs a new lexer with the given + * source code of which is should tokenize + * + * Params: + * sourceCode = the source text + */ this(string sourceCode) { this.sourceCode = sourceCode; } + /** + * Checks whether or not we could shift our + * source text pointer forward if it would + * be within the boundries of the source text + * or not + * + * Returns: `true` if within the boundries, + * `false` otherwise + */ private bool isForward() { - return position+1 < sourceCode.length; - } - - public bool isBackward() - { - return position-1 < sourceCode.length; + return position + 1 < sourceCode.length; } - - - /** - * Used for tokenising a2.b2 - * - * When the `.` is encountered - * and there are some characters - * behind it this checks if we can - * append a further dot to it - */ - private bool isBuildUpValidIdent() + /** + * Checks whether or not we could shift our + * source text pointer backwards and it it + * would be within the boundries of the source + * text or not + * + * Returns: `true` if within the boundries, + * `false` otherwise + */ + private bool isBackward() { - import tlang.compiler.symbols.check; - return isPathIdentifier(currentToken) || isIdentifier(currentToken); + return position - 1 < sourceCode.length; } /** * Returns true if we have a token being built * false otherwise + * + * Returns: `true` if we have a token built-up, + * `false` otherwise */ private bool hasToken() { return currentToken.length != 0; } - /* Perform the lexing process */ - /* TODO: Use return value */ + /** + * Performs the lexing process + * + * Throws: + * LexerException on error tokenizing + */ public void performLex() { - while(position < sourceCode.length) + currentChar = sourceCode[position]; + while (position < sourceCode.length) { // gprintln("SrcCodeLen: "~to!(string)(sourceCode.length)); // gprintln("Position: "~to!(string)(position)); - currentChar = sourceCode[position]; - if(floatMode == true) - { - if(isDigit(currentChar)) - { - /* tack on and move to next iteration */ - currentToken~=currentChar; - position++; - column++; - continue; - } - /* TODO; handle closer case and error case */ - else - { - /* TODO: Throw erropr here */ - if(isSpliter(currentChar)) - { - floatMode = false; - currentTokens ~= new Token(currentToken, line, column); - currentToken = ""; + // // currentChar = sourceCode[position]; + // gprintln("Current Char\"" ~ currentChar ~ "\""); + // gprintln("Current Token\"" ~ currentToken ~ "\""); + // gprintln("Match alpha check" ~ to!(bool)(currentChar == LS.UNDERSCORE || isAlpha(currentChar))); - /* We just flush and catch splitter in next round, hence below is commented out */ - // column++; - // position++; - } - else - { - throw new LexerException(this, "Floating point '"~currentToken~"' cannot be followed by a '"~currentChar~"'"); - } - } - } - else if(currentChar == ' ' && !stringMode) + if (isSplitter(currentChar)) { - /* TODO: Check if current token is fulled, then flush */ - if(currentToken.length != 0) + + if (currentToken.length != 0) { - currentTokens ~= new Token(currentToken, line, column); - currentToken = ""; + flush(); } - - column++; - position++; - } - else if(isSpliter(currentChar) && !stringMode) - { - /* The splitter token to finally insert */ + if (isWhite(currentChar) ) { + if (improvedAdvance()) { + continue; + } else { + break; + } + } /* The splitter token to finally insert */ string splitterToken; - gprintln("Build up: "~currentToken); - gprintln("Current char: "~currentChar); - - /* Check for case of `==` (where we are on the first `=` sign) */ - if(currentChar == '=' && isForward() && sourceCode[position+1] == '=') - { - /* Flush any current token (if exists) */ - if(currentToken.length) - { - currentTokens ~= new Token(currentToken, line, column); - currentToken = ""; + // gprintln("Build up: " ~ currentToken); + // gprintln("Current char, splitter: " ~ currentChar); + if (currentChar == LS.FORWARD_SLASH && isForward() && (sourceCode[position+1] == LS.FORWARD_SLASH || sourceCode[position+1] == LS.STAR)) { + if (!doComment()) { + break; } + } - // Create the `==` token - currentTokens ~= new Token("==", line, column); - - // Skip over the current `=` and the next `=` - position+=2; - - column+=2; - + /* Check for case of `==` or `=<` or `=>` (where we are on the first `=` sign) */ + if (currentChar == LS.EQUALS && isForward() && (sourceCode[position + 1] == LS.EQUALS || sourceCode[position + 1] == LS.LESS_THAN || sourceCode[position + 1] == LS.BIGGER_THAN)) + { + buildAdvance(); + buildAdvance(); + flush(); continue; } - /* FIXME: Add floating point support here */ - /* TODO: IF buildUp is all numerical and we have dot go into float mode */ - /* TODO: Error checking will need to be added */ - if(isNumericalStr(currentToken) && currentChar == '.') + /* Check for case of `<=` or `>=` */ + if ((currentChar == LS.LESS_THAN || currentChar == LS.BIGGER_THAN) && isForward() && (sourceCode[position + 1] == LS.EQUALS || sourceCode[position + 1] == LS.LESS_THAN || sourceCode[position + 1] == LS.BIGGER_THAN)) { - /* Tack on the dot */ - currentToken~="."; - - /* Enable floating point mode and go to next iteration*/ - floatMode = true; - gprintln("Float mode just got enabled: Current build up: \""~currentToken~"\""); - column++; - position++; + buildAdvance(); + buildAdvance(); + flush(); continue; } - /** * Here we check if we have a `.` and that the characters - * preceding us were all godd for an identifier + * preceding us were all good for an identifier */ import misc.utils; - - if(currentChar == '.' && hasToken() && isBuildUpValidIdent()) + + if (currentChar == LS.DOT) { - gprintln("Bruh"); - /** - * Now we check that we have a character infront of us - * and that it is a letter - * - * TODO: Add _ check too as that is a valid identifier start - */ - if(isForward() && isCharacterAlpha(sourceCode[position+1])) + if (isBackward() && isWhite(sourceCode[position - 1])) { - position++; - column+=1; - - currentToken ~= '.'; - - continue; + throw new LexerException(this, "Character '.' is not allowed to follow a whitespace."); + } + if (isForward() && isWhite(sourceCode[position + 1])) + { + throw new LexerException(this, "Character '.' is not allowed to precede a whitespace."); + } + else if (!hasToken() && (isBackward() && !isValidDotPrecede( + sourceCode[position - 1]))) + { + throw new LexerException(this, "Character '.' should be preceded by valid identifier or numerical."); } else { - throw new LexerException(this, "Expected a letter to follow the ."); + splitterToken = EMPTY ~ currentChar; + improvedAdvance(); } - - } + }else if (currentChar == LS.AMPERSAND && (position + 1) != sourceCode.length && sourceCode[position + 1] == LS.AMPERSAND) + { + splitterToken = "&&"; + improvedAdvance(2, false); + } /* Check if we need to do combinators (e.g. for ||, &&) */ /* TODO: Second operand in condition out of bounds */ - else if(currentChar == '|' && (position+1) != sourceCode.length && sourceCode[position+1] == '|') + else if (currentChar == LS.SHEFFER_STROKE && isForward() && sourceCode[position + 1] == LS.SHEFFER_STROKE) { splitterToken = "||"; - column += 2; - position += 2; - } - else if(currentChar == '&' && (position+1) != sourceCode.length && sourceCode[position+1] == '&') - { - splitterToken = "&&"; - column += 2; - position += 2; - } - else if (currentChar == '\n') /* TODO: Unrelated!!!!!, but we shouldn't allow this bahevaipur in string mode */ + improvedAdvance(2, false); + } else if (currentChar == LS.EXCLAMATION && isForward() && sourceCode[position + 1] == LS.EQUALS) { - line++; - column = 1; - - position++; + splitterToken = "!="; + improvedAdvance(2, false); + }else if (currentChar == LS.SHEFFER_STROKE) { + splitterToken = "|"; + improvedAdvance(1, false); + } else if (currentChar == LS.AMPERSAND) { + splitterToken = "&"; + improvedAdvance(1, false); + } else if (currentChar == LS.CARET) { + splitterToken = "^"; + improvedAdvance(1, false); + } else if (currentChar == LS.LESS_THAN) { + splitterToken = [LS.LESS_THAN]; + improvedAdvance(1, false); + } else if (currentChar == LS.BIGGER_THAN) { + splitterToken = [LS.BIGGER_THAN]; + improvedAdvance(1, false); + } + else if (isWhite(currentChar)) { + if (!improvedAdvance()) { + break; + } } else { - splitterToken = ""~currentChar; - column++; - position++; + splitterToken = EMPTY ~ currentChar; + improvedAdvance(); } - /* Flush the current token (if one exists) */ - if(currentToken.length) + if (currentToken.length) { - currentTokens ~= new Token(currentToken, line, column); - currentToken = ""; + flush(); } - + /* Add the splitter token (only if it isn't empty) */ - if(splitterToken.length) + if (splitterToken.length) { currentTokens ~= new Token(splitterToken, line, column); } } - else if(currentChar == '"') - { - /* If we are not in string mode */ - if(!stringMode) - { - /* Add the opening " to the token */ - currentToken ~= '"'; - - /* Enable string mode */ - stringMode = true; - } - /* If we are in string mode */ - else - { - /* Add the closing " to the token */ - currentToken ~= '"'; - - /* Flush the token */ - currentTokens ~= new Token(currentToken, line, column); - currentToken = ""; - - /* Get out of string mode */ - stringMode = false; + //else if (currentChar == LS.UNDERSCORE || ((!isSplitter(currentChar) && !isDigit(currentChar)) && currentChar != LS.DOUBLE_QUOTE && currentChar != LS.SINGLE_QUOTE && currentChar != LS.BACKSLASH)) { + else if (currentChar == LS.UNDERSCORE || isAlpha(currentChar)) { + gprintln("path ident String"); + if (!doIdentOrPath()) { + break; + } else { + continue; } - - column++; - position++; } - else if(currentChar == '\\') + else if (currentChar == LS.DOUBLE_QUOTE) { - /* You can only use these in strings */ - if(stringMode) - { - /* Check if we have a next character */ - if(position+1 != sourceCode.length && isValidEscape_String(sourceCode[position+1])) - { - /* Add to the string */ - currentToken ~= "\\"~sourceCode[position+1]; - - column += 2; - position += 2; - } - /* If we don't have a next character then raise error */ - else - { - throw new LexerException(this, "Unfinished escape sequence"); - } - } - else - { - throw new LexerException(this, "Escape sequences can only be used within strings"); + if (!doString()) { + break; } } - /* Character literal support */ - else if(!stringMode && currentChar == '\'') + else if (currentChar == LS.SINGLE_QUOTE) { - currentToken ~= "'"; - - /* Character literal must be next */ - if(position+1 != sourceCode.length) - { - /* TODO: Escape support for \' */ - - /* Get the character */ - currentToken ~= ""~sourceCode[position+1]; - column++; - position++; - - - /* Closing ' must be next */ - if(position+1 != sourceCode.length && sourceCode[position+1] == '\'') - { - /* Generate and add the token */ - currentToken ~= "'"; - currentTokens ~= new Token(currentToken, line, column); - - /* Flush the token */ - currentToken = ""; - - column += 2; - position += 2; - } - else - { - throw new LexerException(this, "Was expecting closing ' when finishing character literal"); - } - } - else - { - throw new LexerException(this, LexerError.EXHAUSTED_CHARACTERS, "EOSC reached when trying to get character literal"); + if (!doChar()) { + break; } } - /** - * If we are building up a number - * - * TODO: Build up token right at the end (#DuplicateCode) - */ - else if(isBuildUpNumerical()) - { - gprintln("jfdjkhfdjkhfsdkj"); - /* fetch the encoder segment */ - char[] encoderSegment = numbericalEncoderSegmentFetch(); - - gprintln("isBuildUpNumerical(): Enter"); - - /** - * If we don't have any encoders - */ - if(encoderSegment.length == 0) - { - /* We can add a signage encoder */ - if(isNumericalEncoder_Signage(currentChar)) - { - gprintln("Hello"); - - /* Check if the next character is a size (it MUST be) */ - if(isForward() && isNumericalEncoder_Size(sourceCode[position+1])) - { - currentToken ~= currentChar; - column++; - position++; - - - } - else - { - throw new LexerException(this, "You MUST specify a size encoder after a signagae encoder"); - } - - - - - } - /* We can add a size encoder */ - else if(isNumericalEncoder_Size(currentChar)) - { - currentToken ~= currentChar; - column++; - position++; - } - /* We can add more numbers */ - else if(isDigit(currentChar)) - { - currentToken ~= currentChar; - column++; - position++; - } - /* Splitter (TODO) */ - else if(isSpliter(currentChar)) - { - /* Add the numerical literal as a new token */ - currentTokens ~= new Token(currentToken, line, column); - - /* Add the splitter token if not a newline */ - if(currentChar != '\n') - { - currentTokens ~= new Token(""~currentChar, line, column); - } - - - /* Flush the token */ - currentToken = ""; - - /* TODO: Check these */ - column += 2; - position += 2; - } - /* Anything else is invalid */ - else - { - throw new LexerException(this, "Not valid TODO"); - } - } - /** - * If we have one encoder - */ - else if((encoderSegment.length == 1)) - { - /* Check what the encoder is */ - - /** - * If we had a signage then we must have a size after it - */ - if(isNumericalEncoder_Signage(encoderSegment[0])) - { - /** - * Size encoder must then follow - */ - if(isNumericalEncoder_Size(currentChar)) - { - currentToken ~= currentChar; - column++; - position++; - - /* Add the numerical literal as a new token */ - currentTokens ~= new Token(currentToken, line, column); - - /* Flush the token */ - currentToken = ""; - - } - /** - * Anything else is invalid - */ - else - { - throw new LexerException(this, "A size-encoder must follow a signage encoder"); - } - } - else - { - throw new LexerException(this, "Cannot have another encoder after a size encoder"); - } - } - /* It is impossible to reach this as flushing means we cannot add more */ - else - { - assert(false); + else if (isDigit(currentChar)){ + if (!doNumber()) { + break; } - - + currentToken = currentToken.replace("_", ""); } - /* Any other case, keep building the curent token */ - else + else if (currentChar == LS.BACKSLASH) { - currentToken ~= currentChar; - column++; - position++; + throw new LexerException(this, "Escape sequences can only be used within strings"); + } else { + throw new LexerException(this, "Unsupported Character in this position"); + //gprintln("Fuck " ~ " me col" ~ to!(string)(column)); } } /* If there was a token made at the end then flush it */ - if(currentToken.length) + if (currentToken.length) { currentTokens ~= new Token(currentToken, line, column); } @@ -589,311 +376,745 @@ public final class BasicLexer : LexerInterface tokens = currentTokens; } - private char[] numbericalEncoderSegmentFetch() - { - char[] numberPart; - ulong stopped; - for(ulong i = 0; i < currentToken.length; i++) - { - char character = currentToken[i]; + /** + * Processes an ident with or without a dot-path + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool doIdentOrPath () { + if (!buildAdvance()) { + flush(); + return false; + } - if(isDigit(character)) - { - numberPart~=character; - } - else - { - stopped = i; - break; + while (true) { + if (currentChar == LS.DOT) { + if (isForward() && (isSplitter(sourceCode[position + 1]) || isDigit(sourceCode[position + 1]))) { + throw new LexerException(this, "Invalid character in identifier build up."); + } else { + if (!buildAdvance()) { + throw new LexerException(this, "Invalid character in identifier build up."); + //return false; + } + } + } else if (isSplitter(currentChar)) { + flush(); + return true; + } else if (!(isAlpha(currentChar) || isDigit(currentChar) || currentChar == LS.UNDERSCORE)) { + throw new LexerException(this, "Invalid character in identifier build up."); + } else { + if (!buildAdvance()) { + return false; + } } } - - char[] remaining = cast(char[])currentToken[stopped..currentToken.length]; - - return remaining; } - /** - * Returns true if the current build up is entirely - * numerical - * - * FIXME: THis, probably by its own will pick up `UL` - * as a number, or even just `` - */ - private bool isBuildUpNumerical() + /** + * Tokenizes a character + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool doChar() { - import std.ascii : isDigit; + if(!buildAdvance()) + { + throw new LexerException(this, "Expected character, but got EOF"); + } + /* Character literal must be next */ + bool valid; + if(currentChar == LS.BACKSLASH) + { + valid = doEscapeCode(); + } + else + { + valid = buildAdvance(); + } + if(!valid) + { + throw new LexerException(this, "Expected ''', but got EOF"); + } + if(currentChar != LS.SINGLE_QUOTE) + { + throw new LexerException(this, "Expected ''', but got EOF"); + } + if(!buildAdvance()) + { + flush(); + return false; + } + flush(); + return true; + } - char[] numberPart; - ulong stopped; - for(ulong i = 0; i < currentToken.length; i++) + /** + * Tokenizes a string + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool doString() + { + if(!buildAdvance()) { - char character = currentToken[i]; + throw new LexerException(this, "Expected closing \", but got EOF"); + } - if(isDigit(character)) - { - numberPart~=character; - } - else - { - stopped = i; - break; + while (true) { + if (currentChar == LS.DOUBLE_QUOTE) { + if (!buildAdvance) { + flush(); + return false; + } + return true; + } else if (currentChar == LS.BACKSLASH) { + if (!doEscapeCode()) { + throw new LexerException(this, "Expected closing \", but got EOF"); + } + } else if (currentChar == LS.NEWLINE) { + throw new LexerException(this, "Expected closing \", but got NEWLINE"); + } else { + if (!buildAdvance()) { + throw new LexerException(this, "Expected closing \", but got EOF"); + } } } + } - /** - * We need SOME numerical stuff - */ - if(stopped == 0) - { + /** + * Lex a comment, start by consuming the '/' and setting a flag for + * multi-line based on the next character and consume. + * + * Enters a loop that looks for the end of the comment and if not + * builds up the comment. + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool doComment() { + buildAdvance(); + // if (!buildAdvance()) { + // flush(); + // return false; + // } + bool multiLine = currentChar == LS.STAR; + if (!buildAdvance()) { + if (multiLine) { + throw new LexerException(this, "Expected closing Comment, but got EOF"); + } else { + flush(); return false; + } + } + while (true) { + if (!multiLine && currentChar == LS.NEWLINE) { + flush(); + return advanceLine(); + } + if (multiLine && currentChar == LS.STAR && isForward() && sourceCode[position+1] == LS.FORWARD_SLASH) { + buildAdvance(); + if (!buildAdvance()) { + flush(); + return false; + } else { + return true; + } + } else { + if (!buildAdvance()) { + if (multiLine) + { + throw new LexerException(this, "Expected closing Comment, but got EOF"); + } + else + { + flush(); + return false; + } + } + } } + } - char[] remaining = cast(char[])currentToken[stopped..currentToken.length]; + /** + * Lex an escape code. If valid one id found, add it to the token, else throw Exception + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool doEscapeCode() { + if (!buildAdvance()) { + return false; + } + // currentToken ~= LS.BACKSLASH; + if (isValidEscape_String(currentChar)) { + return buildAdvance(); + } else { + throw new LexerException(this, "Invalid escape code"); + } + // flush(); + } - char lstEncoder; - for(ulong i = 0; i < remaining.length; i++) - { - char character = remaining[i]; + /** + * Lex a number, this method lexes a plain number, float or numerically encoded. + * The Float and numerically encoded numbers are deferred to other methods. + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool doNumber() { + while (true) { + if (isDigit(currentChar) || currentChar == LS.UNDERSCORE) { + if(!buildAdvance()) { + currentToken = currentToken.replace("_", ""); + flush(); + return false; + } + } else if (currentChar == LS.DOT) { + return doFloat(); + } else if (isNumericalEncoder(currentChar)) { + return doEncoder(); + } else { + return true; + } + } + } - if(!isNumericalEncoder(character)) - { + /** + * Lex a numerical encoder, looks for Signage followed by Size, or if there is + * no signage, just the size. + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool doEncoder() { + if (isNumericalEncoder_Signage(currentChar)) { + if (!buildAdvance() || !isNumericalEncoder_Size(currentChar)) { + throw new LexerException(this, "Expected size indicator B,I,L,W but got EOF"); + } + } + if (isNumericalEncoder_Size(currentChar)) { + if (!buildAdvance()) { + flush(); return false; + } else { + if (!isSplitter(currentChar)) { + throw new LexerException(this, "Expected splitter but got \"" ~ currentChar ~ "\"."); + } } } - + flush(); return true; - - - } - /** - * Given a string return true if all characters - * are digits, false otherwise and false if - * the string is empty - */ - private static bool isNumericalStr(string input) - { - /** - * If the given input is empty then return false - */ - if(input.length == 0) - { - return false; + /** + * Lex a floating point, the initial part of the number is lexed by the `doNumber()` + * method. Here we consume the '.' and consume digits until a splitter is reached. + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool doFloat() { + if (!buildAdvance()) { + throw new LexerException(this, "Floating point expected digit, got EOF."); + //return false; } + size_t count = 0; + bool valid = false; + while (true) { - /** - * If there are any characters in the string then - * check if all are digits - */ - for(ulong i = 0; i < input.length; i++) - { - char character = input[i]; - - if(!isDigit(character)) + if (isDigit(currentChar) || (count > 0 && currentChar == LS.UNDERSCORE)) { - return false; + /* tack on and move to next iteration */ + valid = true; + if (!buildAdvance()) { + currentToken = currentToken.replace("_", ""); + flush(); + return false; + } + count++; + continue; + } + else + { + /* TODO: Throw erropr here */ + if (isSplitter(currentChar) && valid) + { + currentToken = currentToken.replace("_", ""); + flush(); + return true; + } + else + { + throw new LexerException(this, "Floating point '" ~ currentToken ~ "' cannot be followed by a '" ~ currentChar ~ "'"); + } } } - - return true; } - private bool isSpliter(char character) + /** + * Flush the current token to the token buffer. + */ + private void flush() { - return character == ';' || character == ',' || character == '(' || - character == ')' || character == '[' || character == ']' || - character == '+' || character == '-' || character == '/' || - character == '%' || character == '*' || character == '&' || - character == '{' || character == '}' || character == '=' || - character == '|' || character == '^' || character == '!' || - character == '\n' || character == '~' || character =='.' || - character == ':'; //|| isNumericalEncoder(character); + currentTokens ~= new Token(currentToken, line, column); + currentToken = EMPTY; } - private bool isNumericalEncoder(char character) + /** + * Consume the current char into the current token + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool buildAdvance() { - return isNumericalEncoder_Size(character) || - isNumericalEncoder_Signage(character); + currentToken ~= currentChar; + return improvedAdvance(); } - private bool isNumericalEncoder_Size(char character) + /** + * Advances the source code pointer + * + * Params: + * inc = advancement counter, default 1 + * shouldFlush = whether or not to flush, default is `false` + * Returns: `true` if characters left in buffer, else `false` + */ + private bool improvedAdvance(int inc = 1, bool shouldFlush = false) { - return character == 'B' || character == 'W' || - character == 'I' || character == 'L'; + if (currentChar == LS.NEWLINE) + { + shouldFlush && flush(); + line++; + column = 1; + position++; + } + else + { + column += inc; + position += inc; + } + + if (position >= sourceCode.length) + { + return false; + } + currentChar = sourceCode[position]; + return true; } - private bool isNumericalEncoder_Signage(char character) + /** + * Advance the position, line and current token, reset the column to 1. + * + * Returns: `true` if characters left in buffer, else `false` + */ + private bool advanceLine() { - return character == 'S' || character == 'U'; + column = 1; + line++; + position++; + if (position >= sourceCode.length) + { + return false; + } + currentChar = sourceCode[position]; + return true; } +} - /* Supported escapes \" */ - public bool isValidEscape_String(char character) +version(unittest) +{ + /** + * Does a print out of some text just to show you + * where you are from within the caller + * + * Params: + * __LINE__ = line number (auto-filled) + * __MODULE__ = module name (auto-filled) + * __FUNCTION__ = function name (auto-filled) + */ + private void shout(int i = __LINE__, string mod = __MODULE__, string func = __FUNCTION__) { - return true; /* TODO: Implement me */ + gprintln("Unittest at "~to!(string)(i)~" in "~func~" (within module "~mod~")"); } } -/* Test input: `hello "world";` */ +/** + * Test input: `hello "world";` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = "hello \"world\";"; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("hello", 0, 0), new Token("\"world\"", 0, 0), new Token(";", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("hello", 0, 0), new Token("\"world\"", 0, 0), + new Token(";", 0, 0) + ]); +} + +/** + * Test input: `hello \n "world";` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "hello \n \"world\";"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("hello", 0, 0), new Token("\"world\"", 0, 0), + new Token(";", 0, 0) + ]); +} + +/** + * Test input: `hello "wo\nrld";` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "hello \"wo\nrld\";"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try { + currentLexer.performLex(); + } catch (LexerException) { + assert(true); + + } } -/* Test input: `hello "world"|| ` */ +/** + * Test input: `hello "world"|| ` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = "hello \"world\"|| "; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("hello", 0, 0), new Token("\"world\"", 0, 0), new Token("||", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("hello", 0, 0), new Token("\"world\"", 0, 0), + new Token("||", 0, 0) + ]); } -/* Test input: `hello "world"||` */ +/** + * Test input: `hello "world"&& ` + */ unittest { + shout(); import std.algorithm.comparison; - string sourceCode = "hello \"world\"||"; + + string sourceCode = "hello \"world\"&& "; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("hello", 0, 0), new Token("\"world\"", 0, 0), new Token("||", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("hello", 0, 0), new Token("\"world\"", 0, 0), + new Token("&&", 0, 0) + ]); +} + +/** + * Test input: `hello "wooorld"||` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "hello \"wooorld\"||"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("hello", 0, 0), new Token("\"wooorld\"", 0, 0), + new Token("||", 0, 0) + ]); } -/* Test input: `hello "world"|` */ +/** + * Test input: `hello "world"|` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = "hello \"world\";|"; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("hello", 0, 0), new Token("\"world\"", 0, 0), new Token(";", 0, 0), new Token("|", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("hello", 0, 0), new Token("\"world\"", 0, 0), + new Token(";", 0, 0), new Token("|", 0, 0) + ]); } -/* Test input: ` hello` */ +/** + * Test input: ` hello` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = " hello"; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); assert(currentLexer.getTokens() == [new Token("hello", 0, 0)]); } -/* Test input: `hello;` */ +/** + * Test input: `//trist` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "//trist"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [new Token("//trist", 0, 0)]); +} + +/** + * Test input: `/*trist\*\/` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "/*trist*/"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [new Token("/*trist*/", 0, 0)]); +} + +/** + * Test input: `/*t\nr\ni\ns\nt\*\/` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "/*t\nr\ni\ns\nt*/"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [new Token("/*t\nr\ni\ns\nt*/", 0, 0)]); +} + +/** + * Test input: `/*t\nr\ni\ns\nt\*\/ ` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "/*t\nr\ni\ns\nt*/ "; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [new Token("/*t\nr\ni\ns\nt*/", 0, 0)]); +} + +/** + * Test input: `//trist \n hello` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "//trist \n hello"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("//trist ", 0, 0), + new Token("hello", 0, 0), + ]); +} + +/** + * Test input: `hello;` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = " hello;"; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("hello", 0, 0), new Token(";", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("hello", 0, 0), new Token(";", 0, 0) + ]); +} + +/** + * Test input: `5+5` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "5+5"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("5", 0, 0), + new Token("+", 0, 0), + new Token("5", 0, 0), + ]); } -/* Test input: `hello "world\""` */ +/** + * Test input: `hello "world\""` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = "hello \"world\\\"\""; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("hello", 0, 0), new Token("\"world\\\"\"", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("hello", 0, 0), new Token("\"world\\\"\"", 0, 0) + ]); } -/* Test input: `'c'` */ +/** + * Test input: `'c'` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = "'c'"; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); assert(currentLexer.getTokens() == [new Token("'c'", 0, 0)]); } -/* Test input: `2121\n2121` */ +/** + * Test input: `2121\n2121` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = "2121\n2121"; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("2121", 0, 0), new Token("2121", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("2121", 0, 0), new Token("2121", 0, 0) + ]); } /** -* Test `=`` and `==` handling -*/ + * Test `=`` and `==` handling + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = " =\n"; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); assert(currentLexer.getTokens() == [new Token("=", 0, 0)]); import std.algorithm.comparison; + sourceCode = " = ==\n"; currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("=", 0, 0), new Token("==", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("=", 0, 0), new Token("==", 0, 0) + ]); import std.algorithm.comparison; + sourceCode = " ==\n"; currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); assert(currentLexer.getTokens() == [new Token("==", 0, 0)]); import std.algorithm.comparison; + sourceCode = " = =\n"; currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("=", 0, 0), new Token("=", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("=", 0, 0), new Token("=", 0, 0) + ]); import std.algorithm.comparison; + sourceCode = " ==, = ==\n"; currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("==", 0, 0), new Token(",", 0, 0), new Token("=", 0, 0), new Token("==", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("==", 0, 0), new Token(",", 0, 0), new Token("=", 0, 0), + new Token("==", 0, 0) + ]); // Test flushing of previous token import std.algorithm.comparison; + sourceCode = "i==i=\n"; currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); - assert(currentLexer.getTokens() == [new Token("i", 0, 0), new Token("==", 0, 0), new Token("i", 0, 0), new Token("=", 0, 0)]); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("i", 0, 0), new Token("==", 0, 0), new Token("i", 0, 0), + new Token("=", 0, 0) + ]); } /** -* Test: Literal value encoding -* -* Tests validity -*/ + * Test: Literal value encoding + * + * Tests validity + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode; BasicLexer currentLexer; @@ -901,66 +1122,843 @@ unittest sourceCode = "21L"; currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); assert(currentLexer.getTokens() == [new Token("21L", 0, 0)]); /* 21UL (valid) */ sourceCode = "21UL"; currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); assert(currentLexer.getTokens() == [new Token("21UL", 0, 0)]); - // /* 21U (invalid) */ - // sourceCode = "21U "; - // currentLexer = new Lexer(sourceCode); - // // gprintln(currentLexer.performLex()); - // bool status = currentLexer.performLex(); - // gprintln("Collected "~to!(string)(currentLexer.getTokens())); - // assert(!status); + /* 21U (invalid) */ + sourceCode = "21U "; + currentLexer = new BasicLexer(sourceCode); + // gprintln(currentLexer.performLex()); + try { + currentLexer.performLex(); + assert(false); + } catch (LexerException) { + assert(true); + } + /* 21ULa (invalid) */ + sourceCode = "21ULa"; + currentLexer = new BasicLexer(sourceCode); + // gprintln(currentLexer.performLex()); + try { + currentLexer.performLex(); + assert(false); + } catch (LexerException) { + assert(true); + } - // /* 21UL (valid) */ - // sourceCode = "21UL"; - // currentLexer = new Lexer(sourceCode); - // currentLexer.performLex(); - // gprintln("Collected "~to!(string)(currentLexer.getTokens())); - // assert(currentLexer.getTokens() == [new Token("21UL", 0, 0)]); + /* 21UL (valid) */ + sourceCode = "21SI"; + currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected "~to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [new Token("21SI", 0, 0)]); - + /* 21UL; (valid) */ + sourceCode = "21SI;"; + currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected "~to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("21SI", 0, 0), + new Token(";", 0, 0) + ]); } -/* Test input: `1.5` */ +/** + * Test input: `1.5` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = "1.5"; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); assert(currentLexer.getTokens() == [new Token("1.5", 0, 0)]); } /** -* Test correct handling of dot-operator for -* non-floating point cases -* -* Input: `new A().l.p.p;` -*/ + * Test correct handling of dot-operator for + * non-floating point cases + * + * Input: `new A().l.p.p;` + */ unittest { + shout(); import std.algorithm.comparison; + string sourceCode = "new A().l.p.p;"; BasicLexer currentLexer = new BasicLexer(sourceCode); currentLexer.performLex(); - gprintln("Collected "~to!(string)(currentLexer.getTokens())); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); assert(currentLexer.getTokens() == [ - new Token("new", 0, 0), - new Token("A", 0, 0), - new Token("(", 0, 0), - new Token(")", 0, 0), - new Token(".", 0, 0), - new Token("l.p.p", 0, 0), - new Token(";", 0, 0) - ]); + new Token("new", 0, 0), + new Token("A", 0, 0), + new Token("(", 0, 0), + new Token(")", 0, 0), + new Token(".", 0, 0), + new Token("l.p.p", 0, 0), + new Token(";", 0, 0) + ]); +} + +/** + * Tab testing + */ +unittest +{ + shout(); + /** + * Test tab dropping in front of a float. + * Test calssification: Valid + * Test input: `\t1.5` + */ + gprintln("Tab Unit Test"); + import std.algorithm.comparison; + + string sourceCode = "\t1.5"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [new Token("1.5", 0, 0)]); + + /** + * Test tab dropping before '.' of float. + * Catch fail for verification. + * Test calssification: Invalid + * Test input: `1\t.5` + */ + import std.algorithm.comparison; + + bool didFail = false; + sourceCode = "1\t.5"; + currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); + + /** + * Testing Float EOF after '.'. + * Test calssification: Invalid + * Test input: `1.` + */ + sourceCode = "1."; + currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + assert(false); + } + catch (LexerException e) + { + } + + /** + * Testing illegal backslash. + * Test calssification: Invalid + * Test input: `1.` + */ + sourceCode = "hello \\ "; + currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + assert(false); + } + catch (LexerException e) + { + } + + /** + * Test tab dropping after '.' of float. + * Catch fail for verification. + * Test calssification: Invalid + * Test input: `1.\t5` + */ + import std.algorithm.comparison; + + didFail = false; + sourceCode = "1.\t5"; + currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); + + /** + * Test tab dropping for an empty token array. + * Test calssification: Valid + * Test input: `\t\t\t\t\t` + */ + gprintln("Tab Unit Test"); + import std.algorithm.comparison; + + sourceCode = "\t\t\t\t\t"; + currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens().length == 0); +} + +/** + * Test correct handling of dot-operator for + * non-floating point cases where whitespace has been inserted before and after. + * Test Classification: Invalid + * + * Input: `new A() .l.p.p;` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + bool didFail = false; + string sourceCode = "new A(). l.p.p;"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException) + { + didFail = true; + } + assert(didFail); +} + +/** + * Test correct handling of dot-operator for + * non-floating point cases where whitespace has been inserted before and after. + * Test Classification: Invalid + * + * Input: `new A() . l.p.p;` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + bool didFail = false; + string sourceCode = "new A() . l.p.p;"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException) + { + didFail = true; + } + assert(didFail); +} + +unittest +{ + shout(); + + /** + * Test dot for fail on dot operator with no buildup and invalid lead + * Catch fail for verification. + * Test calssification: Invalid + * Test input: `1.5.5` + */ + import std.algorithm.comparison; + + bool didFail = false; + string sourceCode = "1.5.5"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); + + /** + * Test for fail on space following dot operator. + * Test Classification: Invalid + * Input: `1. a` + */ + didFail = false; + sourceCode = "1. a"; + currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); + + /** + * Test for correct lex space following paren + * Test Classification: Valid + * Input: `).x` + */ + sourceCode = ").x"; + currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token(")", 0, 0), + new Token(".", 0, 0), + new Token("x", 0, 0), + ]); + /** + * Test for fail on space preceding dot operator. + * Test Classification: Invalid + * Input: `1 .a` + */ + didFail = false; + sourceCode = "1 .a"; + currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Test newlines + * Test Classification: Valid + * Input: `\n\n\n\n` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "\n\n\n\n"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens().length == 0); +} + +/** + * Test for character escape codes + * + * Input: `'\\'` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "'\\\\'"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("'\\\\'", 0, 0), + ]); +} + +/** + * Test for character escape codes + * + * Input: `'\a'` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "'\\a'"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("'\\a'", 0, 0), + ]); +} + +/** + * Test for invalid escape sequence + * Input: `'\f'` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "\\f"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Test for invalid char in ident + * Input: `hello$k` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "hello$k"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Test for invalid char in ident + * Input: `$` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "$"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing Underscores in numbers + * + * Input: `1_ 1_2 1_2.3 1_2.3_ 1__2 1__2.3 1__.23__` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "1_ 1_2 1_2.3 1_2.3_ 1__2 1__2.3 1__.23__"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("1", 0, 0), + new Token("12", 0, 0), + new Token("12.3", 0, 0), + new Token("12.3", 0, 0), + new Token("12", 0, 0), + new Token("12.3", 0, 0), + new Token("1.23", 0, 0), + ]); +} + +/** + * Testing Comparison in numbers + * + * Input: `<= >= ==` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "<= >= =< => == != < > ^"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("<=", 0, 0), + new Token(">=", 0, 0), + new Token("=<", 0, 0), + new Token("=>", 0, 0), + new Token("==", 0, 0), + new Token("!=", 0, 0), + new Token("<", 0, 0), + new Token(">", 0, 0), + new Token("^", 0, 0), + ]); +} + +/** + * Testing Chars + * + * Input: `'a'` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "'a'"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("'a'", 0, 0), + ]); +} + +/** + * Test for invalid ident + * Input: `hello. ` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "hello. "; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Test for invalid ident + * Input: `hello.` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "hello."; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing Chars + * Input: `'` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "'"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing Chars + * Input: `'a` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "'a"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing Chars + * Input: `'aa` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "'aa"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing String EOF + * Input: `"a` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "\"a"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing String EOF + * Input: `"a` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "\""; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing String EOF + * Input: `"\` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "\"\\"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing Comment EOF + * Input: `/*` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "/*"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing Comment EOF + * Input: `/* ` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "/* "; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** +* Testing Line comment EOF +* +* Input: `//` +*/ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "//"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("//", 0, 0) + ]); +} + +/** + * Testing invalid Escape Code + * Input: `\p` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "\"\\p"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing invalid Escape Code + * Input: `\p` + */ +unittest +{ + shout(); + + bool didFail = false; + string sourceCode = "\\p"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + try + { + currentLexer.performLex(); + } + catch (LexerException e) + { + didFail = true; + } + assert(didFail); +} + +/** + * Testing comment + * + * Input: `'a' ` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "'a' "; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("'a'", 0, 0) + ]); +} + +/** + * Testing comment + * + * Input: `// \n` + */ +unittest +{ + shout(); + import std.algorithm.comparison; + + string sourceCode = "// \n"; + BasicLexer currentLexer = new BasicLexer(sourceCode); + currentLexer.performLex(); + gprintln("Collected " ~ to!(string)(currentLexer.getTokens())); + assert(currentLexer.getTokens() == [ + new Token("// ", 0, 0) + ]); } \ No newline at end of file diff --git a/source/tlang/compiler/parsing/core.d b/source/tlang/compiler/parsing/core.d index 72b8e25a..d84482c3 100644 --- a/source/tlang/compiler/parsing/core.d +++ b/source/tlang/compiler/parsing/core.d @@ -562,8 +562,12 @@ public final class Parser /* If it is a type */ if (symbolType == SymbolType.IDENT_TYPE) { - /* Might be a function, might be a variable, or assignment */ - structMember = parseName(); + /* Might be a function definition or variable declaration */ + structMember = parseTypedDeclaration(); + + /* Should have a semi-colon and consume it */ + expect(SymbolType.SEMICOLON, lexer.getCurrentToken()); + lexer.nextToken(); } /* If it is an accessor */ else if (isAccessor(lexer.getCurrentToken())) @@ -1880,8 +1884,12 @@ public final class Parser /* If it is a type */ if (symbolType == SymbolType.IDENT_TYPE) { - /* Might be a function, might be a variable, or assignment */ - structMember = parseName(); + /* Might be a function definition or variable declaration */ + structMember = parseTypedDeclaration(); + + /* Should have a semi-colon and consume it */ + expect(SymbolType.SEMICOLON, lexer.getCurrentToken()); + lexer.nextToken(); } /* If it is a class */ else if(symbolType == SymbolType.CLASS) @@ -2008,6 +2016,124 @@ public final class Parser return statement; } + import std.container.slist : SList; + private SList!(Token) commentStack; + private void pushComment(Token commentToken) + { + // Sanity check + assert(getSymbolType(commentToken) == SymbolType.SINGLE_LINE_COMMENT || + getSymbolType(commentToken) == SymbolType.MULTI_LINE_COMMENT + ); + + // Push it onto top of stack + commentStack.insertFront(commentToken); + } + //TODO: Add a popToken() (also think if we want a stack-based mechanism) + private bool hasCommentsOnStack() + { + return getCommentCount() != 0; + } + + private ulong getCommentCount() + { + import std.range : walkLength; + return walkLength(commentStack[]); + } + + private void parseComment() + { + gprintln("parseComment(): Enter", DebugType.WARNING); + + Token curCommentToken = lexer.getCurrentToken(); + + pushComment(curCommentToken); + + // TODO: Do something here like placing it on some kind of stack + gprintln("Comment is: '"~curCommentToken.getToken()~"'"); + lexer.nextToken(); // Move off comment + + gprintln("parseComment(): Leave", DebugType.WARNING); + } + + /** + * Tests the handling of comments + */ + unittest + { + import tlang.compiler.lexer.kinds.arr : ArrLexer; + + string sourceCode = `module myCommentModule; + // Hello`; + + LexerInterface currentLexer = new BasicLexer(sourceCode); + (cast(BasicLexer)currentLexer).performLex(); + + Parser parser = new Parser(currentLexer); + + try + { + Module modulle = parser.parse(); + + assert(parser.hasCommentsOnStack()); + assert(parser.getCommentCount() == 1); + } + catch(TError e) + { + assert(false); + } + + sourceCode = `module myCommntedModule; + /*Hello */ + + /* Hello*/`; + + currentLexer = new BasicLexer(sourceCode); + (cast(BasicLexer)currentLexer).performLex(); + parser = new Parser(currentLexer); + + try + { + Module modulle = parser.parse(); + + assert(parser.hasCommentsOnStack()); + assert(parser.getCommentCount() == 2); + } + catch(TError e) + { + assert(false); + } + + sourceCode = `module myCommentedModule; + + void function() + { + /*Hello */ + /* Hello */ + // Hello + //Hello + } + `; + + currentLexer = new BasicLexer(sourceCode); + (cast(BasicLexer)currentLexer).performLex(); + parser = new Parser(currentLexer); + + try + { + Module modulle = parser.parse(); + + assert(parser.hasCommentsOnStack()); + assert(parser.getCommentCount() == 4); + } + catch(TError e) + { + assert(false); + } + } + + // TODO: We need to add `parseComment()` + // support here (see issue #84) + // TODO: This ic currently dead code and ought to be used/implemented private Statement parseStatement(SymbolType terminatingSymbol = SymbolType.SEMICOLON) { gprintln("parseStatement(): Enter", DebugType.WARNING); @@ -2080,6 +2206,12 @@ public final class Parser { statement = parseDerefAssignment(); } + /* If it is a kind-of comment */ + else if(symbol == SymbolType.SINGLE_LINE_COMMENT || symbol == SymbolType.MULTI_LINE_COMMENT) + { + gprintln("COMMENTS NOT YET PROPERLY SUPOORTED", DebugType.ERROR); + parseComment(); + } /* Error out */ else { @@ -2303,6 +2435,12 @@ public final class Parser modulle.addStatement(externStatement); } + /* If it is a kind-of comment */ + else if(symbol == SymbolType.SINGLE_LINE_COMMENT || symbol == SymbolType.MULTI_LINE_COMMENT) + { + gprintln("COMMENTS NOT YET PROPERLY SUPOORTED", DebugType.ERROR); + parseComment(); + } else { expect("parse(): Unknown '" ~ tok.getToken() ~ "'"); diff --git a/source/tlang/compiler/symbols/check.d b/source/tlang/compiler/symbols/check.d index 9cd8c849..121c923b 100644 --- a/source/tlang/compiler/symbols/check.d +++ b/source/tlang/compiler/symbols/check.d @@ -290,6 +290,16 @@ public enum SymbolType */ GENERIC_TYPE_DECLARE, + /** + * Multi-line comment (frwd-slash-star) + */ + MULTI_LINE_COMMENT, + + /** + * Singleiline comment (frwd-slash-slash) + */ + SINGLE_LINE_COMMENT, + /** * Unknown symbol */ @@ -780,6 +790,16 @@ public SymbolType getSymbolType(Token tokenIn) { return SymbolType.STAR; } + /* Multi-line comment (fwrd-slash-star) check */ + else if(token[0] == '/' && token.length >= 2 && token[1]=='*') + { + return SymbolType.MULTI_LINE_COMMENT; + } + /* Single-line comment (fwrd-slash-slash) check */ + else if(token[0] == '/' && token.length >= 2 && token[1]=='/') + { + return SymbolType.SINGLE_LINE_COMMENT; + } /* Divide `/` operator check */ else if(token[0] == '/') { diff --git a/source/tlang/compiler/typecheck/core.d b/source/tlang/compiler/typecheck/core.d index 4b22e00d..46a42c17 100644 --- a/source/tlang/compiler/typecheck/core.d +++ b/source/tlang/compiler/typecheck/core.d @@ -16,6 +16,8 @@ import std.container.slist; import std.algorithm : reverse; import tlang.compiler.typecheck.meta; import tlang.compiler.configuration; +import tlang.compiler.typecheck.dependency.store.interfaces : IFuncDefStore; +import tlang.compiler.typecheck.dependency.store.impls : FuncDefStore; /** * The Parser only makes sure syntax @@ -107,7 +109,6 @@ public final class TypeChecker /* TODO: Implement me */ checkClassInherit(modulle); - /** * Dependency tree generation * @@ -118,8 +119,15 @@ public final class TypeChecker * */ + // Create a pooling mechanism + import tlang.compiler.typecheck.dependency.pool.interfaces; + import tlang.compiler.typecheck.dependency.pool.impls; + - DNodeGenerator dNodeGenerator = new DNodeGenerator(this); + /* Create the dependency generator */ + IPoolManager poolManager = new PoolManager(); + IFuncDefStore funcDefStore = new FuncDefStore(this, poolManager); + DNodeGenerator dNodeGenerator = new DNodeGenerator(this, poolManager, funcDefStore); /* Generate the dependency tree */ DNode rootNode = dNodeGenerator.generate(); /* TODO: This should make it acyclic */ @@ -150,7 +158,7 @@ public final class TypeChecker assert(codeQueue.empty() == true); /* Grab functionData ??? */ - FunctionData[string] functionDefinitions = grabFunctionDefs(); + FunctionData[string] functionDefinitions = funcDefStore.grabFunctionDefs(); gprintln("Defined functions: "~to!(string)(functionDefinitions)); foreach(FunctionData funcData; functionDefinitions.values) @@ -201,8 +209,6 @@ public final class TypeChecker gprintln("FUNCDEF DONE: "~to!(string)(functionBodyCodeQueues[funcData.name])); } - - } diff --git a/source/tlang/compiler/typecheck/dependency/classes/classObject.d b/source/tlang/compiler/typecheck/dependency/classes/classObject.d index 2b65110e..fab413fd 100644 --- a/source/tlang/compiler/typecheck/dependency/classes/classObject.d +++ b/source/tlang/compiler/typecheck/dependency/classes/classObject.d @@ -27,9 +27,9 @@ public class ObjectInitializationNode : DNode /* Object actual type */ private Clazz clazz; - this(DNodeGenerator dnodegen, Clazz objectActualType, NewExpression entity) + this(Clazz objectActualType, NewExpression entity) { - super(dnodegen, entity); + super(entity); // this.newExpression = entity; this.clazz = objectActualType; diff --git a/source/tlang/compiler/typecheck/dependency/classes/classStaticDep.d b/source/tlang/compiler/typecheck/dependency/classes/classStaticDep.d index 5f3961a0..5566e029 100644 --- a/source/tlang/compiler/typecheck/dependency/classes/classStaticDep.d +++ b/source/tlang/compiler/typecheck/dependency/classes/classStaticDep.d @@ -13,19 +13,22 @@ import tlang.compiler.typecheck.core; import tlang.compiler.symbols.typing.core; import tlang.compiler.symbols.typing.builtins; import tlang.compiler.typecheck.dependency.core; +import std.conv : to; public class ClassStaticNode : DNode { + private Clazz entity; - this(DNodeGenerator dnodegen, Clazz entity) + this(Clazz entity) { - super(dnodegen, entity); + super(entity); + this.entity = entity; initName(); } private void initName() { - name = "ClassStaticInit: "~resolver.generateName(cast(Container)dnodegen.root.getEntity(), cast(Entity)entity); + name = "ClassStaticInit: "~to!(string)(entity); } } \ No newline at end of file diff --git a/source/tlang/compiler/typecheck/dependency/classes/classVirtualInit.d b/source/tlang/compiler/typecheck/dependency/classes/classVirtualInit.d index 75297d06..f02071ff 100644 --- a/source/tlang/compiler/typecheck/dependency/classes/classVirtualInit.d +++ b/source/tlang/compiler/typecheck/dependency/classes/classVirtualInit.d @@ -13,11 +13,23 @@ import tlang.compiler.typecheck.core; import tlang.compiler.symbols.typing.core; import tlang.compiler.symbols.typing.builtins; import tlang.compiler.typecheck.dependency.core; +import std.conv : to; + public class ClassVirtualInit : DNode { - this(DNodeGenerator dnodegen, Clazz clazz) + private Clazz clazz; + + this(Clazz clazz) + { + super(clazz); + + this.clazz = clazz; + initName(); + } + + private void initName() { - super(dnodegen, clazz); + name = "ClassVirtualInit: "~to!(string)(clazz); } } \ No newline at end of file diff --git a/source/tlang/compiler/typecheck/dependency/core.d b/source/tlang/compiler/typecheck/dependency/core.d index 564723fb..10b5d8ee 100644 --- a/source/tlang/compiler/typecheck/dependency/core.d +++ b/source/tlang/compiler/typecheck/dependency/core.d @@ -13,6 +13,9 @@ import tlang.compiler.typecheck.core; import tlang.compiler.symbols.typing.core; import tlang.compiler.symbols.typing.builtins; import tlang.compiler.typecheck.dependency.exceptions : DependencyException, DependencyError; +import tlang.compiler.typecheck.dependency.pool.interfaces; +import tlang.compiler.typecheck.dependency.pool.impls; +import tlang.compiler.typecheck.dependency.store.interfaces : IFuncDefStore; /** @@ -91,55 +94,6 @@ public struct FunctionData } } -/** -* All declared functions -*/ -private FunctionData[string] functions; - - -/** -* Returns the declared functions -*/ -public FunctionData[string] grabFunctionDefs() -{ - return functions; -} - -/** -* Creates a new FunctionData and adds it to the -* list of declared functions -* -* Requires a TypeChecker `tc` -*/ -private void addFunctionDef(TypeChecker tc, Function func) -{ - /* (Sanity Check) This should never be called again */ - foreach(string cFuncKey; functions.keys()) - { - FunctionData cFuncData = functions[cFuncKey]; - Function cFunc = cFuncData.func; - - if(cFunc == func) - { - assert(false); - } - } - - /** - * Create the FunctionData, coupled with it own DNodeGenerator - * context etc. - */ - FunctionData funcData; - funcData.ownGenerator = new DFunctionInnerGenerator(tc, func); - funcData.name = func.getName(); - funcData.func = func; - - - functions[funcData.name] = funcData; - - -} - /** * DNode * @@ -155,18 +109,13 @@ public class DNode protected string name; - protected DNodeGenerator dnodegen; - protected Resolver resolver; - private bool visited; private bool complete; private DNode[] dependencies; - this(DNodeGenerator dnodegen, Statement entity) + this(Statement entity) { this.entity = entity; - this.dnodegen = dnodegen; - this.resolver = dnodegen.resolver; initName(); } @@ -391,9 +340,9 @@ public final class DFunctionInnerGenerator : DNodeGenerator { private Function func; - this(TypeChecker tc, Function func) + this(TypeChecker tc, IPoolManager poolManager, IFuncDefStore funcDefStore, Function func) { - super(tc); + super(tc, poolManager, funcDefStore); this.func = func; } @@ -424,56 +373,29 @@ public class DNodeGenerator */ private DNode[string] functionDefinitions; - /** - * Given a DNode generated by a Function (function definition) - * this will extract the name of the function and save the DNode - * into the map for later retrieval by `retrieveFunctionDefinitionNode` - */ - private void saveFunctionDefinitionNode(DNode funcDefNode) - { - gprintln("saveFunctionDefinitionNode: Implement me please"); - - // Extract the name of the function - Function functionDefinition = cast(Function)funcDefNode.getEntity(); - assert(functionDefinition); - string functionNameAbsolutePath = resolver.generateName(cast(Container)root.getEntity(), cast(Entity)functionDefinition); - - // Save to the map - functionDefinitions[functionNameAbsolutePath] = funcDefNode; - } - - /** - * Given the absolute path to a function, this will retrieve the - * Function (function definition) DNode from the map - */ - private DNode retrieveFunctionDefinitionNode(string functionAbsolutePath) - { - gprintln("retrieveFunctionDefinitionNode: Implement me please"); + private IFuncDefStore funcDefStore; - // TODO: Add an assertion for failed lookup - return functionDefinitions[functionAbsolutePath]; - } - - - /** - * DNode pool - * - * This holds unique pool entries - */ - private static DNode[] nodePool; + /** + * Dependency node pooling + * management + */ + private IPoolManager poolManager; - this(TypeChecker tc) + this(TypeChecker tc, IPoolManager poolManager, IFuncDefStore funcDefStore) { // /* NOTE: NEW STUFF 1st Oct 2022 */ // Module modulle = tc.getModule(); // Context context = new Context(modulle, InitScope.STATIC); // super(tc, context, context.getContainer().getStatements()); - - + // TODO: See if we can pass it in rather, but + // ... because this needs a this we must make + // ... it here + this.poolManager = poolManager; this.tc = tc; this.resolver = tc.getResolver(); + this.funcDefStore = funcDefStore; /* TODO: Make this call in the TypeChecker instance */ //generate(); @@ -512,23 +434,7 @@ public class DNodeGenerator private DNode pool(Statement entity) { - foreach(DNode dnode; nodePool) - { - if(dnode.getEntity() == entity) - { - return dnode; - } - } - - /** - * If no DNode is found that is associated with - * the provided Entity then create a new one and - * pool it - */ - DNode newDNode = new DNode(this, entity); - nodePool ~= newDNode; - - return newDNode; + return this.poolManager.pool(entity); } /** @@ -538,23 +444,27 @@ public class DNodeGenerator */ private DNodeType poolT(DNodeType, EntityType)(EntityType entity) { - foreach(DNode dnode; nodePool) + static if(__traits(isSame, DNodeType, ExpressionDNode)) { - if(dnode.getEntity() == entity) - { - return cast(DNodeType)dnode; - } + return this.poolManager.poolExpression(cast(Expression)entity); + } + else static if(__traits(isSame, DNodeType, VariableNode)) + { + return this.poolManager.poolVariable(cast(Variable)entity); + } + else static if(__traits(isSame, DNodeType, StaticVariableDeclaration)) + { + return this.poolManager.poolStaticVariable(cast(Variable)entity); + } + else static if(__traits(isSame, DNodeType, FuncDecNode)) + { + return this.poolManager.poolFuncDec(cast(Function)entity); + } + else + { + pragma(msg, "This is an invalid case"); + static assert(false); } - - /** - * If no DNode is found that is associated with - * the provided Entity then create a new one and - * pool it - */ - DNodeType newDNode = new DNodeType(this, entity); - nodePool ~= newDNode; - - return newDNode; } @@ -584,7 +494,7 @@ public class DNodeGenerator { /* We don't pool anything here - a constructor call is unique */ - ObjectInitializationNode node = new ObjectInitializationNode(this, clazz, newExpression); + ObjectInitializationNode node = new ObjectInitializationNode(clazz, newExpression); /* TODO: Call a virtual pass over the class */ @@ -1016,23 +926,7 @@ public class DNodeGenerator import tlang.compiler.typecheck.dependency.variables; private ModuleVariableDeclaration pool_module_vardec(Variable entity) { - foreach(DNode dnode; nodePool) - { - if(dnode.getEntity() == entity) - { - return cast(ModuleVariableDeclaration)dnode; - } - } - - /** - * If no DNode is found that is associated with - * the provided Entity then create a new one and - * pool it - */ - ModuleVariableDeclaration newDNode = new ModuleVariableDeclaration(this, entity); - nodePool ~= newDNode; - - return newDNode; + return this.poolManager.poolModuleVariableDeclaration(entity); } // TODO: Work in progress @@ -1263,7 +1157,7 @@ public class DNodeGenerator /* Add funtion definition */ gprintln("Hello"); - addFunctionDef(tc, func); + this.funcDefStore.addFunctionDef(func); return null; } @@ -1610,25 +1504,7 @@ public class DNodeGenerator // assert(clazz.getModifierType() == InitScope.STATIC); } - - foreach(DNode dnode; nodePool) - { - Statement entity = dnode.getEntity(); - if(entity == clazz && cast(ClassStaticNode)dnode) - { - return cast(ClassStaticNode)dnode; - } - } - - /** - * If no DNode is found that is associated with - * the provided Entity then create a new one and - * pool it - */ - ClassStaticNode newDNode = new ClassStaticNode(this, clazz); - nodePool ~= newDNode; - - return newDNode; + return this.poolManager.poolClassStatic(clazz); } /** diff --git a/source/tlang/compiler/typecheck/dependency/expression.d b/source/tlang/compiler/typecheck/dependency/expression.d index addc095a..61fc1758 100644 --- a/source/tlang/compiler/typecheck/dependency/expression.d +++ b/source/tlang/compiler/typecheck/dependency/expression.d @@ -13,12 +13,13 @@ import tlang.compiler.typecheck.core; import tlang.compiler.symbols.typing.core; import tlang.compiler.symbols.typing.builtins; import tlang.compiler.typecheck.dependency.core; +import std.conv : to; public class ExpressionDNode : DNode { - this(DNodeGenerator dnodegen, Expression entity) + this(Expression entity) { - super(dnodegen, entity); + super(entity); initName(); } @@ -52,14 +53,16 @@ public class ExpressionDNode : DNode */ public class AccessDNode : DNode { + private Entity entity; + /** * Construct a new AccessNode given the `entity` * being accessed */ - this(DNodeGenerator dnodegen, Entity entity) + this(Entity entity) { - super(dnodegen, entity); - // this.entity = entity; + super(entity); + this.entity = entity; initName(); @@ -67,8 +70,6 @@ public class AccessDNode : DNode private void initName() { - name = resolver.generateName(cast(Container)dnodegen.root.getEntity(), cast(Entity)entity); - name = "[AccessNode] (Name: "~name~")"; - + name = "[AccessNode] (Name: "~to!(string)(entity)~")"; } } \ No newline at end of file diff --git a/source/tlang/compiler/typecheck/dependency/pool/impls.d b/source/tlang/compiler/typecheck/dependency/pool/impls.d new file mode 100644 index 00000000..b3fc55d0 --- /dev/null +++ b/source/tlang/compiler/typecheck/dependency/pool/impls.d @@ -0,0 +1,216 @@ +/** + * Implementation of the `IPoolManager` + * interface + */ +module tlang.compiler.typecheck.dependency.pool.impls; + +import tlang.compiler.typecheck.dependency.pool.interfaces; +import tlang.compiler.typecheck.dependency.core : DNode, DNodeGenerator; +import tlang.compiler.typecheck.dependency.expression : ExpressionDNode; +import tlang.compiler.typecheck.dependency.variables : VariableNode, FuncDecNode, StaticVariableDeclaration, ModuleVariableDeclaration; +import tlang.compiler.typecheck.dependency.classes.classStaticDep : ClassStaticNode; +import tlang.compiler.symbols.data : Statement, Expression, Variable, Function, Clazz; +import std.traits : isAssignable; + +/** + * Provides an implementation of + * the `IPoolManager` interface + * such that you can use this + * as part of the dependency + * generation process + */ +public final class PoolManager : IPoolManager +{ + /** + * The pool itself + */ + private DNode[] nodePool; + + /** + * Constructs a new pooling manager + */ + this() + { + } + + /** + * Pools the provided AST node + * to a dependency node, creating + * one if one did not yet exist + * + * Params: + * statement = the AST node + * Returns: the dependency node + */ + public DNode pool(Statement statement) + { + return poolT!(DNode, Statement)(statement); + } + + /** + * Pools the provided `Clazz` + * AST node but with an additional + * check that it should match + * against a `ClassStaticNode` + * and if one does not exist + * then one such dependency + * node should be created + * + * Params: + * clazz = the class to statcally + * pool + * Returns: the pooled `ClassStaticNode` + */ + public ClassStaticNode poolClassStatic(Clazz clazz) + { + foreach(DNode dnode; nodePool) + { + Statement entity = dnode.getEntity(); + if(entity == clazz && cast(ClassStaticNode)dnode) + { + return cast(ClassStaticNode)dnode; + } + } + + /** + * If no DNode is found that is associated with + * the provided Entity then create a new one and + * pool it + */ + ClassStaticNode newDNode = new ClassStaticNode(clazz); + nodePool ~= newDNode; + + return newDNode; + } + + /** + * Pools the provided `Expression` + * AST node into an `ExpressionDNode` + * + * Params: + * expression = the AST node + * Returns: the pooled `ExpressionDNode` + */ + public ExpressionDNode poolExpression(Expression expression) + { + return poolT!(ExpressionDNode, Expression)(expression); + } + + /** + * Pools the provided `Variable` + * AST node into a `VariableNode` + * + * Params: + * variable = the AST node + * Returns: the pooled `VariableNode` + */ + public VariableNode poolVariable(Variable variable) + { + return poolT!(VariableNode, Variable)(variable); + } + + /** + * Pools the provided `Variable` + * AST node into a `StaticVariableDeclaration` + * + * Params: + * variable = the AST node + * Returns: the pooled `StaticVariableDeclaration` + */ + public StaticVariableDeclaration poolStaticVariable(Variable variable) + { + return poolT!(StaticVariableDeclaration, Variable)(variable); + } + + /** + * Pools the provided `Variable` + * AST node into a `ModuleVariableDeclaration` + * + * Params: + * variable = the AST node + * Returns: the pooled `ModuleVariableDeclaration` + */ + public ModuleVariableDeclaration poolModuleVariableDeclaration(Variable variable) + { + return poolT!(ModuleVariableDeclaration, Variable)(variable); + } + + /** + * Pools the provided `Function` + * AST node into a `FuncDecNode` + * + * Params: + * func = the AST node + * Returns: the pooled `FUncDecNode` + */ + public FuncDecNode poolFuncDec(Function func) + { + return poolT!(FuncDecNode, Function)(func); + } + + /** + * Pools the provided AST node + * to a dependency node, creating + * one if one did not yet exist. + * + * This is a templatised version + * which lets you specify the + * kind-of `DNode` to be constructed + * (if it does not yet exist) and + * the incoming type of AST node. + * + * Params: + * entity = the AST node + * Returns: the dependency node + */ + public DNodeType poolT(DNodeType, EntityType)(EntityType entity) + if(isAssignable!(DNode, DNodeType)) + { + foreach(DNode dnode; nodePool) + { + if(dnode.getEntity() == entity) + { + return cast(DNodeType)dnode; + } + } + + /** + * If no DNode is found that is associated with + * the provided Entity then create a new one and + * pool it + */ + DNodeType newDNode = new DNodeType(entity); + nodePool ~= newDNode; + + return newDNode; + } +} + +version(unittest) +{ + import tlang.compiler.symbols.data : Module, Variable; + import tlang.compiler.typecheck.core : TypeChecker; +} + +/** + * Tests the pooling of AST nodes + * to dependency nodes using the + * `PoolManager` implementation + */ +unittest +{ + // Create bogus module and type checker + Module testModule = new Module("myModule"); + TypeChecker tc = new TypeChecker(testModule); + + // Create a pool manager + IPoolManager pool = new PoolManager(); + + // Pool an AST node + Variable astNode = new Variable("int", "age"); + DNode astDNode = pool.pool(astNode); + + // Now pool it (again) and ensure it matches + // the dependency node just created + assert(astDNode is pool.pool(astNode)); +} \ No newline at end of file diff --git a/source/tlang/compiler/typecheck/dependency/pool/interfaces.d b/source/tlang/compiler/typecheck/dependency/pool/interfaces.d new file mode 100644 index 00000000..1df5cd87 --- /dev/null +++ b/source/tlang/compiler/typecheck/dependency/pool/interfaces.d @@ -0,0 +1,106 @@ +/** + * Defines interfaces for managing + * the pooling of AST nodes to + * dependency nodes which are to be + * used within the dependency generator + * and later the codegen/typechecker + * which consumes these dependency + * nodes + */ +module tlang.compiler.typecheck.dependency.pool.interfaces; + +import tlang.compiler.typecheck.dependency.core : DNode; +import tlang.compiler.typecheck.dependency.expression : ExpressionDNode; +import tlang.compiler.typecheck.dependency.variables : VariableNode, FuncDecNode, StaticVariableDeclaration, ModuleVariableDeclaration; +import tlang.compiler.typecheck.dependency.classes.classStaticDep : ClassStaticNode; +import tlang.compiler.symbols.data : Statement, Expression, Variable, Function, Clazz; + +// TODO: In future if we do not require the specific `ExpressionDNode` et al +// ... then remove them from the interface definition below + +/** + * Defines an interface by which + * `Statement`s (i.e. AST nodes) + * can be mapped to a `DNode` + * and if one does not exist + * then it is created on the + * first use + */ +public interface IPoolManager +{ + /** + * Pools the provided AST node + * to a dependency node + * + * Params: + * statement = the AST node + * Returns: the pooled `DNode` + */ + public DNode pool(Statement statement); + + /** + * Pools the provided `Clazz` + * AST node but with an additional + * check that it should match + * against a `ClassStaticNode` + * and if one does not exist + * then one such dependency + * node should be created + * + * Params: + * clazz = the class to statcally + * pool + * Returns: the pooled `ClassStaticNode` + */ + public ClassStaticNode poolClassStatic(Clazz clazz); + + /** + * Pools the provided `Expression` + * AST node into an `ExpressionDNode` + * + * Params: + * expression = the AST node + * Returns: the pooled `ExpressionDNode` + */ + public ExpressionDNode poolExpression(Expression expression); + + /** + * Pools the provided `Variable` + * AST node into a `VariableNode` + * + * Params: + * variable = the AST node + * Returns: the pooled `VariableNode` + */ + public VariableNode poolVariable(Variable variable); + + /** + * Pools the provided `Variable` + * AST node into a `StaticVariableDeclaration` + * + * Params: + * variable = the AST node + * Returns: the pooled `StaticVariableDeclaration` + */ + public StaticVariableDeclaration poolStaticVariable(Variable variable); + + /** + * Pools the provided `Variable` + * AST node into a `ModuleVariableDeclaration` + * + * Params: + * variable = the AST node + * Returns: the pooled `ModuleVariableDeclaration` + */ + public ModuleVariableDeclaration poolModuleVariableDeclaration(Variable variable); + + /** + * Pools the provided `Function` + * AST node into a `FuncDecNode` + * + * Params: + * func = the AST node + * Returns: the pooled `FUncDecNode` + */ + public FuncDecNode poolFuncDec(Function func); +} \ No newline at end of file diff --git a/source/tlang/compiler/typecheck/dependency/store/impls.d b/source/tlang/compiler/typecheck/dependency/store/impls.d new file mode 100644 index 00000000..b84e150e --- /dev/null +++ b/source/tlang/compiler/typecheck/dependency/store/impls.d @@ -0,0 +1,127 @@ +/** + * Provides implementation of the `IFuncDefStore` + * interface + */ +module tlang.compiler.typecheck.dependency.store.impls; + +import tlang.compiler.typecheck.dependency.store.interfaces; +import tlang.compiler.symbols.data : Function; +import tlang.compiler.typecheck.dependency.core : FunctionData, DFunctionInnerGenerator; +import tlang.compiler.typecheck.core : TypeChecker; +import tlang.compiler.typecheck.dependency.pool.interfaces : IPoolManager; + +/** + * An implementation of the `IFuncDefStore` + * which provides us with a way to store + * function definitions and retrieve them + * later + */ +public final class FuncDefStore : IFuncDefStore +{ + /** + * All declared functions + */ + private FunctionData[string] functions; + + /** + * The type checker instance + */ + private TypeChecker tc; + + /** + * The pool management + */ + private IPoolManager poolManager; + + /** + * Constructs a new function + * definition store with + * the provided type + * checking instance + * + * Params: + * typeChecker = the `TypeChecker` + * poolManager = the `IPoolManager` + */ + this(TypeChecker typeChecker, IPoolManager poolManager) + { + this.tc = typeChecker; + this.poolManager = poolManager; + } + + /** + * Adds the function definition + * to the store + * + * Params: + * func = the function to add + * Throws: + * FuncDefStoreException if the function + * has already been added + */ + public void addFunctionDef(Function func) + { + /* (Sanity Check) This should never be called again */ + foreach(string cFuncKey; functions.keys()) + { + FunctionData cFuncData = functions[cFuncKey]; + Function cFunc = cFuncData.func; + + if(cFunc == func) + { + throw new FuncDefStoreException("The provided Function already exists within the store"); + } + } + + /** + * Create the FunctionData, coupled with it own DNodeGenerator + * context etc. + */ + FunctionData funcData; + funcData.ownGenerator = new DFunctionInnerGenerator(tc, this.poolManager, this, func); + // TODO: Should we not generate a HELLA long name rather, to avoid duplication problems and overwrites of key values + + funcData.name = tc.getResolver().generateName(tc.getModule(), func); + + funcData.name = func.getName(); + funcData.func = func; + + + functions[funcData.name] = funcData; + } + + /** + * Grabs all of the function + * definitions currently stored + * + * Returns: a `FunctionData[string]` + * map + */ + public FunctionData[string] grabFunctionDefs() + { + return this.functions.dup; + } + + /** + * Grabs a function definition by its + * name + * + * Params: + * name = the name of the function + * Returns: the `FunctionData` + * Throws: + * FuncDefStoreException if the function + * could not be found + */ + public FunctionData grabFunctionDef(string name) + { + if(name in this.functions) + { + return this.functions[name]; + } + else + { + throw new FuncDefStoreException("Could not find function by name '"~name~"'"); + } + } +} \ No newline at end of file diff --git a/source/tlang/compiler/typecheck/dependency/store/interfaces.d b/source/tlang/compiler/typecheck/dependency/store/interfaces.d new file mode 100644 index 00000000..0e6ace4c --- /dev/null +++ b/source/tlang/compiler/typecheck/dependency/store/interfaces.d @@ -0,0 +1,70 @@ +/** + * Provides the definition of a function definition + * store and retrieval system + */ +module tlang.compiler.typecheck.dependency.store.interfaces; + +import tlang.compiler.symbols.data : Function; +import tlang.compiler.typecheck.dependency.core : FunctionData; +import misc.exceptions : TError; + +/** + * Represents a storage mechanism + * which can store and retrieve + * function definition datas + */ +public interface IFuncDefStore +{ + /** + * Adds the function definition + * to the store + * + * Params: + * func = the function to add + * Throws: + * FuncDefStoreException if the function + * has already been added + */ + public void addFunctionDef(Function func); + + /** + * Grabs all of the function + * definitions currently stored + * + * Returns: a `FunctionData[string]` + * map + */ + public FunctionData[string] grabFunctionDefs(); + + /** + * Grabs a function definition by its + * name + * + * Params: + * name = the name of the function + * Returns: the `FunctionData` + * Throws: + * FuncDefStoreException if the function + * could not be found + */ + public FunctionData grabFunctionDef(string name); +} + +/** + * Exception thrown when an error occurs + * with the `IFuncDefStore` system + */ +public final class FuncDefStoreException : TError +{ + /** + * Constructs a new `FuncDefStoreException` + * with the given error message + * + * Params: + * msg = the error message + */ + this(string msg) + { + super(msg); + } +} \ No newline at end of file diff --git a/source/tlang/compiler/typecheck/dependency/variables.d b/source/tlang/compiler/typecheck/dependency/variables.d index 61b21498..97b2b98a 100644 --- a/source/tlang/compiler/typecheck/dependency/variables.d +++ b/source/tlang/compiler/typecheck/dependency/variables.d @@ -2,6 +2,7 @@ module tlang.compiler.typecheck.dependency.variables; import tlang.compiler.typecheck.dependency.core; import tlang.compiler.symbols.data; +import std.conv : to; /** * This module holds types related to variable declarations @@ -15,9 +16,9 @@ public class VariableNode : DNode { private Variable variable; - this(DNodeGenerator dnodegen, Variable variable) + this(Variable variable) { - super(dnodegen, variable); + super(variable); this.variable = variable; @@ -26,60 +27,55 @@ public class VariableNode : DNode private void initName() { - name = resolver.generateName(cast(Container)dnodegen.root.getEntity(), cast(Entity)entity); + name = to!(string)(variable); } - - } public class FuncDecNode : DNode { private Function funcHandle; - this(DNodeGenerator dnodegen, Function funcHandle) + this(Function funcHandle) { - super(dnodegen, funcHandle); + super(funcHandle); this.funcHandle = funcHandle; - initName(); } private void initName() { - name = "FuncHandle:"~resolver.generateName(cast(Container)dnodegen.root.getEntity(), cast(Entity)entity); + name = "FuncHandle:"~to!(string)(funcHandle); } - - } public class ModuleVariableDeclaration : VariableNode { - this(DNodeGenerator dnodegen, Variable variable) + this(Variable variable) { - super(dnodegen, variable); + super(variable); initName(); } private void initName() { - name = "(S) "~resolver.generateName(cast(Container)dnodegen.root.getEntity(), cast(Entity)entity); + name = "(S) "~to!(string)(variable); } } public class StaticVariableDeclaration : VariableNode { - this(DNodeGenerator dnodegen, Variable variable) + this(Variable variable) { - super(dnodegen, variable); + super(variable); initName(); } private void initName() { - name = "(S) "~resolver.generateName(cast(Container)dnodegen.root.getEntity(), cast(Entity)entity); + name = "(S) "~to!(string)(variable); } } @@ -87,12 +83,11 @@ public class VariableAssignmentNode : DNode { private VariableAssignment variableAssignment; - this(DNodeGenerator dnodegen, VariableAssignment variableAssignment) + this(VariableAssignment variableAssignment) { - super(dnodegen, variableAssignment); + super(variableAssignment); this.variableAssignment = variableAssignment; - initName(); } @@ -101,6 +96,6 @@ public class VariableAssignmentNode : DNode /* get the associated variable */ Variable associatedVariable = variableAssignment.getVariable(); - name = resolver.generateName(cast(Container)dnodegen.root.getEntity(), associatedVariable)~" (assignment)"; + name = to!(string)(associatedVariable)~" (assignment)"; } } \ No newline at end of file diff --git a/tlang b/tlang index 1f782fc1..d1948c51 100755 Binary files a/tlang and b/tlang differ