API tools faq
paste
Advertisement
Frumkin

Untitled

Frumkin
Aug 16th, 2021
293
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
C++ 4.73 KB | None | 0 0
raw download clone embed print report
  1. #pragma once
  2.  
  3. #include <iostream>
  4. #include <string>
  5. #include <vector>
  6. #include <cctype>
  7.  
  8. #include "Token.hpp"
  9.  
  10. class Tokenizer {
  11.     private:
  12.         void throwSyntaxError(const std::string& message);
  13.         void skipWhitespace(std::size_t& selection);
  14.         std::string readWord(const std::size_t selection);
  15.         bool isKeyword(const std::string& identifier);
  16.         bool isValid(const char symbol);
  17.         bool skipComment(std::size_t& selection);
  18.         std::string readCharacterLiteral(std::size_t& selection);
  19.         std::string readStringLiteral(std::size_t& selection);
  20.         void readNumberLiteral(std::size_t& selection, std::vector<Token>& tokens, bool& foundToken);
  21.         std::string readIdentifier(std::size_t& selection);
  22.     public:
  23.         std::string Code;
  24.  
  25.         void Tokenize(std::vector<Token>& tokens);
  26.         std::string Stringify(const TokenType type);
  27.         void Log(const std::vector<Token>& tokens);
  28.  
  29.         Tokenizer() = default;
  30.  
  31.         static const std::string ENTRY;
  32.         static const std::string IMPORT;
  33.         static const std::string MODULE;
  34.         static const std::string UNIFORM;
  35.         static const std::string FUNCTION;
  36.         static const std::string RETURNS;
  37.         static const std::string RETURN;
  38.         static const std::string CONTINUE;
  39.         static const std::string BREAK;
  40.         static const std::string CONSTRUCT;
  41.         static const std::string DESTRUCT;
  42.         static const std::string INHERITS;
  43.         static const std::string SHARED;
  44.         static const std::string PUBLIC;
  45.         static const std::string PROTECTED;
  46.         static const std::string READONLY;
  47.         static const std::string GETTER;
  48.         static const std::string MUTABLE;
  49.         static const std::string VAR;
  50.         static const std::string TRY;
  51.         static const std::string CATCH;
  52.         static const std::string THROW;
  53.         static const std::string IF;
  54.         static const std::string ELSE;
  55.         static const std::string WHILE;
  56.         static const std::string FOR;
  57.         static const std::string IN;
  58.         static const std::string EQUALS;
  59.         static const std::string AND;
  60.         static const std::string OR;
  61.         static const std::string REF;
  62.         static const std::string VAL;
  63.         static const std::string AS;
  64.         static const std::string PUSH;
  65.         static const std::string POP;
  66.         static const std::string INCREMENT;
  67.         static const std::string DECREMENT;
  68.         static const std::string MULTIPLY;
  69.         static const std::string DIVIDE;
  70.         static const std::string BY;
  71.  
  72.         static constexpr std::size_t TOTAL_KEYWORDS = 40;
  73.  
  74.         static const std::string KEYWORDS[TOTAL_KEYWORDS];
  75.  
  76.         static constexpr char HASH = '#';
  77.         static constexpr char APOSTROPHE = '\'';
  78.         static constexpr char QUOTATIONS = '"';
  79.         static constexpr char UNDERSCORE = '_';
  80.  
  81.         static constexpr char DOT = '.';
  82.         static constexpr char COMMA = ',';
  83.         static constexpr char QUESTION_MARK = '?';
  84.         static constexpr char EXCLIMATION_MARK = '!';
  85.         static constexpr char COLON = ':';
  86.         static constexpr char SEMICOLON = ';';
  87.         static constexpr char EQUAL = '=';
  88.         static constexpr char MODULO = '%';
  89.         static constexpr char RAISE = '^';
  90.         static constexpr char PLUS = '+';
  91.         static constexpr char DASH = '-';
  92.         static constexpr char ASTERISK = '*';
  93.         static constexpr char AMPERSAND = '&';
  94.         static constexpr char DOLLAR = '$';
  95.         static constexpr char AT = '@';
  96.         static constexpr char SLASH = '/';
  97.         static constexpr char BACKSLASH = '\\';
  98.         static constexpr char VERTICAL_BAR = '|';
  99.         static constexpr char OPEN_PARENTHESIS = '(';
  100.         static constexpr char CLOSED_PARENTHESIS = ')';
  101.         static constexpr char OPEN_CURLY_BRACKET = '{';
  102.         static constexpr char CLOSED_CURLY_BRACKET = '}';
  103.         static constexpr char OPEN_BRACKET = '[';
  104.         static constexpr char CLOSED_BRACKET = ']';
  105.         static constexpr char OPEN_ANGLE_BRACKET = '<';
  106.         static constexpr char CLOSED_ANGLE_BRACKET = '>';
  107.  
  108.         static constexpr std::size_t TOTAL_SYMBOLS = 30 - 4; // excluding HASH, APOSTROPHE, QUOTATIONS, UNDERSCORE
  109.  
  110.         // symbols that exclude HASH, APOSTROPHE, QUOTATIONS, UNDERSCORE because:
  111.         // HASH is used for comments                        # this is a comment #
  112.         // APOSTROPHE is used for character literals        'this is a character'    
  113.         // QUOTATIONS is used for string literals           "this is a string"
  114.         // UNDERSCORE can be used in an identifier name     _variable_name
  115.         static const char SYMBOLS[TOTAL_SYMBOLS];
  116. };
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!