Completed lexer

6 files changed, 91 insertions, 22 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 9bbd091..73575b4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -59,7 +59,7 @@ dependencies = [
 
 [[package]]
 name = "axisc"
-version = "0.1.0"
+version = "0.0.1"
 dependencies = [
  "clap",
 ]
diff --git a/axis-rt/Cargo.toml b/axis-rt/Cargo.toml
index 03c5358..e80997b 100644
--- a/axis-rt/Cargo.toml
+++ b/axis-rt/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "axis-rt"
-version = "0.0.1"
+version = "0.0.2"
 edition = "2021"
 description = "A runtime for the Axis programming language."
 authors = ["Raindrops", "ryze312"]
diff --git a/axisc/Cargo.toml b/axisc/Cargo.toml
index c2c11de..63583de 100644
--- a/axisc/Cargo.toml
+++ b/axisc/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "axisc"
-version = "0.0.1"
+version = "0.0.2"
 edition = "2021"
 description = "A compiler for the Axis programming language."
 authors = ["Raindrops", "ryze312"]
diff --git a/axisc/src/lexer.rs b/axisc/src/lexer.rs
index ae45497..ff13fd3 100644
--- a/axisc/src/lexer.rs
+++ b/axisc/src/lexer.rs
@@ -3,7 +3,38 @@ use crate::reader::TokenScanner;
 #[derive(Debug)]
 pub enum Token {
     Identifier(String),
+    Keyword(Keyword),
     Literal(Literal),
+    SimpleToken(SimpleToken)
+}
+
+#[derive(Debug)]
+pub enum Delimiter {
+    WhiteSpace,
+    Semicolon,
+    LineFeed,
+    Equals,
+    Invalid
+}
+
+impl Delimiter {
+    pub fn is_delimiter(c: &char) -> bool {
+        matches!(c, ' ' | ';' | '\n' | '=')
+    }
+
+    pub fn from(c: char) -> Self {
+        match c {
+            ' ' => Delimiter::WhiteSpace,
+            ';' => Delimiter::Semicolon,
+            '\n' => Delimiter::LineFeed,
+            '=' => Delimiter::Equals,
+            _ => Delimiter::Invalid
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum SimpleToken {
     Multiply,
     Divide,
     Pound,
@@ -29,6 +60,17 @@ pub enum Literal {
 }
 
 #[derive(Debug)]
+pub enum Keyword {
+    Into,
+    Boolean,
+    Array,
+    Number,
+    String,
+    As,
+    Let,
+}
+
+#[derive(Debug)]
 pub enum Array {
     StringArray(Vec<String>),
     NumberArray(Vec<f64>),
diff --git a/axisc/src/main.rs b/axisc/src/main.rs
index 9d358e8..6b49b6f 100644
--- a/axisc/src/main.rs
+++ b/axisc/src/main.rs
@@ -1,9 +1,11 @@
+use std::num::ParseFloatError;
+use std::str::FromStr;
 use cli::Cli;
 use clap::Parser;
 use reader::read_file;
 use reader::TokenScanner;
 use lexer::Token;
-use crate::lexer::Literal;
+use crate::lexer::{Keyword, Literal, SimpleToken};
 
 mod cli;
 mod reader;
@@ -21,64 +23,87 @@ fn main() {
             None => break,
             Some('#') => {
                 scanner.advance(1);
-                tokens.push(Token::Pound)
+                tokens.push(Token::SimpleToken(SimpleToken::Pound))
             },
             Some('*') => {
                 scanner.advance(1);
-                tokens.push(Token::Multiply)
+                tokens.push(Token::SimpleToken(SimpleToken::Multiply))
             },
             Some('-') => {
                 scanner.advance(1);
-                tokens.push(Token::Subtract)
+                tokens.push(Token::SimpleToken(SimpleToken::Subtract))
             },
             Some('+') => {
                 scanner.advance(1);
-                tokens.push(Token::Add)
+                tokens.push(Token::SimpleToken(SimpleToken::Add))
             },
             Some('<') => {
                 scanner.advance(1);
-                tokens.push(Token::LessThan)
+                tokens.push(Token::SimpleToken(SimpleToken::LessThan))
             },
             Some('>') => {
                 scanner.advance(1);
-                tokens.push(Token::GreaterThan)
+                tokens.push(Token::SimpleToken(SimpleToken::GreaterThan))
             },
             Some('=') => {
                 scanner.advance(1);
-                tokens.push(Token::Equals)
+                tokens.push(Token::SimpleToken(SimpleToken::Equals))
             },
             Some(';') => {
                 scanner.advance(1);
-                tokens.push(Token::Semicolon)
+                tokens.push(Token::SimpleToken(SimpleToken::Semicolon))
             },
             Some('/') => {
                 scanner.advance(1);
-                tokens.push(Token::Divide)
+                tokens.push(Token::SimpleToken(SimpleToken::Divide))
             },
             Some('%') => {
                 scanner.advance(1);
-                tokens.push(Token::Percent)
+                tokens.push(Token::SimpleToken(SimpleToken::Percent))
             },
             Some('^') => {
                 scanner.advance(1);
-                tokens.push(Token::Caret)
+                tokens.push(Token::SimpleToken(SimpleToken::Caret))
             },
             Some(':') => {
                 scanner.advance(1);
-                tokens.push(Token::Colon)
+                tokens.push(Token::SimpleToken(SimpleToken::Colon))
             },
             Some('$') => {
                 scanner.advance(1);
-                tokens.push(Token::Dollar)
+                tokens.push(Token::SimpleToken(SimpleToken::Dollar))
             },
             Some('\n') => {
                 scanner.advance(1);
-                tokens.push(Token::LineFeed)
+                tokens.push(Token::SimpleToken(SimpleToken::LineFeed))
             },
             Some('"') => {
                 tokens.push(Token::Literal(Literal::string_literal_from_scanner(&mut scanner)))
+            },
+            Some(' ' | '\n') => {
+                scanner.advance(1);
+            },
+            Some(_) => {
+                let word = scanner.advance_word();
+                let word_number: Result<f64, ParseFloatError> = f64::from_str(&word);
+
+                match word_number {
+                    Ok(n) => tokens.push(Token::Literal(Literal::Number(n))),
+                    Err(_) => match word.as_ref() {
+                        "" => (),
+                        "True" => tokens.push(Token::Literal(Literal::Boolean(true))),
+                        "False" => tokens.push(Token::Literal(Literal::Boolean(false))),
+                        "Into" => tokens.push(Token::Keyword(Keyword::Into)),
+                        "Let" => tokens.push(Token::Keyword(Keyword::Let)),
+                        "As" => tokens.push(Token::Keyword(Keyword::As)),
+                        "String" => tokens.push(Token::Keyword(Keyword::String)),
+                        "Number" => tokens.push(Token::Keyword(Keyword::Number)),
+                        "Array" => tokens.push(Token::Keyword(Keyword::Array)),
+                        "Boolean" => tokens.push(Token::Keyword(Keyword::Boolean)),
+                        _ => tokens.push(Token::Identifier(word))
+                    }
+                }
             }
-            Some(_) => tokens.push(Token::Identifier(scanner.advance_word()))
         }
     }
 
diff --git a/axisc/src/reader.rs b/axisc/src/reader.rs
index b75649a..fd600a7 100644
--- a/axisc/src/reader.rs
+++ b/axisc/src/reader.rs
@@ -4,6 +4,7 @@ use std::iter::Peekable;
 use std::path::PathBuf;
 use std::process::exit;
 use std::str::Chars;
+use crate::lexer::Delimiter;
 
 pub fn read_file(source: &PathBuf) -> String {
     let file = File::open(&source).unwrap_or_else(|e| {
@@ -45,12 +46,13 @@ impl <'a> TokenScanner<'a> {
     pub fn advance_word(&mut self) -> String {
         let mut word = String::new();
 
-        while let Some(char) = self.chars.next() {
-            if char.is_ascii_whitespace() {
+        while let Some(char) = self.chars.peek() {
+            if Delimiter::is_delimiter(char) {
                 break;
             }
 
-            word.push(char);
+            word.push(*char);
+            self.chars.next();
         }
 
         word