1 files changed, 71 insertions, 0 deletions
diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go
new file mode 100644
index 0000000..5eadc5e
--- /dev/null
+++ b/users/fcuny/exp/monkey/pkg/token/token.go
@@ -0,0 +1,71 @@
+// Package token provides a tokenizer for the monkey language.
+package token
+
+// TokenType represents the type of the token
+type TokenType string
+
+// Token represents a token, with the type and the literal value of the token
+type Token struct {
+	Type    TokenType
+	Literal string
+}
+
+const (
+	ILLEGAL = "ILLEGAL"
+	EOF     = "EOF"
+
+	IDENT = "IDENT"
+	INT   = "INT"
+
+	COMMA     = ","
+	SEMICOLON = ";"
+
+	LPAREN = "("
+	RPAREN = ")"
+	LBRACE = "{"
+	RBRACE = "}"
+
+	// The following tokens are keywords
+	FUNCTION = "FUNCTION"
+	LET      = "LET"
+	TRUE     = "TRUE"
+	FALSE    = "FALSE"
+	IF       = "IF"
+	ELSE     = "ELSE"
+	RETURN   = "RETURN"
+
+	// The following tokens are for operators
+	ASSIGN   = "="
+	PLUS     = "+"
+	MINUS    = "-"
+	BANG     = "!"
+	ASTERISK = "*"
+	SLASH    = "/"
+	LT       = "<"
+	GT       = ">"
+
+	EQ     = "=="
+	NOT_EQ = "!="
+)
+
+// List of our keywords for the language
+var keywords = map[string]TokenType{
+	"fn":     FUNCTION,
+	"let":    LET,
+	"true":   TRUE,
+	"false":  FALSE,
+	"if":     IF,
+	"else":   ELSE,
+	"return": RETURN,
+}
+
+// LookupIdent returns the token type for a given identifier.
+// First we check if the identifier is a keyword. If it is, we return they
+// keyword TokenType constant. If it isn't, we return the token.IDENT which is
+// the TokenType for all user-defined identifiers.
+func LookupIdent(ident string) TokenType {
+	if tok, ok := keywords[ident]; ok {
+		return tok
+	}
+	return IDENT
+}