token: initial tokenizer.

This is the initial tokenizer for the monkey language. For now we
recognize a limited number of tokens.

We only have two keywords at this stage: `fn` and `let`. `fn` is used to
create function, while `let` is used for assigning variables.

The other tokens are mostly to parse the source code, and recognize
things like brackets, parentheses, etc.

1 files changed, 48 insertions, 0 deletions

diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go
new file mode 100644
index 0000000..fda4449
--- /dev/null
+++ b/users/fcuny/exp/monkey/pkg/token/token.go
@@ -0,0 +1,48 @@
+package token
+
+// TokenType represents the type of the token
+type TokenType string
+
+// Token represents a token, with the type and the literal value of the token
+type Token struct {
+	Type    TokenType
+	Literal string
+}
+
+const (
+	ILLEGAL = "ILLEGAL"
+	EOF     = "EOF"
+
+	IDENT = "IDENT"
+	INT   = "INT"
+
+	ASSIGN = "="
+	PLUS   = "+"
+
+	COMMA     = ","
+	SEMICOLON = ";"
+
+	LPAREN = "("
+	RPAREN = ")"
+	LBRACE = "{"
+	RBRACE = "}"
+
+	FUNCTION = "FUNCTION"
+	LET      = "LET"
+)
+
+// List of our keywords for the language
+var keywords = map[string]TokenType{
+	"fn":  FUNCTION,
+	"let": LET,
+}
+
+// LookupIdent returns the token type for a given identifier. If the identifier
+// is one of our keyword, we return the corresponding value, otherwise we return
+// the given identifier.
+func LookupIdent(ident string) TokenType {
+	if tok, ok := keywords[ident]; ok {
+		return tok
+	}
+	return IDENT
+}