Tokenizer beginnings

fudge-chez.ss

@@ -1 +1,32 @@
-(print "Hello, world!")
+(import (rnrs))
+
+(define get-word
+  (lambda (file word)
+    (let ([c (lookahead-char file)])
+      (if (char-alphabetic? c)
+          (begin
+            (get-char file)
+            (get-word file (string-append word (string c))))
+          word))))
+
+(define get-words
+  (lambda (file)
+    (letrec ([loop
+      (lambda (file words)
+        (let ([c (lookahead-char file)])
+          (cond
+            [(eof-object? c) words]
+            [(char-whitespace? c)
+              (get-char file)
+              (loop file words)]
+            [(char-alphabetic? c)
+              (loop file (cons (get-word file "") words))]
+            [else (error 'get-words "wtf" c)])))])
+    (reverse (loop file '())))))
+
+(define tokenize
+  (lambda (filename)
+    (let ([file (open-input-file filename)])
+      (get-words file))))
+  
+(display (tokenize "tokens.txt"))

sigma

@@ -0,0 +1 @@
+Hello, world!