(import (rnrs))

(define get-word
  (lambda (file word)
    (let ([c (lookahead-char file)])
      (if (char-alphabetic? c)
          (begin
            (get-char file)
            (get-word file (string-append word (string c))))
          word))))

(define get-words
  (lambda (file)
    (letrec ([loop
      (lambda (file words)
        (let ([c (lookahead-char file)])
          (cond
            [(eof-object? c) words]
            [(char-whitespace? c)
              (get-char file)
              (loop file words)]
            [(char-alphabetic? c)
              (loop file (cons (get-word file "") words))]
            [else (error 'get-words "wtf" c)])))])
    (reverse (loop file '())))))

(define tokenize
  (lambda (filename)
    (let ([file (open-input-file filename)])
      (get-words file))))
  
(display (tokenize "tokens.txt"))