scheme-fudge/fudge-chez.ss

(import (rnrs))

(define get-word
  (lambda (file)
    (let ([c (lookahead-char file)])
      (cond
        [(char-alphabetic? c)
          (get-char file)
          (string-append (string c) (get-word file))]
        [else ""]))))

(define get-words
  (lambda (file)
    (let ([c (lookahead-char file)])
      (cond
        [(eof-object? c) '()]
        [(char-whitespace? c)
          (get-char file)
          (get-words file)]
        [(char-alphabetic? c)
          (cons (get-word file) (get-words file))]))))

(define tokenize
  (lambda (filename)
    (let ([file (open-input-file filename)])
      (get-words file))))
  
(display (tokenize "tokens.txt"))