33 lines
845 B
Scheme
33 lines
845 B
Scheme
(import (rnrs))
|
|
|
|
(define get-word
|
|
(lambda (file word)
|
|
(let ([c (lookahead-char file)])
|
|
(if (char-alphabetic? c)
|
|
(begin
|
|
(get-char file)
|
|
(get-word file (string-append word (string c))))
|
|
word))))
|
|
|
|
(define get-words
|
|
(lambda (file)
|
|
(letrec ([loop
|
|
(lambda (file words)
|
|
(let ([c (lookahead-char file)])
|
|
(cond
|
|
[(eof-object? c) words]
|
|
[(char-whitespace? c)
|
|
(get-char file)
|
|
(loop file words)]
|
|
[(char-alphabetic? c)
|
|
(loop file (cons (get-word file "") words))]
|
|
[else (error 'get-words "wtf" c)])))])
|
|
(reverse (loop file '())))))
|
|
|
|
(define tokenize
|
|
(lambda (filename)
|
|
(let ([file (open-input-file filename)])
|
|
(get-words file))))
|
|
|
|
(display (tokenize "tokens.txt"))
|