29 lines
663 B
Scheme
29 lines
663 B
Scheme
(import (rnrs))
|
|
|
|
(define get-word
|
|
(lambda (file)
|
|
(let ([c (lookahead-char file)])
|
|
(cond
|
|
[(char-alphabetic? c)
|
|
(get-char file)
|
|
(string-append (string c) (get-word file))]
|
|
[else ""]))))
|
|
|
|
(define get-words
|
|
(lambda (file)
|
|
(let ([c (lookahead-char file)])
|
|
(cond
|
|
[(eof-object? c) '()]
|
|
[(char-whitespace? c)
|
|
(get-char file)
|
|
(get-words file)]
|
|
[(char-alphabetic? c)
|
|
(cons (get-word file) (get-words file))]))))
|
|
|
|
(define tokenize
|
|
(lambda (filename)
|
|
(let ([file (open-input-file filename)])
|
|
(get-words file))))
|
|
|
|
(display (tokenize "tokens.txt"))
|