package net.sourceforge.pmd.cpd;

import net.sourceforge.pmd.cpd.rtfast.RTFParserTokenManager;
import net.sourceforge.pmd.cpd.rtfast.SimpleCharStream;
import net.sourceforge.pmd.cpd.rtfast.Token;

import java.io.StringReader;

public class RTFTokenizer implements Tokenizer {

    public void tokenize(SourceCode tokens, Tokens tokenEntries) {
        StringBuffer sb = tokens.getCodeBuffer();
        RTFParserTokenManager tokenMgr = new RTFParserTokenManager(new SimpleCharStream(new StringReader(sb.toString())));
        Token currToken = tokenMgr.getNextToken();
        while (currToken.image.length() > 0) {
            String image = currToken.image;
            tokenEntries.add(new TokenEntry(image, tokens.getFileName(), currToken.beginLine));
            currToken = tokenMgr.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
    }
}