source: branches/Void/UTokenizer.pas

unit UTokenizer;

{$mode Delphi}{$H+}

interface

uses
  Classes, SysUtils;

type

  TTokenType = (ttIdentifier, ttWhiteSpace, ttString, ttNumber, ttBrackets,
    ttSymbol);

  TTokenParseState = (psNone, psString, psStringEnd, psNumber,
    psIdentifier, psRowComment, psSpecialSymbol, psBlockComment1,
    psBlockComment2, psBlockComment1End, psWhiteSpace);

  { TVoidParser }

  TVoidParser = class
  private
    ParseState: TTokenParseState;
  public
    TokenValue: string;
    TokenType: TTokenType;
    Position: TPoint;
    TokenStartPosition: TPoint;
    Source: TStream;
    procedure Open(ASource: TStream);
    procedure ParseNextToken;
  end;

implementation

{ TVoidParser }

procedure TVoidParser.Open(ASource: TStream);
begin
  Position := Point(0, 1);
  Source := ASource;
  ParseState := psNone;
  ParseNextToken;
end;

procedure TVoidParser.ParseNextToken;
var
  Character: Char;
begin
  repeat
    if Source.Position >= Source.Size then Break;
    Character := Char(TMemoryStream(Source).ReadByte);

    if Character = #13 then Position.X := 0
    else if Character = #10 then Inc(Position.Y)
    else Inc(Position.X);

    case ParseState of
      psNone: begin
        TokenStartPosition := Position;
        if Character in [#13, #10, ' '] then begin
          TokenType := ttWhiteSpace;
          ParseState := psWhiteSpace;
        end else
        if Character = '''' then begin
          TokenValue := '';
          ParseState := psString;
        end else
        if ((Character >= 'A') and (Character <= 'Z')) or
          ((Character >= 'a') and (Character <= 'z')) then begin
          TokenValue := Character;
          TokenType := ttIdentifier;
          ParseState := psIdentifier;
        end else
        if ((Character >= '0') and (Character <= '9')) then begin
          TokenType := ttNumber;;
          TokenValue := Character;
          ParseState := psNumber;
        end else
        if Character = '{' then begin
          ParseState := psBlockComment2;
        end else
        if Character in ['/', '(', ')', '+', '-', '*', ':', ';', ','] then begin
          TokenValue := Character;
          ParseState := psSpecialSymbol;
        end;
      end;
      psNumber: begin
        if (Character >= '0') and (Character <= '9') then
        TokenValue := TokenValue + Character else
        begin
          ParseState := psNone;
          Source.Position := Source.Position - 1;
          Dec(Position.X);
          Break;
        end;
      end;
      psRowComment: begin
        if Character in [#13, #10] then begin
          ParseState := psNone;
          Source.Position := Source.Position - 1;
          Dec(Position.X);
        end;
      end;
      psWhiteSpace: begin
        if Character > ' ' then begin
          ParseState := psNone;
          TokenValue := '';
          TokenType := ttWhiteSpace;
          Source.Position := Source.Position - 1;
          Dec(Position.X);
          Break;
        end;
      end;
      psSpecialSymbol: begin
        if (TokenValue = '/') and (Character = '/') then begin
          ParseState := psRowComment;
        end else
        if (TokenValue = '(') and (Character = '*') then begin
          ParseState := psBlockComment1;
        end else
        if ((TokenValue = '>') and (Character = '=')) or
          ((TokenValue = '<') and (Character = '=')) or
          ((TokenValue = '<') and (Character = '>')) or
          ((TokenValue = ':') and (Character = '=')) then begin
          TokenValue := TokenValue + Character;
          TokenType := ttSymbol;
          ParseState := psNone;
          Break;
        end else begin
          TokenType := ttSymbol;
          ParseState := psNone;
          Source.Position := Source.Position - 1;
          Dec(Position.X);
          Break;
        end;
      end;
      psBlockComment1: begin
        if Character = '*' then begin
          ParseState := psBlockComment1End;
        end;
      end;
      psBlockComment1End: begin
        if Character = ')' then begin
          ParseState := psNone;
        end else begin
          ParseState := psBlockComment1;
          Source.Position := Source.Position - 1;
          Dec(Position.X);
        end;
      end;
      psBlockComment2: begin
        if Character = '}' then
          ParseState := psNone;
      end;
      psString: begin
        if Character = '''' then begin
          ParseState := psStringEnd;
        end else TokenValue := TokenValue + Character;
      end;
      psStringEnd: begin
        if Character = '''' then begin
          TokenValue := TokenValue + Character;
          ParseState := psString;
        end else begin
          TokenType := ttString;
          ParseState := psNone;
          Source.Position := Source.Position - 1;
          Dec(Position.X);
          Break;
          end;
      end;
      psIdentifier: begin
        if ((Character >= 'A') and (Character <= 'Z')) or
          ((Character >= 'a') and (Character <= 'z')) or
          ((Character >= '0') and (Character <= '9')) then begin
          TokenValue := TokenValue + Character;
        end else begin
          TokenType := ttIdentifier;
          ParseState := psNone;
          Source.Position := Source.Position - 1;
          Dec(Position.X);
          Break;
        end;
      end;
    end;
  until False;
end;

end.