Changeset 202 for branches/interpreter2/UTokenizer.pas

Timestamp:

Apr 17, 2020, 12:09:15 AM (5 years ago)

Author:

chronos

Message:

• Added: Support for String, Integer and Boolean types.
• Added: Support for more system defined functions..
• Added: Support for line comments.
• Added: For-To-Do construction.
• Added: Defined function parameters and and parsing function calls with parameters.
• Modified: Improved handling of expressions.

File:

• branches/interpreter2/UTokenizer.pas (modified) (7 diffs)

branches/interpreter2/UTokenizer.pas

@@ -30 +30 @@
 
   TTokenizerState = (tsNone, tsIdentifier, tsString, tsStringEnd, tsNumber,
-    tsSpecialSymbol);
+    tsSpecialSymbol, tsLineComment);
 
   { TTokenizer }
@@ -48 +48 @@
     function IsSpecialSymbol2(Text: string): Boolean;
     function IsIdentifier(Text: string): Boolean;
+    function IsOperator(Text: string): Boolean;
     function IsKeyword(Text: string): Boolean;
     procedure Init;
     function GetNext: TToken;
-    function CheckNext(Text: string; Kind: TTokenKind = tkUnknown): Boolean;
+    function CheckNext(Text: string; Kind: TTokenKind): Boolean;
     function CheckNextKind(Kind: TTokenKind): Boolean;
-    procedure Expect(Text: string; Kind: TTokenKind = tkUnknown);
+    procedure Expect(Text: string; Kind: TTokenKind);
     procedure Error(Text: string);
     property OnError: TErrorEvent read FOnError write FOnError;
@@ -109 +110 @@
 begin
   Result := (C = ';') or (C = '.') or (C = '(') or (C = ')') or (C = '=') or
-    (C = ':');
+    (C = ':') or (C = '+') or (C = '-') or (C = ',') or (C = '/') or
+    (C = '<') or (C = '>');
 end;
 
 function TTokenizer.IsSpecialSymbol2(Text: string): Boolean;
 begin
-  Result := (Text = ':=');
+  Result := (Text = ':=') or (Text = '//') or (Text = '<>');
 end;
 
@@ -137 +139 @@
 end;
 
+function TTokenizer.IsOperator(Text: string): Boolean;
+begin
+  Result := (Text = '+') or (Text = '-') or (Text = '=') or (Text = '<>');
+end;
+
 function TTokenizer.IsKeyword(Text: string): Boolean;
 begin
   Result := (Text = 'begin') or (Text = 'end') or (Text = 'program') or
   (Text = 'var') or (Text = 'const') or (Text = 'if') or (Text = 'then') or
-  (Text = 'else') or (Text = 'while') or (Text = 'do');
+  (Text = 'else') or (Text = 'while') or (Text = 'do') or (Text = 'for') or
+  (Text = 'to');
 end;
 
@@ -229 +237 @@
       end;
     end else
+    if State = tsLineComment then begin
+      if C = #10 then begin
+        State := tsNone;
+      end else Pos.Increment;
+    end else
     if State = tsSpecialSymbol then begin
       if IsSpecialSymbol2(Result.Text + C) then begin
         Result.Text := Result.Text + C;
         Pos.Increment;
-        Break;
+        if Result.Text = '//' then begin
+          Result.Text := '';
+          State := tsLineComment;
+        end else Break;
       end else begin
         Break;
@@ -242 +258 @@
 end;
 
-function TTokenizer.CheckNext(Text: string; Kind: TTokenKind = tkUnknown): Boolean;
+function TTokenizer.CheckNext(Text: string; Kind: TTokenKind): Boolean;
 var
   LastPos: TTokenizerPos;
@@ -264 +280 @@
 end;
 
-procedure TTokenizer.Expect(Text: string; Kind: TTokenKind = tkUnknown);
+procedure TTokenizer.Expect(Text: string; Kind: TTokenKind);
 var
   Token: TToken;