Author: Alejandro Castro

A general way for get tokens, parse strings, count words and search for a specific 
token on a string.

Answer:

There are many functions and ways for get a token on a string. I have written a 
general class for handle tokens and parse strings. 

The class is named as TToken. I want to describe it with some examples. 

1   var
2     xTk: TToken;
3     i: Integer;
4     s: string;
5     b: Boolean;
6   begin
7     xTk := TToken.Create;
8   
9     {//////////
10    the class has some variables:
11    TEXT contains the string to handle
12    SEPS contains the set of characters that separate tokens
13    }
14  
15    xTk.Text := 'Here is my example. Contains commas, dots and spaces.';
16    xTk.Seps := [' ', ',', '.'];
17  
18    {//////////
19    with the method COUNT I can count the number of tokens.
20    I can use it on two ways, I can call the method and the variable NUMTOKENS save 
21  the number of tokens or I can assign the method to a memory variable. Here is the 
22  example of the two ways.
23    }
24  
25    i := xTk.Count;
26    ShowMessage(IntToStr(i));
27    ShowMessage(IntToStr(xTk.NumTokens));
28  
29    {//////////
30    When I want to search all tokens on a sequential way Im going to use the methods 
31  FIRT and NEXT. Im going to use two Variables: MORETOKENS and LASTTOKEN. MORETOKENS 
32  is a boolean variabale that indicates that after I execute the First or Next method 
33  I have a token that is saved on the LASTTOKEN variable
34    }
35  
36    xTk.First;
37    while xTk.MoreTokens do
38    begin
39      ShowMessage(xTk.LastToken);
40      xTk.Next;
41    end;
42  
43    {//////////
44    I can assign the Firt and Next method to a memory variable and I can use the 
45  variable NOTOKEN that contains the negative value of MORETOKENS
46    }
47  
48    s := xTk.First;
49    while not xTk.NoToken do
50    begin
51      ShowMessage(s);
52      s := xTk.Next;
53    end;
54  
55    {//////////
56    I can search for a specific token with the SEARCH method
57    }
58  
59    b := xTk.Search('IS');
60    if b then
61      ShowMessage('Found it')
62    else
63      ShowMessage('Not found it');
64  
65    b := xTk.Search('YOUR');
66    if b then
67      ShowMessage('Found it')
68    else
69      ShowMessage('Not found it');
70  
71    xTk.Free;
72  
73  end;


The class is: 

74  unit UToken;
75  
76  {
77  Class for handle Tokens
78  Author: Alejandro Castro
79  }
80  
81  interface
82  
83  uses SysUtils;
84  
85  type
86    TToken = class(Tobject)
87    private
88      xCharText: string;
89      function xGetToken(xText: string): string;
90    public
91      Seps: set of char; // Separators
92      Text: string; // string to handle
93      LastToken: string; // token readed with first or next method
94      NoToken: Boolean; // flag that indicate that there ARENT more tokens
95      MoreTokens: Boolean; // flag that indicate that there ARE more tokens
96      NumTokens: Integer; // no of tokens on Text
97  
98      constructor Create;
99      function Count: Integer; // count the number of tokens
100     function First: string; // Find the First Token
101     function Next: string; // Find the Next Token
102     function Search(TokSearch: string): Boolean; // Search for a specific token
103 
104   end;
105 
106 implementation
107 
108 constructor TToken.Create;
109 begin
110   Seps := [];
111   Text := '';
112   xCharText := '';
113   NoToken := True;
114   MoreTokens := False;
115   LastToken := '';
116 end;
117 
118 function TToken.Count: Integer;
119 var
120   i, xLen: Integer;
121   xFlag: Boolean;
122 begin
123   NumTokens := 0;
124   xLen := length(Text);
125   i := 1;
126   xFlag := False;
127   while (i <= xLen) and (xLen <> 0) do
128   begin
129     if (Text[i] in Seps) then
130       xFlag := False
131     else
132     begin
133       if (not xFlag) then
134       begin
135         xFlag := True;
136         inc(NumTokens);
137       end;
138     end;
139     inc(i);
140   end;
141   Result := NumTokens;
142 end;
143 
144 function TToken.Next: string;
145 begin
146   Result := xGetToken(xCharText);
147   LastToken := Result;
148   if Result = '' then
149     NoToken := True
150   else
151     NoToken := False;
152   MoreTokens := not NoToken;
153 end;
154 
155 function TToken.First: string;
156 begin
157 
158   Result := xGetToken(Text);
159   LastToken := Result;
160   if Result = '' then
161     NoToken := True
162   else
163     NoToken := False;
164   MoreTokens := not NoToken;
165 
166 end;
167 
168 function TToken.xGetToken(xText: string): string;
169 var
170   i, xLen: Integer;
171   xFlag, xAgain: Boolean;
172 begin
173   Result := '';
174   xLen := length(xText);
175   i := 1;
176   xFlag := False;
177   xAgain := True;
178   while (i <= xLen) and (xLen <> 0) and (xAgain) do
179   begin
180     if (xText[i] in Seps) then
181     begin
182       xAgain := (xAgain and (not xFlag));
183       xFlag := False
184     end
185     else
186     begin
187       if (not xFlag) then
188       begin
189         xFlag := True;
190         xAgain := true;
191       end;
192       Result := Result + xText[i];
193     end;
194     inc(i);
195   end;
196   xCharText := copy(xText, i, xLen);
197 end;
198 
199 function TToken.Search(TokSearch: string): Boolean;
200 var
201   xAgain: Boolean;
202 begin
203   Result := False;
204   xAgain := True;
205   First;
206   while (not noToken) and (xAgain) do
207   begin
208     if UpperCase(LastToken) = UpperCase(TokSearch) then
209     begin
210       Result := true;
211       xAgain := False;
212     end;
213     Next;
214   end;
215 end;
216 end.



Component Download: http://www.baltsoft.com/files/dkb/attachment/UToken.zip