@@ -3,29 +3,28 @@ module CodeRay
33 # The Tokens class represents a list of tokens returned from
44 # a Scanner. It's actually just an Array with a few helper methods.
55 #
6- # A token itself is not a special object, just a two-element Array
7- # consisting of
6+ # A token itself is not a special object, just two elements in an Array:
87 # * the _token_ _text_ (the original source of the token in a String) or
98 # a _token_ _action_ (begin_group, end_group, begin_line, end_line)
109 # * the _token_ _kind_ (a Symbol representing the type of the token)
1110 #
1211 # It looks like this:
1312 #
14- # [ '# It looks like this', :comment]
15- # [ '3.1415926', :float]
16- # [ '$^', :error]
13+ # ..., '# It looks like this', :comment, ...
14+ # ..., '3.1415926', :float, ...
15+ # ..., '$^', :error, ...
1716 #
1817 # Some scanners also yield sub-tokens, represented by special
1918 # token actions, for example :begin_group and :end_group.
2019 #
2120 # The Ruby scanner, for example, splits "a string" into:
2221 #
2322 # [
24- # [ :begin_group, :string] ,
25- # [ '"', :delimiter] ,
26- # [ 'a string', :content] ,
27- # [ '"', :delimiter] ,
28- # [ :end_group, :string]
23+ # :begin_group, :string,
24+ # '"', :delimiter,
25+ # 'a string', :content,
26+ # '"', :delimiter,
27+ # :end_group, :string
2928 # ]
3029 #
3130 # Tokens can be used to save the output of a Scanners in a simple
0 commit comments