Skip to content

Commit b608944

Browse files
runrevmarkrunrevmark
committed
[[ LCB ]] Improve tokenisation of numbers and add hex and bin literals
This patch improves the tokenisation of numbers in lc-compile and adds support for hexadecimal and binary integer literals. The compiler will now complain about any invalid suffixes on numeric literals, meaning that it is not possible to accidentally elide a numeric literal with an identifier. Hexadecimal literals are specified using the prefix "0x". Binary literals are specified using the prefix "0b". Real literals have been made more flexible and accurate. In particular, leading zeros are now checked for in all but the fractional part, and considering a real number to be of the form: Int DecPoint Frac Exp The allowed forms are: Int+DecPoint Int+Exp Int+DecPoint+Exp Int+DecPoint+Frac Int+DecPoint+Frac+Exp DecPoint+Frac DecPoint+Frac+Exp All other combinations will be flagged as a 'malformed real literal' error. Note: As the regex for integers now intersects with that for reals (e.g. 1e1 could be an integer with suffix, or a real) rules for both DOUBLE and INTEGER literals are now in DOUBLE_LITERAL.t with the DOUBLE rules first. This ensures DOUBLE_LITERAL matches in preference to INTEGER_LITERAL.
1 parent 03e4efc commit b608944

10 files changed

Lines changed: 552 additions & 27 deletions

File tree

‎docs/guides/LiveCode Builder Language Reference.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ follows:
4040

4141
- **Identifier**: [A-Za-z_][A-Za-z0-9_.]*
4242
- **Integer**: [0-9]+
43+
- **Binary Integer**: 0b[01]+
44+
- **Hexadecimal Integer**: 0x[0-9a-fA-F]+
4345
- **Real**: [0-9]+"."[0-9]+([eE][-+]?[0-9]+)?
4446
- **String**: "[^\n\r"]*"
4547
- **Separator**: Any whitespace containing at least one newline

‎docs/lcb/notes/feature-hex_bin_literals.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# LiveCode Builder Language
2+
## Numeric literal improvements
3+
4+
* Base 2 (binary) integer literals can now be specified by using a "0b" prefix.
5+
e.g.
6+
7+
0b0000
8+
0b1010
9+
10+
* Base 16 (hexadecimal) integer literals can now be specified by using a "0x" prefix.
11+
e.g.
12+
13+
0xdeadbeef
14+
0x0123fedc
15+
16+
* Parsing of numeric literals, in general, has been tightened up. In particular,
17+
the compiler will detect invalid suffixes on numeric literals meaning you cannot
18+
accidentally elide a number with an identifier.
19+
e.g.
20+
21+
1.344foo -- ERROR
22+
0xabcdefgh -- ERROR
23+
0b010432 -- ERROR

‎tests/lcb/compiler/frontend/numeric-literals.compilertest‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
%% Copyright (C) 2016 LiveCode Ltd.
2+
%%
3+
%% This file is part of LiveCode.
4+
%%
5+
%% LiveCode is free software; you can redistribute it and/or modify it under
6+
%% the terms of the GNU General Public License v3 as published by the Free
7+
%% Software Foundation.
8+
%%
9+
%% LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY
10+
%% WARRANTY; without even the implied warranty of MERCHANTABILITY or
11+
%% FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12+
%% for more details.
13+
%%
14+
%% You should have received a copy of the GNU General Public License
15+
%% along with LiveCode. If not see <http://www.gnu.org/licenses/>.
16+
17+
%TEST DecimalIntegerLiterals
18+
module compiler_test
19+
constant Valid is [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
20+
constant InvalidPrefix is %{BEFORE_PREFIX}00
21+
constant InvalidSuffix is %{BEFORE_SUFFIX}100foo
22+
handler Compute()
23+
variable tVar
24+
put 100+200 into tVar
25+
end handler
26+
end module
27+
%EXPECT PASS
28+
%ERROR "Malformed integer literal" AT BEFORE_PREFIX
29+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX
30+
%ENDTEST
31+
32+
%% Note that decimal integer literals are always unsigned
33+
%TEST DecimalIntegerLiteralRange
34+
module compiler_test
35+
constant BiggerThanU8 is %{U8}256
36+
constant SmallerThanU8 is %{U8S}255
37+
constant BiggerThanU16 is %{U16}65536
38+
constant SmallerThanU16 is %{U16S}65535
39+
constant BiggerThanU32 is %{U32}4294967296
40+
constant SmallerThanU32 is %{U32S}4294967295
41+
constant BiggerThanU64 is %{U64}9223372036854775808
42+
constant SmallerThanU64 is %{U64S}9223372036854775807
43+
end module
44+
%EXPECT PASS
45+
%ERROR "Integer literal too big" at U32
46+
%ERROR "Integer literal too big" at U64
47+
%ERROR "Integer literal too big" at U64S
48+
%ENDTEST
49+
50+
%TEST BinaryIntegerLiterals
51+
module compiler_test
52+
constant ValidLower is [ 0b0, 0b1, 0b01, 0b10, 0b010, 0b101 ]
53+
constant ValidUpper is [ 0B0, 0B1, 0B01, 0B10, 0B010, 0B101 ]
54+
constant InvalidSuffixLower is %{BEFORE_SUFFIX_LOWER}0b01foo
55+
constant InvalidSuffixUpper is %{BEFORE_SUFFIX_UPPER}0B01foo
56+
handler Compute()
57+
variable tVar
58+
put 0b100+0b001 into tVar
59+
end handler
60+
end module
61+
%EXPECT PASS
62+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX_LOWER
63+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX_UPPER
64+
%ENDTEST
65+
66+
%TEST BinaryIntegerLiteralRange
67+
module compiler_test
68+
constant BiggerThanU8 is %{U8}0b111111111
69+
constant SmallerThanU8 is %{U8S}0b011111111
70+
constant BiggerThanU16 is %{U16}0b11111111111111111
71+
constant SmallerThanU16 is %{U16S}0b01111111111111111
72+
constant BiggerThanU32 is %{U32}0b111111111111111111111111111111111
73+
constant SmallerThanU32 is %{U32S}0b011111111111111111111111111111111
74+
constant BiggerThanU64 is %{U64}0b11111111111111111111111111111111111111111111111111111111111111111
75+
constant SmallerThanU64 is %{U64S}0b01111111111111111111111111111111111111111111111111111111111111111
76+
end module
77+
%EXPECT PASS
78+
%ERROR "Integer literal too big" at U32
79+
%ERROR "Integer literal too big" at U64
80+
%ERROR "Integer literal too big" at U64S
81+
%ENDTEST
82+
83+
%TEST HexadecimalIntegerLiterals
84+
module compiler_test
85+
constant ValidLower is [ 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa,\
86+
0xb, 0xc, 0xd, 0xe, 0xf, 0x01, 0x10, 0x01, 0x0F, 0xF0 ]
87+
constant ValidLower is [ 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa,\
88+
0xB, 0xC, 0xD, 0xE, 0xF, 0x01, 0x10, 0x01, 0x0F, 0xF0 ]
89+
constant InvalidSuffixLower is %{BEFORE_SUFFIX_LOWER}0x01foo
90+
constant InvalidSuffixUpper is %{BEFORE_SUFFIX_UPPER}0X01foo
91+
handler Compute()
92+
variable tVar
93+
put 0x100f+0xf001 into tVar
94+
end handler
95+
end module
96+
%EXPECT PASS
97+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX_LOWER
98+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX_UPPER
99+
%ENDTEST
100+
101+
%TEST HexadecimalIntegerLiteralRange
102+
module compiler_test
103+
constant BiggerThanU8 is %{U8}0x1FF
104+
constant SmallerThanU8 is %{U8S}0x0FF
105+
constant BiggerThanU16 is %{U16}0x1FFFF
106+
constant SmallerThanU16 is %{U16S}0x0FFFF
107+
constant BiggerThanU32 is %{U32}0x1FFFFFFFF
108+
constant SmallerThanU16 is %{U32S}0x0FFFFFFFF
109+
constant BiggerThanU64 is %{U64}0x1FFFFFFFFFFFFFFFF
110+
constant SmallerThanU16 is %{U64S}0x0FFFFFFFFFFFFFFFF
111+
end module
112+
%EXPECT PASS
113+
%ERROR "Integer literal too big" at U32
114+
%ERROR "Integer literal too big" at U64
115+
%ERROR "Integer literal too big" at U64S
116+
%ENDTEST
117+
118+
%% The general pattern for real literals is:
119+
%% I:Int P:Period F:ZeroesInt E:(Exp PlusMinus Int)
120+
%% Assuming all are optional then we have the following combinations
121+
%% I, IP, IPF, IPFE, IF, IFE, IE, IPE
122+
%% P, PF, PFE, PE
123+
%% F, FE
124+
%% EE
125+
%% Due to the underlying regex patterns there are the folllowing
126+
%% combinations which are the same:
127+
%% IF == I, IFE == IE
128+
%% Which leaves the following classification:
129+
%% I -- matches as integer
130+
%% IP - e.g. 1. -- VALID
131+
%% IPF - e.g. 1.1 -- VALID
132+
%% IPFE - e.g. 1.1e1 -- VALID
133+
%% IE - e.g. 1e1 -- VALID
134+
%% IPE - e.g. 1.e1 -- VALID
135+
%% P - . -- INVALID
136+
%% PF - e.g. .1 -- VALID
137+
%% PFE - e.g. .1e1 -- VALID
138+
%% PE - .e1 -- INVALID
139+
%% F - e.g. 01 -- INVALID
140+
%% FE - e.g. 01e1 -- INVALID
141+
%% E - e.g. e1 -- matches as identifier
142+
%%
143+
%TEST RealLiterals
144+
module compiler_test
145+
constant ValidInt is [ 0. , 1. , 10. ]
146+
constant ValidFrac is [ .0, .1, .00, .10, .01 ]
147+
constant ValidIntFrac is [ 0.0, 0.1, 0.00, 0.10, 0.01, \
148+
1.0, 1.1, 1.00, 1.10, 1.01, \
149+
10.0, 10.1, 10.00, 10.10, 10.01 ]
150+
constant ValidExp is [ 0e10, 1e1 , 10e10, 1e1 ]
151+
constant ValidDecExp is [ 0.e10, 0.e1 , .0e10, .0e1 ]
152+
constant ValidPosExp is [ 0.e+10, 0.e+1 , .0e+10, .0e+1 ]
153+
constant ValidNegExp is [ 0.e-10, 0.e-1 , .0e-10, .0e-1 ]
154+
155+
constant Invalid_PE is %{INVALID_PE}.e1
156+
constant Invalid_F is %{INVALID_F}01
157+
constant Invalid_FE is %{INVALID_FE}01e1
158+
159+
constant InvalidInt is [ %{INVALID_INT}01. ]
160+
constant InvalidIntFrac is [ %{INVALID_INTFRAC}01.0 ]
161+
constant InvalidExp is [ %{INVALID_EXP1}00e10, %{INVALID_EXP2}0e010 ]
162+
163+
constant IntSuffix is %{INT_SUFFIX}0.foo
164+
constant FracSuffix is %{FRAC_SUFFIX}.0foo
165+
constant IntFracSuffix is %{INTFRAC_SUFFIX}0.1foo
166+
constant ExpSuffix is %{EXP_SUFFIX}0e10foo
167+
constant DecExpSuffix is %{DECEXP_SUFFIX}.0e1foo
168+
constant PosExpSuffix is %{POSEXP_SUFFIX}.0e+1foo
169+
constant NegExpSuffix is %{NEGEXP_SUFFIX}.0e-1foo
170+
171+
constant IntDotSuffix is %{INTDOT_SUFFIX}0..
172+
constant FracDotSuffix is %{FRACDOT_SUFFIX}.0.
173+
constant IntFracDotSuffix is %{INTFRACDOT_SUFFIX}0.1.
174+
constant ExpDotSuffix is %{EXPDOT_SUFFIX}0e10.
175+
constant DecExpDotSuffix is %{DECEXPDOT_SUFFIX}.0e1.
176+
constant PosExpDotSuffix is %{POSEXPDOT_SUFFIX}.0e+1.
177+
constant NegExpDotSuffix is %{NEGEXPDOT_SUFFIX}.0e-1.
178+
179+
end module
180+
%EXPECT PASS
181+
%ERROR "Malformed real literal" AT INVALID_PE
182+
%ERROR "Malformed integer literal" AT INVALID_F
183+
%ERROR "Malformed real literal" AT INVALID_FE
184+
185+
%ERROR "Malformed real literal" AT INVALID_INT
186+
%ERROR "Malformed real literal" AT INVALID_INTFRAC
187+
%ERROR "Malformed real literal" AT INVALID_EXP1
188+
%ERROR "Malformed real literal" AT INVALID_EXP2
189+
190+
%ERROR "Malformed real literal" AT INT_SUFFIX
191+
%ERROR "Malformed real literal" AT FRAC_SUFFIX
192+
%ERROR "Malformed real literal" AT INTFRAC_SUFFIX
193+
%ERROR "Malformed real literal" AT EXP_SUFFIX
194+
%ERROR "Malformed real literal" AT DECEXP_SUFFIX
195+
%ERROR "Malformed real literal" AT POSEXP_SUFFIX
196+
%ERROR "Malformed real literal" AT NEGEXP_SUFFIX
197+
198+
%ERROR "Malformed real literal" AT INTDOT_SUFFIX
199+
%ERROR "Malformed real literal" AT FRACDOT_SUFFIX
200+
%ERROR "Malformed real literal" AT INTFRACDOT_SUFFIX
201+
%ERROR "Malformed real literal" AT EXPDOT_SUFFIX
202+
%ERROR "Malformed real literal" AT DECEXPDOT_SUFFIX
203+
%ERROR "Malformed real literal" AT POSEXPDOT_SUFFIX
204+
%ERROR "Malformed real literal" AT NEGEXPDOT_SUFFIX
205+
%ENDTEST
206+
207+
%TEST RealLiteralRange
208+
module compiler_test
209+
constant BiggerThanFloat is %{BIGF}3.40282347E+39
210+
constant SmallerThanFloat is %{SMALLF}3.40282347E+38
211+
constant BiggerThanDouble is %{BIGD}1.7976931348623157E+309
212+
constant SmallerThanDouble is %{SMALLD}1.7976931348623157E+308
213+
end module
214+
%EXPECT PASS
215+
%ERROR "Real literal too big" at BIGD
216+
%ENDTEST

‎toolchain/lc-compile/src/DOUBLE_LITERAL.t‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,51 @@
1-
[0-9]+"."[0-9]+([eE][-+]?[0-9]+)? {
2-
MakeDoubleLiteral(yytext, &yylval.attr[1]);
3-
yysetpos();
4-
return DOUBLE_LITERAL;
1+
/* This contains both INTEGER_LITERAL and DOUBLE_LITERAL rules */
2+
/* This ensures that they appear in this order, to ensure they match */
3+
/* correctly. */
4+
5+
[0-9]+"."[0-9]*([eE][-+]?[0-9]*)?([0-9a-zA-Z_\.]*) |
6+
[0-9]*"."[0-9]+([eE][-+]?[0-9]*)?([0-9a-zA-Z_\.]*) |
7+
[0-9]*"."[0-9]*([eE][-+]?[0-9]*)([0-9a-zA-Z_\.]*) |
8+
[0-9]+([eE][-+]?[0-9]*)([0-9a-zA-Z_\.]*) {
9+
yysetpos();
10+
if (IsDoubleLiteral(yytext))
11+
{
12+
if (MakeDoubleLiteral(yytext, &yylval.attr[1]) == 0)
13+
{
14+
long t_position;
15+
GetCurrentPosition(&t_position);
16+
Error_DoubleLiteralOutOfRange(t_position);
17+
yylval.attr[1] = 0;
18+
}
19+
}
20+
else
21+
{
22+
long t_position;
23+
GetCurrentPosition(&t_position);
24+
Error_InvalidDoubleLiteral(t_position);
25+
yylval.attr[1] = 0;
26+
}
27+
return DOUBLE_LITERAL;
28+
}
29+
30+
([0-9]+|0b[01]*|0x[0-9A-Fa-f]*)[0-9a-zA-Z_]* {
31+
yysetpos();
32+
if (IsIntegerLiteral(yytext))
33+
{
34+
if (MakeIntegerLiteral(yytext, &yylval.attr[1]) == 0)
35+
{
36+
long t_position;
37+
GetCurrentPosition(&t_position);
38+
Error_IntegerLiteralOutOfRange(t_position);
39+
yylval.attr[1] = 0;
40+
}
41+
}
42+
else
43+
{
44+
long t_position;
45+
GetCurrentPosition(&t_position);
46+
Error_InvalidIntegerLiteral(t_position);
47+
yylval.attr[1] = 0;
48+
}
49+
return INTEGER_LITERAL;
550
}
651

‎toolchain/lc-compile/src/INTEGER_LITERAL.t‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1 @@
1-
0|([1-9][0-9]*) {
2-
yysetpos();
3-
if (MakeIntegerLiteral(yytext, &yylval.attr[1]) == 0)
4-
{
5-
long t_position;
6-
GetCurrentPosition(&t_position);
7-
Error_IntegerLiteralOutOfRange(t_position);
8-
yylval.attr[1] = 0;
9-
}
10-
return INTEGER_LITERAL;
11-
}
12-
1+
/* See DOUBLE_LITERAL.t */

0 commit comments

Comments
 (0)