Treat namespaced names as single token

nikic · nikic · commit 7a3dcc3e339c · 2020-07-22T12:36:05.000+02:00
Namespace names are now lexed as single tokens of type T_NAME_QUALIFIED, T_NAME_FULLY_QUALIFIED or T_NAME_RELATIVE. RFC: https://wiki.php.net/rfc/namespaced_names_as_token Closes GH-5827.
diff --git a/UPGRADING b/UPGRADING
@@ -204,6 +204,10 @@ PHP 8.0 UPGRADE NOTES
     numbers and numeric strings continue to work as before. Notably, this means
     that `0 == "not-a-number"` is considered false now.
     RFC: https://wiki.php.net/rfc/string_to_number_comparison
+  . Namespaced names can no longer contain whitespace: While `Foo\Bar` will be
+    recognized as a namespaced name, `Foo \ Bar` will not. Conversely, reserved
+    keywords are now permitted as namespace segments.
+    RFC: https://wiki.php.net/rfc/namespaced_names_as_token
 
 - COM:
   . Removed the ability to import case-insensitive constants from type
@@ -509,6 +513,11 @@ PHP 8.0 UPGRADE NOTES
     instead be part of a following T_WHITESPACE token. It should be noted that
     T_COMMENT is not always followed by whitespace, it may also be followed by
     T_CLOSE_TAG or end-of-file.
+  . Namespaced names are now represented using the T_NAME_QUALIFIED (Foo\Bar),
+    T_NAME_FULLY_QUALIFIED (\Foo\Bar) and T_NAME_RELATIVE (namespace\Foo\Bar)
+    tokens. T_NS_SEPARATOR is only used for standalone namespace separators,
+    and only syntactially valid in conjunction with group use declarations.
+    RFC: https://wiki.php.net/rfc/namespaced_names_as_token
 
 - XML:
   . xml_parser_create(_ns) will now return an XmlParser object rather than a
diff --git a/Zend/tests/attributes/019_variable_attribute_name.phpt b/Zend/tests/attributes/019_variable_attribute_name.phpt
@@ -8,4 +8,4 @@ class A {}
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected variable "$x", expecting identifier or "static" or "namespace" or "\" in %s on line %d
+Parse error: syntax error, unexpected variable "$x" in %s on line %d
diff --git a/Zend/tests/bug43343.phpt b/Zend/tests/bug43343.phpt
@@ -8,4 +8,4 @@ $foo = 'bar';
 var_dump(new namespace::$foo);
 ?>
 --EXPECTF--
-Parse error: %s error%sexpecting%s"\"%sin %sbug43343.php on line 5
+Parse error: syntax error, unexpected token "namespace" in %s on line %d
diff --git a/Zend/tests/bug55086.phpt b/Zend/tests/bug55086.phpt
@@ -26,7 +26,7 @@ namespace N2 {
     echo $a->hello(), PHP_EOL;
     echo $a->foo(), PHP_EOL;
     try {
-    } catch(namespace \Foo $e)
+    } catch (namespace\Foo $e)
     {
     }
 }
diff --git a/Zend/tests/grammar/regression_010.phpt b/Zend/tests/grammar/regression_010.phpt
@@ -5,8 +5,11 @@ Test to check regressions on T_IMPLEMENTS followed by a T_NS_SEPARATOR
 
 interface A{}
 
+// No longer considered legal in PHP 8.
 class B implements\A {}
 
 echo "Done", PHP_EOL;
---EXPECT--
-Done
+
+?>
+--EXPECTF--
+Parse error: syntax error, unexpected namespaced name "implements\A", expecting "{" in %s on line %d
diff --git a/Zend/tests/namespace_name_namespace.phpt b/Zend/tests/namespace_name_namespace.phpt
@@ -0,0 +1,10 @@
+--TEST--
+Cannot use "namespace" as namespace name, due to conflict with ns-relative names
+--FILE--
+<?php
+
+namespace NAMEspace;
+
+?>
+--EXPECTF--
+Fatal error: Cannot use 'NAMEspace' as namespace name in %s on line %d
diff --git a/Zend/tests/namespace_name_namespace_start.phpt b/Zend/tests/namespace_name_namespace_start.phpt
@@ -0,0 +1,10 @@
+--TEST--
+Cannot use "namespace\xyz" as namespace name, due to conflict with ns-relative names
+--FILE--
+<?php
+
+namespace NAMEspace\xyz;
+
+?>
+--EXPECTF--
+Parse error: syntax error, unexpected namespace-relative name "NAMEspace\xyz", expecting "{" in %s on line %d
diff --git a/Zend/tests/namespace_name_reserved_keywords.phpt b/Zend/tests/namespace_name_reserved_keywords.phpt
@@ -0,0 +1,37 @@
+--TEST--
+Reserved keywords in namespace name
+--FILE--
+<?php
+
+namespace iter\fn {
+    function test() {
+        echo __FUNCTION__, "\n";
+    }
+}
+
+namespace fn {
+    function test() {
+        echo __FUNCTION__, "\n";
+    }
+}
+
+namespace self {
+    function test() {
+        echo __FUNCTION__, "\n";
+    }
+}
+
+namespace {
+    use iter\fn;
+    use function fn\test as test2;
+    use function self\test as test3;
+    fn\test();
+    test2();
+    test3();
+}
+
+?>
+--EXPECT--
+iter\fn\test
+fn\test
+self\test
diff --git a/Zend/tests/namespaced_name_whitespace.phpt b/Zend/tests/namespaced_name_whitespace.phpt
@@ -0,0 +1,10 @@
+--TEST--
+Whitespace between namespace separators is no longer allowed
+--FILE--
+<?php
+
+Foo \ Bar \ Baz;
+
+?>
+--EXPECTF--
+Parse error: syntax error, unexpected token "\" in %s on line %d
diff --git a/Zend/tests/ns_096.phpt b/Zend/tests/ns_096.phpt
@@ -7,4 +7,4 @@ use Foo\Bar\{\Baz};
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected token "\", expecting identifier or "function" or "const" in %s on line %d
+Parse error: syntax error, unexpected fully qualified name "\Baz", expecting identifier or namespaced name or "function" or "const" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_01.phpt b/Zend/tests/ns_trailing_comma_error_01.phpt
@@ -5,4 +5,4 @@ Group use declarations mustn't be empty
 use Baz\{};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected token "}", expecting identifier or "function" or "const" in %s on line %d
+Parse error: syntax error, unexpected token "}", expecting identifier or namespaced name or "function" or "const" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_02.phpt b/Zend/tests/ns_trailing_comma_error_02.phpt
@@ -5,4 +5,4 @@ Group use declarations mustn't contain just a comma
 use Baz\{,};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected token ",", expecting identifier or "function" or "const" in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting identifier or namespaced name or "function" or "const" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_04.phpt b/Zend/tests/ns_trailing_comma_error_04.phpt
@@ -5,4 +5,4 @@ Group use declarations mustn't begin with a comma
 use Baz\{,Foo};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected token ",", expecting identifier or "function" or "const" in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting identifier or namespaced name or "function" or "const" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_07.phpt b/Zend/tests/ns_trailing_comma_error_07.phpt
@@ -5,4 +5,4 @@ Unmixed group use declarations mustn't begin with a comma
 use function Baz\{,Foo};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected token ",", expecting identifier in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting identifier or namespaced name in %s on line %d
diff --git a/Zend/tests/special_name_error1.phpt b/Zend/tests/special_name_error1.phpt
diff --git a/Zend/zend_compile.c b/Zend/zend_compile.c
@@ -7432,7 +7432,7 @@ void zend_compile_namespace(zend_ast *ast) /* {{{ */
 	if (name_ast) {
 		name = zend_ast_get_str(name_ast);
 
-		if (ZEND_FETCH_CLASS_DEFAULT != zend_get_class_fetch_type(name)) {
+		if (zend_string_equals_literal_ci(name, "namespace")) {
 			zend_error_noreturn(E_COMPILE_ERROR, "Cannot use '%s' as namespace name", ZSTR_VAL(name));
 		}
 
diff --git a/Zend/zend_language_parser.y b/Zend/zend_language_parser.y
@@ -89,6 +89,9 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %token <ast> T_LNUMBER   "integer"
 %token <ast> T_DNUMBER   "floating-point number"
 %token <ast> T_STRING    "identifier"
+%token <ast> T_NAME_FULLY_QUALIFIED "fully qualified name"
+%token <ast> T_NAME_RELATIVE "namespace-relative name"
+%token <ast> T_NAME_QUALIFIED "namespaced name"
 %token <ast> T_VARIABLE  "variable"
 %token <ast> T_INLINE_HTML
 %token <ast> T_ENCAPSED_AND_WHITESPACE  "string content"
@@ -231,7 +234,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %token T_ERROR
 
 %type <ast> top_statement namespace_name name statement function_declaration_statement
-%type <ast> class_declaration_statement trait_declaration_statement
+%type <ast> class_declaration_statement trait_declaration_statement legacy_namespace_name
 %type <ast> interface_declaration_statement interface_extends_list
 %type <ast> group_use_declaration inline_use_declarations inline_use_declaration
 %type <ast> mixed_group_use_declaration use_declaration unprefixed_use_declaration
@@ -261,7 +264,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %type <ast> identifier type_expr_without_static union_type_without_static
 %type <ast> inline_function union_type
 %type <ast> attributed_statement attributed_class_statement attributed_parameter
-%type <ast> attribute_decl attribute attributes
+%type <ast> attribute_decl attribute attributes namespace_declaration_name
 %type <ast> match match_arm_list non_empty_match_arm_list match_arm match_arm_cond_list
 
 %type <num> returns_ref function fn is_reference is_variadic variable_modifiers
@@ -308,15 +311,29 @@ top_statement_list:
 	|	%empty { $$ = zend_ast_create_list(0, ZEND_AST_STMT_LIST); }
 ;
 
+/* Name usable in a namespace declaration. */
+namespace_declaration_name:
+		identifier								{ $$ = $1; }
+	|	T_NAME_QUALIFIED						{ $$ = $1; }
+;
+
+/* Name usable in "use" declarations (loading separator forbidden). */
 namespace_name:
 		T_STRING								{ $$ = $1; }
-	|	namespace_name T_NS_SEPARATOR T_STRING	{ $$ = zend_ast_append_str($1, $3); }
+	|	T_NAME_QUALIFIED						{ $$ = $1; }
+;
+
+/* Name usable in "use" declarations (leading separator allowed). */
+legacy_namespace_name:
+		namespace_name							{ $$ = $1; }
+	|	T_NAME_FULLY_QUALIFIED					{ $$ = $1; }
 ;
 
 name:
-		namespace_name								{ $$ = $1; $$->attr = ZEND_NAME_NOT_FQ; }
-	|	T_NAMESPACE T_NS_SEPARATOR namespace_name	{ $$ = $3; $$->attr = ZEND_NAME_RELATIVE; }
-	|	T_NS_SEPARATOR namespace_name				{ $$ = $2; $$->attr = ZEND_NAME_FQ; }
+		T_STRING									{ $$ = $1; $$->attr = ZEND_NAME_NOT_FQ; }
+	|	T_NAME_QUALIFIED							{ $$ = $1; $$->attr = ZEND_NAME_NOT_FQ; }
+	|	T_NAME_FULLY_QUALIFIED						{ $$ = $1; $$->attr = ZEND_NAME_FQ; }
+	|	T_NAME_RELATIVE								{ $$ = $1; $$->attr = ZEND_NAME_RELATIVE; }
 ;
 
 attribute_decl:
@@ -350,10 +367,10 @@ top_statement:
 			{ $$ = zend_ast_create(ZEND_AST_HALT_COMPILER,
 			      zend_ast_create_zval_from_long(zend_get_scanned_file_offset()));
 			  zend_stop_lexing(); }
-	|	T_NAMESPACE namespace_name ';'
+	|	T_NAMESPACE namespace_declaration_name ';'
 			{ $$ = zend_ast_create(ZEND_AST_NAMESPACE, $2, NULL);
 			  RESET_DOC_COMMENT(); }
-	|	T_NAMESPACE namespace_name { RESET_DOC_COMMENT(); }
+	|	T_NAMESPACE namespace_declaration_name { RESET_DOC_COMMENT(); }
 		'{' top_statement_list '}'
 			{ $$ = zend_ast_create(ZEND_AST_NAMESPACE, $2, $5); }
 	|	T_NAMESPACE { RESET_DOC_COMMENT(); }
@@ -372,17 +389,13 @@ use_type:
 ;
 
 group_use_declaration:
-		namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations possible_comma '}'
+		legacy_namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations possible_comma '}'
 			{ $$ = zend_ast_create(ZEND_AST_GROUP_USE, $1, $4); }
-	|	T_NS_SEPARATOR namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations possible_comma '}'
-			{ $$ = zend_ast_create(ZEND_AST_GROUP_USE, $2, $5); }
 ;
 
 mixed_group_use_declaration:
-		namespace_name T_NS_SEPARATOR '{' inline_use_declarations possible_comma '}'
+		legacy_namespace_name T_NS_SEPARATOR '{' inline_use_declarations possible_comma '}'
 			{ $$ = zend_ast_create(ZEND_AST_GROUP_USE, $1, $4);}
-	|	T_NS_SEPARATOR namespace_name T_NS_SEPARATOR '{' inline_use_declarations possible_comma '}'
-			{ $$ = zend_ast_create(ZEND_AST_GROUP_USE, $2, $5); }
 ;
 
 possible_comma:
@@ -424,8 +437,10 @@ unprefixed_use_declaration:
 ;
 
 use_declaration:
-		unprefixed_use_declaration                { $$ = $1; }
-	|	T_NS_SEPARATOR unprefixed_use_declaration { $$ = $2; }
+		legacy_namespace_name
+			{ $$ = zend_ast_create(ZEND_AST_USE_ELEM, $1, NULL); }
+	|	legacy_namespace_name T_AS T_STRING
+			{ $$ = zend_ast_create(ZEND_AST_USE_ELEM, $1, $3); }
 ;
 
 const_list:
diff --git a/Zend/zend_language_scanner.l b/Zend/zend_language_scanner.l
@@ -1596,10 +1596,6 @@ NEWLINE ("\r"|"\n"|"\r\n")
 	RETURN_TOKEN(T_PAAMAYIM_NEKUDOTAYIM);
 }
 
-<ST_IN_SCRIPTING>"\\" {
-	RETURN_TOKEN(T_NS_SEPARATOR);
-}
-
 <ST_IN_SCRIPTING>"..." {
 	RETURN_TOKEN(T_ELLIPSIS);
 }
@@ -2288,6 +2284,22 @@ inline_char_handler:
 	RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
 }
 
+<ST_IN_SCRIPTING>"namespace"("\\"{LABEL})+ {
+	RETURN_TOKEN_WITH_STR(T_NAME_RELATIVE, sizeof("namespace\\") - 1);
+}
+
+<ST_IN_SCRIPTING>{LABEL}("\\"{LABEL})+ {
+	RETURN_TOKEN_WITH_STR(T_NAME_QUALIFIED, 0);
+}
+
+<ST_IN_SCRIPTING>"\\"{LABEL}("\\"{LABEL})* {
+	RETURN_TOKEN_WITH_STR(T_NAME_FULLY_QUALIFIED, 1);
+}
+
+<ST_IN_SCRIPTING>"\\" {
+	RETURN_TOKEN(T_NS_SEPARATOR);
+}
+
 <ST_IN_SCRIPTING,ST_VAR_OFFSET>{LABEL} {
 	RETURN_TOKEN_WITH_STR(T_STRING, 0);
 }
diff --git a/ext/tokenizer/tests/namespaced_names.phpt b/ext/tokenizer/tests/namespaced_names.phpt
@@ -0,0 +1,39 @@
+--TEST--
+Tokenization of namespaced names
+--FILE--
+<?php
+
+$code = <<<'CODE'
+<?php
+Foo
+Foo\Bar
+\Foo\Bar
+namespace\Foo
+Foo \ Bar
+CODE;
+
+foreach (PhpToken::getAll($code) as $token) {
+    echo "{$token->getTokenName()}: \"$token->text\"\n";
+}
+
+?>
+--EXPECT--
+T_OPEN_TAG: "<?php
+"
+T_STRING: "Foo"
+T_WHITESPACE: "
+"
+T_NAME_QUALIFIED: "Foo\Bar"
+T_WHITESPACE: "
+"
+T_NAME_FULLY_QUALIFIED: "\Foo\Bar"
+T_WHITESPACE: "
+"
+T_NAME_RELATIVE: "namespace\Foo"
+T_WHITESPACE: "
+"
+T_STRING: "Foo"
+T_WHITESPACE: " "
+T_NS_SEPARATOR: "\"
+T_WHITESPACE: " "
+T_STRING: "Bar"
diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
@@ -77,6 +77,9 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_NAME_FULLY_QUALIFIED", T_NAME_FULLY_QUALIFIED, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_NAME_RELATIVE", T_NAME_RELATIVE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_NAME_QUALIFIED", T_NAME_QUALIFIED, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INLINE_HTML", T_INLINE_HTML, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ENCAPSED_AND_WHITESPACE", T_ENCAPSED_AND_WHITESPACE, CONST_CS | CONST_PERSISTENT);
@@ -221,6 +224,9 @@ char *get_token_type_name(int token_type)
 		case T_LNUMBER: return "T_LNUMBER";
 		case T_DNUMBER: return "T_DNUMBER";
 		case T_STRING: return "T_STRING";
+		case T_NAME_FULLY_QUALIFIED: return "T_NAME_FULLY_QUALIFIED";
+		case T_NAME_RELATIVE: return "T_NAME_RELATIVE";
+		case T_NAME_QUALIFIED: return "T_NAME_QUALIFIED";
 		case T_VARIABLE: return "T_VARIABLE";
 		case T_INLINE_HTML: return "T_INLINE_HTML";
 		case T_ENCAPSED_AND_WHITESPACE: return "T_ENCAPSED_AND_WHITESPACE";

Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@ namespace N2 {`
`26`	`26`	`echo $a->hello(), PHP_EOL;`
`27`	`27`	`echo $a->foo(), PHP_EOL;`
`28`	`28`	`try {`
`29`		`- } catch(namespace \Foo $e)`
	`29`	`+ } catch (namespace\Foo $e)`
`30`	`30`	`{`
`31`	`31`	`}`
`32`	`32`	`}`