diff --git a/src/Language/Lexer.php b/src/Language/Lexer.php index de2314319..928a49550 100644 --- a/src/Language/Lexer.php +++ b/src/Language/Lexer.php @@ -34,6 +34,15 @@ class Lexer private const TOKEN_PIPE = 124; private const TOKEN_BRACE_R = 125; + /** + * Bytes that must be individually inspected while scanning a string body: + * the closing quote ("), the escape character (\), and all C0 control + * characters (0x00-0x1F) except TAB (0x09), which is a legal, unescaped + * SourceCharacter per the GraphQL spec and is scanned through like any + * other ordinary character. + */ + private const STRING_STOP_BYTES = "\"\\\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"; + public Source $source; /** @phpstan-var ParserOptions */ @@ -391,18 +400,43 @@ private function readString(int $line, int $col, Token $prev): Token { $start = $this->position; - // Skip leading quote and read first string char: - [$char, $code, $bytes] = $this->moveStringCursor(1, 1) - ->readChar(); + // Skip leading quote + $this->moveStringCursor(1, 1); - $chunk = ''; + // Bulk-scan runs of bytes that need no special handling via strcspn() + // (a single native call) instead of decoding/validating one UTF-8 + // character at a time. + // + // The stop-byte set includes ASCII control characters (0x00-0x1F), + // not just backslash/quote/line-terminators, because + // assertValidStringCharacterCode() must still reject most of them. + // See self::STRING_STOP_BYTES for details on the excluded TAB (0x09). + + $body = $this->source->body; + $bodyLength = strlen($body); $value = ''; - while (! in_array($code, [null, 10, 13], true)) { // not LineTerminator - if ($code === 34) { // Closing Quote (") + while (true) { + $byteStart = $this->byteStreamPosition; + $runLength = strcspn($body, self::STRING_STOP_BYTES, $byteStart); + + if ($runLength > 0) { + $chunk = substr($body, $byteStart, $runLength); $value .= $chunk; + // $this->position counts decoded characters, not bytes: use + // mb_strlen() to get the codepoint count for multi-byte UTF-8 content. + $this->position += mb_strlen($chunk, 'UTF-8'); + $this->byteStreamPosition = $byteStart + $runLength; + } + + $bytePos = $this->byteStreamPosition; + if ($bytePos >= $bodyLength) { + break; // EOF mid-string + } + + $code = ord($body[$bytePos]); - // Skip quote + if ($code === 34) { // Closing Quote (") $this->moveStringCursor(1, 1); return new Token( @@ -416,79 +450,79 @@ private function readString(int $line, int $col, Token $prev): Token ); } - $this->assertValidStringCharacterCode($code, $this->position); - $this->moveStringCursor(1, $bytes); - - if ($code === 92) { // \ - $value .= $chunk; - [, $code] = $this->readChar(true); + if ($code === 10 || $code === 13) { // LineTerminator + break; + } - switch ($code) { - case 34: - $value .= '"'; - break; - case 47: - $value .= '/'; - break; - case 92: - $value .= '\\'; - break; - case 98: - $value .= chr(8); // \b (backspace) - break; - case 102: - $value .= "\f"; - break; - case 110: - $value .= "\n"; - break; - case 114: - $value .= "\r"; - break; - case 116: - $value .= "\t"; - break; - case 117: - $position = $this->position; - [$hex] = $this->readChars(4); - if (preg_match('/[0-9a-fA-F]{4}/', $hex) !== 1) { - throw new SyntaxError($this->source, $position - 1, "Invalid character escape sequence: \\u{$hex}"); - } + // Any other control character reaching here must be invalid + // (TAB, the only legal one, is excluded from the stop-byte set). + $this->assertValidStringCharacterCode($code, $this->position); - $code = hexdec($hex); - assert(is_int($code), 'Since only a single char is read'); + // $code === 92 (\), i.e. an escape sequence. + $this->moveStringCursor(1, 1); + [, $code] = $this->readChar(true); - // UTF-16 surrogate pair detection and handling. - $highOrderByte = $code >> 8; - if ($highOrderByte >= 0xD8 && $highOrderByte <= 0xDF) { - [$utf16Continuation] = $this->readChars(6); - if (preg_match('/^\\\u[0-9a-fA-F]{4}$/', $utf16Continuation) !== 1) { - throw new SyntaxError($this->source, $this->position - 5, 'Invalid UTF-16 trailing surrogate: ' . $utf16Continuation); - } + // null means EOF; checked separately since switch is not a strict comparison. + if ($code === null) { + break; + } - $surrogatePairHex = $hex . substr($utf16Continuation, 2, 4); - $value .= mb_convert_encoding(pack('H*', $surrogatePairHex), 'UTF-8', 'UTF-16'); - break; + switch ($code) { + case 34: + $value .= '"'; + break; + case 47: + $value .= '/'; + break; + case 92: + $value .= '\\'; + break; + case 98: + $value .= chr(8); // \b (backspace) + break; + case 102: + $value .= "\f"; + break; + case 110: + $value .= "\n"; + break; + case 114: + $value .= "\r"; + break; + case 116: + $value .= "\t"; + break; + case 117: + $position = $this->position; + [$hex] = $this->readChars(4); + if (preg_match('/[0-9a-fA-F]{4}/', $hex) !== 1) { + throw new SyntaxError($this->source, $position - 1, "Invalid character escape sequence: \\u{$hex}"); + } + + $code = hexdec($hex); + assert(is_int($code), 'Since only a single char is read'); + + // UTF-16 surrogate pair detection and handling. + $highOrderByte = $code >> 8; + if ($highOrderByte >= 0xD8 && $highOrderByte <= 0xDF) { + [$utf16Continuation] = $this->readChars(6); + if (preg_match('/^\\\u[0-9a-fA-F]{4}$/', $utf16Continuation) !== 1) { + throw new SyntaxError($this->source, $this->position - 5, 'Invalid UTF-16 trailing surrogate: ' . $utf16Continuation); } - $this->assertValidStringCharacterCode($code, $position - 2); - - $value .= Utils::chr($code); + $surrogatePairHex = $hex . substr($utf16Continuation, 2, 4); + $value .= mb_convert_encoding(pack('H*', $surrogatePairHex), 'UTF-8', 'UTF-16'); break; - // null means EOF, will delegate to general handling of unterminated strings - case null: - continue 2; - default: - $chr = Utils::chr($code); - throw new SyntaxError($this->source, $this->position - 1, "Invalid character escape sequence: \\{$chr}"); - } + } - $chunk = ''; - } else { - $chunk .= $char; - } + $this->assertValidStringCharacterCode($code, $position - 2); - [$char, $code, $bytes] = $this->readChar(); + $value .= Utils::chr($code); + break; + default: + $chr = Utils::chr($code); + throw new SyntaxError($this->source, $this->position - 1, "Invalid character escape sequence: \\{$chr}"); + } } throw new SyntaxError($this->source, $this->position, 'Unterminated string.'); diff --git a/tests/Language/LexerTest.php b/tests/Language/LexerTest.php index 25ad8f11d..fa7969978 100644 --- a/tests/Language/LexerTest.php +++ b/tests/Language/LexerTest.php @@ -296,6 +296,23 @@ public function testLexesStrings(): void ); } + public function testLexesStringsWithUnescapedTabCharacter(): void + { + $source = <<<'GRAPHQL' + "before after" + GRAPHQL; + + self::assertArraySubset( + [ + 'kind' => Token::STRING, + 'start' => 0, + 'end' => 14, + 'value' => "before\tafter", + ], + (array) $this->lexOne($source) + ); + } + /** @see it('lexes block strings') */ public function testLexesBlockString(): void { @@ -423,6 +440,7 @@ public static function reportsUsefulStringErrors(): iterable yield ['"bad \\uD835\\uXXXX esc"', 'Invalid UTF-16 trailing surrogate: \\uXXXX', self::loc(1, 13)]; yield ['"bad \\uD835\\uFXXX esc"', 'Invalid UTF-16 trailing surrogate: \\uFXXX', self::loc(1, 13)]; yield ['"bad \\uD835\\uXXXF esc"', 'Invalid UTF-16 trailing surrogate: \\uXXXF', self::loc(1, 13)]; + yield ['"bad \\' . "\0" . ' esc"', 'Invalid character escape sequence: \\' . "\0", self::loc(1, 7)]; } /**