Skip to content

Commit 08d714b

Browse files
JanTvrdikJanTvrdik
committed
simplify and improve whitespace handling
1 parent 3fed7e2 commit 08d714b

File tree

4 files changed

+120
-59
lines changed

4 files changed

+120
-59
lines changed

‎src/Lexer/Lexer.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,8 @@ private function initialize()
143143
self::TOKEN_HORIZONTAL_WS => '\\h++',
144144
self::TOKEN_EOL => '(?:\\r?+\\n)++',
145145

146-
// anything but TOKEN_PHPDOC_TAG or TOKEN_CLOSE_PHPDOC or TOKEN_EOL
147-
self::TOKEN_OTHER => '(?:(?!@[a-z-])(?!\\*/)(?!\\r?+\\n).)++',
146+
// anything but TOKEN_PHPDOC_TAG or TOKEN_CLOSE_PHPDOC or TOKEN_HORIZONTAL_WS or TOKEN_EOL
147+
self::TOKEN_OTHER => '(?:(?!@[a-z-])(?!\\*/)[^\\s])++',
148148
];
149149

150150
$this->regexp = '~(' . implode(')|(', $patterns) . ')~Asi';

‎src/Parser/PhpDocParser.php‎

Lines changed: 22 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -29,38 +29,28 @@ public function __construct(TypeParser $typeParser, ConstExprParser $constantExp
2929

3030
public function parse(TokenIterator $tokens): Ast\PhpDoc\PhpDocNode
3131
{
32-
$textNode = '';
33-
$children = [];
3432
$tokens->consumeTokenType(Lexer::TOKEN_OPEN_PHPDOC);
3533

36-
if ($tokens->tryConsumeHorizontalWhiteSpace()) {
37-
$textNode .= $tokens->prevTokenValue();
38-
}
34+
$children = [];
35+
$children[] = $this->parseText($tokens);
3936

40-
while (!$tokens->tryConsumeTokenType(Lexer::TOKEN_CLOSE_PHPDOC)) {
41-
if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG)) {
42-
if ($textNode !== '') {
43-
$children[] = new Ast\PhpDoc\PhpDocTextNode($textNode);
44-
$textNode = '';
45-
}
37+
while (!$tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_PHPDOC) && !$tokens->isCurrentTokenType(Lexer::TOKEN_END)) {
38+
$children[] = $this->parseTag($tokens);
39+
$children[] = $this->parseText($tokens);
40+
}
4641

47-
$children[] = $this->parseTag($tokens);
42+
$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_PHPDOC);
4843

49-
} else {
50-
$textNode .= $tokens->currentTokenValue();
51-
$tokens->next();
44+
return new Ast\PhpDoc\PhpDocNode(array_values($children));
45+
}
5246

53-
if ($tokens->tryConsumeHorizontalWhiteSpace()) {
54-
$textNode .= $tokens->prevTokenValue();
55-
}
56-
}
57-
}
5847

59-
if ($textNode !== '') {
60-
$children[] = new Ast\PhpDoc\PhpDocTextNode($textNode);
61-
}
48+
private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode
49+
{
50+
$text = $tokens->getSkippedHorizontalWhiteSpaceIfAny();
51+
$text .= $tokens->joinUntil(Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_CLOSE_PHPDOC);
6252

63-
return new Ast\PhpDoc\PhpDocNode($children);
53+
return new Ast\PhpDoc\PhpDocTextNode($text);
6454
}
6555

6656

@@ -222,7 +212,7 @@ private function parseMethodTagValueParameter(TokenIterator $tokens): Ast\PhpDoc
222212

223213
private function parseOptionalVariableName(TokenIterator $tokens): string
224214
{
225-
if ($tokens->tryConsumeHorizontalWhiteSpace() && $tokens->isCurrentTokenType(Lexer::TOKEN_VARIABLE)) {
215+
if ($tokens->isCurrentTokenType(Lexer::TOKEN_VARIABLE)) {
226216
$parameterName = $tokens->currentTokenValue();
227217
$tokens->next();
228218

@@ -235,7 +225,6 @@ private function parseOptionalVariableName(TokenIterator $tokens): string
235225

236226
private function parseRequiredVariableName(TokenIterator $tokens): string
237227
{
238-
$tokens->tryConsumeHorizontalWhiteSpace();
239228
$parameterName = $tokens->currentTokenValue();
240229
$tokens->consumeTokenType(Lexer::TOKEN_VARIABLE);
241230

@@ -253,12 +242,14 @@ private function parseOptionalDescription(TokenIterator $tokens, bool $limitStar
253242
}
254243
}
255244

256-
// description MUST separated from any previous node by horizontal whitespace
257-
if ($tokens->tryConsumeHorizontalWhiteSpace()) {
258-
return $tokens->joinUntil(Lexer::TOKEN_EOL, Lexer::TOKEN_PHPDOC_TAG, Lexer::TOKEN_CLOSE_PHPDOC);
259-
}
245+
$description = $tokens->joinUntil(
246+
Lexer::TOKEN_EOL,
247+
Lexer::TOKEN_PHPDOC_TAG,
248+
Lexer::TOKEN_CLOSE_PHPDOC
249+
);
260250

261-
return '';
251+
// the trimmed characters MUST match Lexer::TOKEN_HORIZONTAL_WS
252+
return rtrim($description, " \t");
262253
}
263254

264255
}

‎src/Parser/TokenIterator.php‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,6 @@ public function __construct(array $tokens, int $index = 0)
2727
}
2828

2929

30-
public function prevTokenValue(): string
31-
{
32-
return $this->tokens[$this->index - 1][Lexer::VALUE_OFFSET];
33-
}
34-
35-
36-
public function prevTokenType(): int
37-
{
38-
return $this->tokens[$this->index - 1][Lexer::TYPE_OFFSET];
39-
}
40-
41-
4230
public function currentTokenValue(): string
4331
{
4432
return $this->tokens[$this->index][Lexer::VALUE_OFFSET];
@@ -124,9 +112,13 @@ public function tryConsumeTokenType(int $tokenType): bool
124112
}
125113

126114

127-
public function tryConsumeHorizontalWhiteSpace(): bool
115+
public function getSkippedHorizontalWhiteSpaceIfAny(): string
128116
{
129-
return $this->tokens[$this->index - 1][Lexer::TYPE_OFFSET] === Lexer::TOKEN_HORIZONTAL_WS;
117+
if ($this->tokens[$this->index - 1][Lexer::TYPE_OFFSET] === Lexer::TOKEN_HORIZONTAL_WS) {
118+
return $this->tokens[$this->index - 1][Lexer::VALUE_OFFSET];
119+
}
120+
121+
return '';
130122
}
131123

132124

‎tests/PHPStan/Parser/PhpDocParserTest.php‎

Lines changed: 90 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,9 @@ public function provideParseData(): array
7373
new PhpDocTextNode(' '),
7474
new PhpDocTagNode(
7575
'@foo',
76-
new GenericTagValueNode('lorem ')
76+
new GenericTagValueNode('lorem')
7777
),
78+
new PhpDocTextNode(' '),
7879
]),
7980
],
8081
[
@@ -83,8 +84,9 @@ public function provideParseData(): array
8384
new PhpDocTextNode(' '),
8485
new PhpDocTagNode(
8586
'@foo',
86-
new GenericTagValueNode('lorem ipsum ')
87+
new GenericTagValueNode('lorem ipsum')
8788
),
89+
new PhpDocTextNode(' '),
8890
]),
8991
],
9092
[
@@ -100,6 +102,7 @@ public function provideParseData(): array
100102
''
101103
)
102104
),
105+
new PhpDocTextNode(' '),
103106
]),
104107
],
105108
[
@@ -109,7 +112,7 @@ public function provideParseData(): array
109112
new PhpDocTagNode(
110113
'@param',
111114
new InvalidTagValueNode(
112-
'Foo optional description ',
115+
'Foo optional description',
113116
new \PHPStan\PhpDocParser\Parser\ParserException(
114117
'optional',
115118
Lexer::TOKEN_IDENTIFIER,
@@ -118,6 +121,7 @@ public function provideParseData(): array
118121
)
119122
)
120123
),
124+
new PhpDocTextNode(' '),
121125
]),
122126
],
123127
[
@@ -130,9 +134,10 @@ public function provideParseData(): array
130134
new IdentifierTypeNode('Foo'),
131135
false,
132136
'$foo',
133-
'optional description '
137+
'optional description'
134138
)
135139
),
140+
new PhpDocTextNode(' '),
136141
]),
137142
],
138143
[
@@ -145,9 +150,10 @@ public function provideParseData(): array
145150
new IdentifierTypeNode('Foo'),
146151
true,
147152
'$foo',
148-
'optional description '
153+
'optional description'
149154
)
150155
),
156+
new PhpDocTextNode(' '),
151157
]),
152158
],
153159
[
@@ -161,6 +167,7 @@ public function provideParseData(): array
161167
''
162168
)
163169
),
170+
new PhpDocTextNode(' '),
164171
]),
165172
],
166173
[
@@ -171,9 +178,10 @@ public function provideParseData(): array
171178
'@return',
172179
new ReturnTagValueNode(
173180
new IdentifierTypeNode('Foo'),
174-
'optional description '
181+
'optional description'
175182
)
176183
),
184+
new PhpDocTextNode(' '),
177185
]),
178186
],
179187
[
@@ -184,9 +192,10 @@ public function provideParseData(): array
184192
'@return',
185193
new ReturnTagValueNode(
186194
new IdentifierTypeNode('array'),
187-
'[int] '
195+
'[int]'
188196
)
189197
),
198+
new PhpDocTextNode(' '),
190199
]),
191200
],
192201
[
@@ -196,7 +205,7 @@ public function provideParseData(): array
196205
new PhpDocTagNode(
197206
'@return',
198207
new InvalidTagValueNode(
199-
'[int, string] ',
208+
'[int, string]',
200209
new \PHPStan\PhpDocParser\Parser\ParserException(
201210
'[',
202211
Lexer::TOKEN_OPEN_SQUARE_BRACKET,
@@ -205,6 +214,7 @@ public function provideParseData(): array
205214
)
206215
)
207216
),
217+
new PhpDocTextNode(' '),
208218
]),
209219
],
210220
[
@@ -214,7 +224,7 @@ public function provideParseData(): array
214224
new PhpDocTagNode(
215225
'@return',
216226
new InvalidTagValueNode(
217-
'A & B | C ',
227+
'A & B | C',
218228
new \PHPStan\PhpDocParser\Parser\ParserException(
219229
'|',
220230
Lexer::TOKEN_UNION,
@@ -223,6 +233,7 @@ public function provideParseData(): array
223233
)
224234
)
225235
),
236+
new PhpDocTextNode(' '),
226237
]),
227238
],
228239
[
@@ -232,7 +243,7 @@ public function provideParseData(): array
232243
new PhpDocTagNode(
233244
'@return',
234245
new InvalidTagValueNode(
235-
'A | B & C ',
246+
'A | B & C',
236247
new \PHPStan\PhpDocParser\Parser\ParserException(
237248
'&',
238249
Lexer::TOKEN_INTERSECTION,
@@ -241,6 +252,7 @@ public function provideParseData(): array
241252
)
242253
)
243254
),
255+
new PhpDocTextNode(' '),
244256
]),
245257
],
246258
[
@@ -250,7 +262,7 @@ public function provideParseData(): array
250262
new PhpDocTagNode(
251263
'@return',
252264
new InvalidTagValueNode(
253-
'A | B < 123 ',
265+
'A | B < 123',
254266
new \PHPStan\PhpDocParser\Parser\ParserException(
255267
'123',
256268
Lexer::TOKEN_INTEGER,
@@ -259,6 +271,7 @@ public function provideParseData(): array
259271
)
260272
)
261273
),
274+
new PhpDocTextNode(' '),
262275
]),
263276
],
264277
[
@@ -272,9 +285,10 @@ public function provideParseData(): array
272285
new IdentifierTypeNode('callable')
273286
),
274287
'',
275-
'function (Configurator $sender, DI\Compiler $compiler); Occurs after the compiler is created '
288+
'function (Configurator $sender, DI\Compiler $compiler); Occurs after the compiler is created'
276289
)
277290
),
291+
new PhpDocTextNode(' '),
278292
]),
279293
],
280294
[
@@ -289,10 +303,74 @@ public function provideParseData(): array
289303
''
290304
)
291305
),
306+
new PhpDocTextNode(' '),
292307
new PhpDocTagNode(
293308
'@inject',
294309
new GenericTagValueNode('')
295310
),
311+
new PhpDocTextNode(' '),
312+
]),
313+
],
314+
[
315+
'/** @var \\\\Foo $foo */',
316+
new PhpDocNode([
317+
new PhpDocTextNode(' '),
318+
new PhpDocTagNode(
319+
'@var',
320+
new InvalidTagValueNode(
321+
'\\\\Foo $foo',
322+
new \PHPStan\PhpDocParser\Parser\ParserException(
323+
'\\\\Foo',
324+
Lexer::TOKEN_OTHER,
325+
9,
326+
Lexer::TOKEN_IDENTIFIER
327+
)
328+
)
329+
),
330+
new PhpDocTextNode(' '),
331+
]),
332+
],
333+
[
334+
'/** @varFoo $foo */',
335+
new PhpDocNode([
336+
new PhpDocTextNode(' '),
337+
new PhpDocTagNode(
338+
'@varFoo',
339+
new GenericTagValueNode(
340+
'$foo'
341+
)
342+
),
343+
new PhpDocTextNode(' '),
344+
]),
345+
],
346+
[
347+
'/** @var Foo$foo */',
348+
new PhpDocNode([
349+
new PhpDocTextNode(' '),
350+
new PhpDocTagNode(
351+
'@var',
352+
new VarTagValueNode(
353+
new IdentifierTypeNode('Foo'),
354+
'$foo',
355+
''
356+
)
357+
),
358+
new PhpDocTextNode(' '),
359+
]),
360+
],
361+
[
362+
'/**@var(Foo)$foo#desc*/',
363+
new PhpDocNode([
364+
new PhpDocTextNode(''),
365+
new PhpDocTagNode(
366+
'@var',
367+
new VarTagValueNode(
368+
new IdentifierTypeNode('Foo'),
369+
'$foo',
370+
'#desc'
371+
)
372+
),
373+
new PhpDocTextNode(''),
296374
]),
297375
],
298376
[

0 commit comments

Comments
 (0)