Skip to content

Commit 60bcd13

Browse files
wchristianwchristian
committed
proof-of-concept for feature-tracking and perl sub signatures (see #273)
1 parent 3cd40bb commit 60bcd13

29 files changed

+407
-133
lines changed

‎lib/PPI/Document.pm

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -128,17 +128,22 @@ In all cases, the document is considered to be "anonymous" and not tied back
128128
to where it was created from. Specifically, if you create a PPI::Document from
129129
a filename, the document will B<not> remember where it was created from.
130130
131+
Returns a C<PPI::Document> object, or C<undef> if parsing fails.
132+
L<PPI::Exception> objects can also be thrown if there are parsing problems.
133+
131134
The constructor also takes attribute flags.
132135
133-
At this time, the only available attribute is the C<readonly> flag.
136+
=head3 readonly
134137
135-
Setting C<readonly> to true will allow various systems to provide
136-
additional optimisations and caching. Note that because C<readonly> is an
137-
optimisation flag, it is off by default and you will need to explicitly
138-
enable it.
138+
Setting C<readonly> to true will allow various systems to provide additional
139+
optimisations and caching. Note that because C<readonly> is an optimisation
140+
flag, it is off by default and you will need to explicitly enable it.
139141
140-
Returns a C<PPI::Document> object, or C<undef> if parsing fails.
141-
L<PPI::Exception> objects can also be thrown if there are parsing problems.
142+
=head3 feature_mods
143+
144+
Setting feature_mods with a hashref allows defining perl parsing features to be
145+
enabled for the whole document. (e.g. when the code is assumed to be run as a
146+
oneliner)
142147
143148
=cut
144149

@@ -181,25 +186,25 @@ sub new {
181186
my $document = $CACHE->get_document($file_contents);
182187
return $class->_setattr( $document, %attr ) if $document;
183188

184-
$document = PPI::Lexer->lex_source( $$file_contents );
189+
$document = PPI::Lexer->lex_source( $$file_contents, %attr );
185190
if ( $document ) {
186191
# Save in the cache
187192
$CACHE->store_document( $document );
188-
return $class->_setattr( $document, %attr );
193+
return $document;
189194
}
190195
} else {
191-
my $document = PPI::Lexer->lex_file( $source );
196+
my $document = PPI::Lexer->lex_file( $source, %attr );
192197
return $class->_setattr( $document, %attr ) if $document;
193198
}
194199

195200
} elsif ( _SCALAR0($source) ) {
196-
my $document = PPI::Lexer->lex_source( $$source );
197-
return $class->_setattr( $document, %attr ) if $document;
201+
my $document = PPI::Lexer->lex_source( $$source, %attr );
202+
return $document if $document;
198203

199204
} elsif ( _ARRAY0($source) ) {
200205
$source = join '', map { "$_\n" } @$source;
201-
my $document = PPI::Lexer->lex_source( $source );
202-
return $class->_setattr( $document, %attr ) if $document;
206+
my $document = PPI::Lexer->lex_source( $source, %attr );
207+
return $document if $document;
203208

204209
} else {
205210
$class->_error("Unknown object or reference was passed to PPI::Document::new");
@@ -229,6 +234,7 @@ sub _setattr {
229234
my ($class, $document, %attr) = @_;
230235
$document->{readonly} = !! $attr{readonly};
231236
$document->{filename} = $attr{filename};
237+
$document->{feature_mods} = $attr{feature_mods};
232238
return $document;
233239
}
234240

@@ -344,6 +350,16 @@ sub tab_width {
344350
$self->{tab_width} = shift;
345351
}
346352

353+
=head2 feature_mods { feature_name => $enabled }
354+
355+
=cut
356+
357+
sub feature_mods {
358+
my $self = shift;
359+
return $self->{feature_mods} unless @_;
360+
$self->{feature_mods} = shift;
361+
}
362+
347363
=pod
348364
349365
=head2 save

‎lib/PPI/Element.pm

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -467,9 +467,25 @@ sub previous_token {
467467
}
468468
}
469469

470+
sub presumed_features {
471+
my ($self) = @_;
472+
473+
my @feature_mods;
474+
my $walker = $self;
475+
while ($walker) {
476+
my $sib_walk = $walker;
477+
while ($sib_walk) {
478+
push @feature_mods, $sib_walk if $sib_walk->can("feature_mods");
479+
$sib_walk = $sib_walk->sprevious_sibling;
480+
}
481+
$walker = $walker->parent;
482+
}
470483

484+
my %feature_mods = map %{$_}, reverse grep defined, map $_->feature_mods,
485+
@feature_mods;
471486

472-
487+
return \%feature_mods;
488+
}
473489

474490
#####################################################################
475491
# Manipulation

‎lib/PPI/Lexer.pm

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ sub lex_file {
143143
unless ( defined $file ) {
144144
return $self->_error("Did not pass a filename to PPI::Lexer::lex_file");
145145
}
146+
my %args = @_;
146147

147148
# Create the Tokenizer
148149
my $Tokenizer = eval {
@@ -154,7 +155,7 @@ sub lex_file {
154155
return $self->_error( $errstr );
155156
}
156157

157-
$self->lex_tokenizer( $Tokenizer );
158+
$self->lex_tokenizer( $Tokenizer, %args );
158159
}
159160

160161
=pod
@@ -175,6 +176,7 @@ sub lex_source {
175176
unless ( defined $source and not ref $source ) {
176177
return $self->_error("Did not pass a string to PPI::Lexer::lex_source");
177178
}
179+
my %args = @_;
178180

179181
# Create the Tokenizer and hand off to the next method
180182
my $Tokenizer = eval {
@@ -186,7 +188,7 @@ sub lex_source {
186188
return $self->_error( $errstr );
187189
}
188190

189-
$self->lex_tokenizer( $Tokenizer );
191+
$self->lex_tokenizer( $Tokenizer, %args );
190192
}
191193

192194
=pod
@@ -206,9 +208,11 @@ sub lex_tokenizer {
206208
return $self->_error(
207209
"Did not pass a PPI::Tokenizer object to PPI::Lexer::lex_tokenizer"
208210
) unless $Tokenizer;
211+
my %args = @_;
209212

210213
# Create the empty document
211214
my $Document = PPI::Document->new;
215+
ref($Document)->_setattr( $Document, %args ) if keys %args;
212216

213217
# Lex the token stream into the document
214218
$self->{Tokenizer} = $Tokenizer;
@@ -239,7 +243,7 @@ sub _lex_document {
239243

240244
# Start the processing loop
241245
my $Token;
242-
while ( ref($Token = $self->_get_token) ) {
246+
while ( ref($Token = $self->_get_token($Document)) ) {
243247
# Add insignificant tokens directly beneath us
244248
unless ( $Token->significant ) {
245249
$self->_add_element( $Document, $Token );
@@ -264,7 +268,7 @@ sub _lex_document {
264268
# Move the lexing down into the statement
265269
$self->_add_delayed( $Document );
266270
$self->_add_element( $Document, $Statement );
267-
$self->_lex_statement( $Statement );
271+
$self->_lex_statement( $Statement, $Document );
268272

269273
next;
270274
}
@@ -275,7 +279,7 @@ sub _lex_document {
275279
$self->_rollback( $Token );
276280
my $Statement = PPI::Statement->new;
277281
$self->_add_element( $Document, $Statement );
278-
$self->_lex_statement( $Statement );
282+
$self->_lex_statement( $Statement, $Document );
279283
next;
280284
}
281285

@@ -384,7 +388,7 @@ my %STATEMENT_CLASSES = (
384388
);
385389

386390
sub _statement {
387-
my ($self, $Parent, $Token) = @_;
391+
my ($self, $Parent, $Token, $Document) = @_;
388392
# my $self = shift;
389393
# my $Parent = _INSTANCE(shift, 'PPI::Node') or die "Bad param 1";
390394
# my $Token = _INSTANCE(shift, 'PPI::Token') or die "Bad param 2";
@@ -399,7 +403,7 @@ sub _statement {
399403
# Is the next significant token a =>
400404
# Read ahead to the next significant token
401405
my $Next;
402-
while ( $Next = $self->_get_token ) {
406+
while ( $Next = $self->_get_token($Document) ) {
403407
unless ( $Next->significant ) {
404408
push @{$self->{delayed}}, $Next;
405409
# $self->_delay_element( $Next );
@@ -620,7 +624,7 @@ sub _statement {
620624
}
621625

622626
sub _lex_statement {
623-
my ($self, $Statement) = @_;
627+
my ( $self, $Statement, $Document ) = @_;
624628
# my $self = shift;
625629
# my $Statement = _INSTANCE(shift, 'PPI::Statement') or die "Bad param 1";
626630

@@ -631,7 +635,7 @@ sub _lex_statement {
631635

632636
# Begin processing tokens
633637
my $Token;
634-
while ( ref( $Token = $self->_get_token ) ) {
638+
while ( ref( $Token = $self->_get_token($Document) ) ) {
635639
# Delay whitespace and comment tokens
636640
unless ( $Token->significant ) {
637641
push @{$self->{delayed}}, $Token;
@@ -675,12 +679,12 @@ sub _lex_statement {
675679

676680
# Determine the class for the structure and create it
677681
my $method = $RESOLVE{$Token->content};
678-
my $Structure = $self->$method($Statement)->new($Token);
682+
my $Structure = $self->$method( $Statement, $Document )->new($Token);
679683

680684
# Move the lexing down into the Structure
681685
$self->_add_delayed( $Statement );
682686
$self->_add_element( $Statement, $Structure );
683-
$self->_lex_structure( $Structure );
687+
$self->_lex_structure( $Structure, $Document );
684688
}
685689

686690
# Was it an error in the tokenizer?
@@ -1130,7 +1134,7 @@ my @CURLY_LOOKAHEAD_CLASSES = (
11301134
# Given a parent element, and a { token to open a structure, determine
11311135
# the class that the structure should be.
11321136
sub _curly {
1133-
my ($self, $Parent) = @_;
1137+
my ( $self, $Parent, $Document ) = @_;
11341138
# my $self = shift;
11351139
# my $Parent = _INSTANCE(shift, 'PPI::Node') or die "Bad param 1";
11361140

@@ -1230,7 +1234,7 @@ sub _curly {
12301234
my $Next;
12311235
my $position = 0;
12321236
my @delayed;
1233-
while ( $Next = $self->_get_token ) {
1237+
while ( $Next = $self->_get_token($Document) ) {
12341238
unless ( $Next->significant ) {
12351239
push @delayed, $Next;
12361240
next;
@@ -1263,13 +1267,13 @@ sub _curly {
12631267

12641268

12651269
sub _lex_structure {
1266-
my ($self, $Structure) = @_;
1270+
my ( $self, $Structure, $Document ) = @_;
12671271
# my $self = shift;
12681272
# my $Structure = _INSTANCE(shift, 'PPI::Structure') or die "Bad param 1";
12691273

12701274
# Start the processing loop
12711275
my $Token;
1272-
while ( ref($Token = $self->_get_token) ) {
1276+
while ( ref($Token = $self->_get_token($Document)) ) {
12731277
# Is this a direct type token
12741278
unless ( $Token->significant ) {
12751279
push @{$self->{delayed}}, $Token;
@@ -1284,11 +1288,11 @@ sub _lex_structure {
12841288
$self->_add_delayed( $Structure );
12851289

12861290
# Determine the class for the Statement and create it
1287-
my $Statement = $self->_statement($Structure, $Token)->new($Token);
1291+
my $Statement = $self->_statement($Structure, $Token, $Document)->new($Token);
12881292

12891293
# Move the lexing down into the Statement
12901294
$self->_add_element( $Structure, $Statement );
1291-
$self->_lex_statement( $Statement );
1295+
$self->_lex_statement( $Statement, $Document );
12921296

12931297
next;
12941298
}
@@ -1299,7 +1303,7 @@ sub _lex_structure {
12991303
$self->_rollback( $Token );
13001304
my $Statement = PPI::Statement->new;
13011305
$self->_add_element( $Structure, $Statement );
1302-
$self->_lex_statement( $Statement );
1306+
$self->_lex_statement( $Statement, $Document );
13031307
next;
13041308
}
13051309

@@ -1363,7 +1367,7 @@ sub _lex_structure {
13631367

13641368
# Get the next token for processing, handling buffering
13651369
sub _get_token {
1366-
shift(@{$_[0]->{buffer}}) or $_[0]->{Tokenizer}->get_token;
1370+
shift(@{$_[0]->{buffer}}) or $_[0]->{Tokenizer}->get_token($_[1]);
13671371
}
13681372

13691373
# Old long version of the above

‎lib/PPI/Statement/Include.pm

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,38 @@ sub arguments {
236236
return @args;
237237
}
238238

239+
sub feature_mods {
240+
my ($self) = @_;
241+
242+
my %known = ( signatures => 1 );
243+
244+
return if $self->type eq "require";
245+
246+
if ( my $perl_version = $self->version ) {
247+
## crude proof of concept hack due to above
248+
return { signatures => 1 } if $perl_version >= 5.035;
249+
250+
# # tried using feature.pm here, but it is impossible to install
251+
# # future versions of it, so e.g. a 5.20 install cannot know about
252+
# # 5.36 features
253+
# $perl_version = join ".", #
254+
# ( split /\./, $perl_version )[0],
255+
# 0 + ( split /\./, $perl_version )[1];
256+
# my $bundle = $feature::feature_bundle{$perl_version};
257+
# return { map +( $_ => 1 ), %{$bundle} };
258+
}
259+
260+
if ( $self->module eq "feature" ) {
261+
my @features = grep $known{$_}, map $_->literal,
262+
map $_->isa("PPI::Structure::List") ? $_->children : $_,
263+
$self->arguments;
264+
my $on_or_off = $self->type eq "use" ? 1 : 0;
265+
return { map +( $_ => $on_or_off ), @features } if @features;
266+
}
267+
268+
return;
269+
}
270+
239271
1;
240272

241273
=pod

‎lib/PPI/Token.pm

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ use PPI::Token::Separator ();
7070
use PPI::Token::Data ();
7171
use PPI::Token::End ();
7272
use PPI::Token::Prototype ();
73+
use PPI::Token::Signature ();
7374
use PPI::Token::Attribute ();
7475
use PPI::Token::Unknown ();
7576

‎lib/PPI/Token/ArrayIndex.pm

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ our @ISA = "PPI::Token";
3939
# Tokenizer Methods
4040

4141
sub __TOKENIZER__on_char {
42-
my $t = $_[1];
42+
my ( undef, $t, $Document ) = @_;
4343

4444
# Suck in till the end of the arrayindex
4545
pos $t->{line} = $t->{line_cursor};
@@ -49,7 +49,7 @@ sub __TOKENIZER__on_char {
4949
}
5050

5151
# End of token
52-
$t->_finalize_token->__TOKENIZER__on_char( $t );
52+
$t->_finalize_token->__TOKENIZER__on_char( $t, $Document );
5353
}
5454

5555
1;

‎lib/PPI/Token/Attribute.pm

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,13 @@ sub parameters {
8888
# Tokenizer Methods
8989

9090
sub __TOKENIZER__on_char {
91-
my $class = shift;
92-
my $t = shift;
91+
my ( $class, $t, $Document ) = @_;
9392
my $char = substr( $t->{line}, $t->{line_cursor}, 1 );
9493

9594
# Unless this is a '(', we are finished.
9695
unless ( $char eq '(' ) {
9796
# Finalise and recheck
98-
return $t->_finalize_token->__TOKENIZER__on_char( $t );
97+
return $t->_finalize_token->__TOKENIZER__on_char( $t, $Document );
9998
}
10099

101100
# This is a bar(...) style attribute.
@@ -111,7 +110,7 @@ sub __TOKENIZER__on_char {
111110

112111
# Found the end of the attribute
113112
$t->{token}->{content} .= $string;
114-
$t->_finalize_token->__TOKENIZER__on_char( $t );
113+
$t->_finalize_token->__TOKENIZER__on_char( $t, $Document );
115114
}
116115

117116
# Scan for a close braced, and take into account both escaping,

0 commit comments

Comments
 (0)