proof-of-concept for feature-tracking and perl sub signatures (see #273)

lib/PPI/Document.pm

@@ -128,17 +128,22 @@ In all cases, the document is considered to be "anonymous" and not tied back
 to where it was created from. Specifically, if you create a PPI::Document from
 a filename, the document will B<not> remember where it was created from.
 
+Returns a C<PPI::Document> object, or C<undef> if parsing fails.
+L<PPI::Exception> objects can also be thrown if there are parsing problems.
+
 The constructor also takes attribute flags.
 
-At this time, the only available attribute is the C<readonly> flag.
+=head3 readonly
 
-Setting C<readonly> to true will allow various systems to provide
-additional optimisations and caching. Note that because C<readonly> is an
-optimisation flag, it is off by default and you will need to explicitly
-enable it.
+Setting C<readonly> to true will allow various systems to provide additional
+optimisations and caching. Note that because C<readonly> is an optimisation
+flag, it is off by default and you will need to explicitly enable it.
 
-Returns a C<PPI::Document> object, or C<undef> if parsing fails.
-L<PPI::Exception> objects can also be thrown if there are parsing problems.
+=head3 feature_mods
+
+Setting feature_mods with a hashref allows defining perl parsing features to be
+enabled for the whole document. (e.g. when the code is assumed to be run as a
+oneliner)
 
 =cut
 
@@ -181,25 +186,25 @@ sub new {
 			my $document = $CACHE->get_document($file_contents);
 			return $class->_setattr( $document, %attr ) if $document;
 
-			$document = PPI::Lexer->lex_source( $$file_contents );
+			$document = PPI::Lexer->lex_source( $$file_contents, %attr );
 			if ( $document ) {
 				# Save in the cache
 				$CACHE->store_document( $document );
-				return $class->_setattr( $document, %attr );
+				return $document;
 			}
 		} else {
-			my $document = PPI::Lexer->lex_file( $source );
+			my $document = PPI::Lexer->lex_file( $source, %attr );
 			return $class->_setattr( $document, %attr ) if $document;
 		}
 
 	} elsif ( _SCALAR0($source) ) {
-		my $document = PPI::Lexer->lex_source( $$source );
-		return $class->_setattr( $document, %attr ) if $document;
+		my $document = PPI::Lexer->lex_source( $$source, %attr );
+		return $document if $document;
 
 	} elsif ( _ARRAY0($source) ) {
 		$source = join '', map { "$_\n" } @$source;
-		my $document = PPI::Lexer->lex_source( $source );
-		return $class->_setattr( $document, %attr ) if $document;
+		my $document = PPI::Lexer->lex_source( $source, %attr );
+		return $document if $document;
 
 	} else {
 		$class->_error("Unknown object or reference was passed to PPI::Document::new");
@@ -229,6 +234,7 @@ sub _setattr {
 	my ($class, $document, %attr) = @_;
 	$document->{readonly} = !! $attr{readonly};
 	$document->{filename} = $attr{filename};
+	$document->{feature_mods} = $attr{feature_mods};
 	return $document;
 }
 
@@ -344,6 +350,16 @@ sub tab_width {
 	$self->{tab_width} = shift;
 }
 
+=head2 feature_mods { feature_name => $enabled }
+
+=cut
+
+sub feature_mods {
+	my $self = shift;
+	return $self->{feature_mods} unless @_;
+	$self->{feature_mods} = shift;
+}
+
 =pod
 
 =head2 save

lib/PPI/Element.pm

@@ -467,9 +467,25 @@ sub previous_token {
 	}
 }
 
+sub presumed_features {
+	my ($self) = @_;
+
+	my @feature_mods;
+	my $walker = $self;
+	while ($walker) {
+		my $sib_walk = $walker;
+		while ($sib_walk) {
+			push @feature_mods, $sib_walk if $sib_walk->can("feature_mods");
+			$sib_walk = $sib_walk->sprevious_sibling;
+		}
+		$walker = $walker->parent;
+	}
 
+	my %feature_mods = map %{$_}, reverse grep defined, map $_->feature_mods,
+	  @feature_mods;
 
-
+	return \%feature_mods;
+}
 
 #####################################################################
 # Manipulation

lib/PPI/Lexer.pm

@@ -143,6 +143,7 @@ sub lex_file {
 	unless ( defined $file ) {
 		return $self->_error("Did not pass a filename to PPI::Lexer::lex_file");
 	}
+	my %args = @_;
 
 	# Create the Tokenizer
 	my $Tokenizer = eval {
@@ -154,7 +155,7 @@ sub lex_file {
 		return $self->_error( $errstr );
 	}
 
-	$self->lex_tokenizer( $Tokenizer );
+	$self->lex_tokenizer( $Tokenizer, %args );
 }
 
 =pod
@@ -175,6 +176,7 @@ sub lex_source {
 	unless ( defined $source and not ref $source ) {
 		return $self->_error("Did not pass a string to PPI::Lexer::lex_source");
 	}
+	my %args = @_;
 
 	# Create the Tokenizer and hand off to the next method
 	my $Tokenizer = eval {
@@ -186,7 +188,7 @@ sub lex_source {
 		return $self->_error( $errstr );
 	}
 
-	$self->lex_tokenizer( $Tokenizer );
+	$self->lex_tokenizer( $Tokenizer, %args );
 }
 
 =pod
@@ -206,9 +208,11 @@ sub lex_tokenizer {
 	return $self->_error(
 		"Did not pass a PPI::Tokenizer object to PPI::Lexer::lex_tokenizer"
 	) unless $Tokenizer;
+	my %args = @_;
 
 	# Create the empty document
 	my $Document = PPI::Document->new;
+	ref($Document)->_setattr( $Document, %args ) if keys %args;
 
 	# Lex the token stream into the document
 	$self->{Tokenizer} = $Tokenizer;
@@ -239,7 +243,7 @@ sub _lex_document {
 
 	# Start the processing loop
 	my $Token;
-	while ( ref($Token = $self->_get_token) ) {
+	while ( ref($Token = $self->_get_token($Document)) ) {
 		# Add insignificant tokens directly beneath us
 		unless ( $Token->significant ) {
 			$self->_add_element( $Document, $Token );
@@ -264,7 +268,7 @@ sub _lex_document {
 			# Move the lexing down into the statement
 			$self->_add_delayed( $Document );
 			$self->_add_element( $Document, $Statement );
-			$self->_lex_statement( $Statement );
+			$self->_lex_statement( $Statement, $Document );
 
 			next;
 		}
@@ -275,7 +279,7 @@ sub _lex_document {
 			$self->_rollback( $Token );
 			my $Statement = PPI::Statement->new;
 			$self->_add_element( $Document, $Statement );
-			$self->_lex_statement( $Statement );
+			$self->_lex_statement( $Statement, $Document );
 			next;
 		}
 
@@ -384,7 +388,7 @@ my %STATEMENT_CLASSES = (
 );
 
 sub _statement {
-	my ($self, $Parent, $Token) = @_;
+	my ($self, $Parent, $Token, $Document) = @_;
 	# my $self   = shift;
 	# my $Parent = _INSTANCE(shift, 'PPI::Node')  or die "Bad param 1";
 	# my $Token  = _INSTANCE(shift, 'PPI::Token') or die "Bad param 2";
@@ -399,7 +403,7 @@ sub _statement {
 			# Is the next significant token a =>
 			# Read ahead to the next significant token
 			my $Next;
-			while ( $Next = $self->_get_token ) {
+			while ( $Next = $self->_get_token($Document) ) {
 				unless ( $Next->significant ) {
 					push @{$self->{delayed}}, $Next;
 					# $self->_delay_element( $Next );
@@ -620,7 +624,7 @@ sub _statement {
 }
 
 sub _lex_statement {
-	my ($self, $Statement) = @_;
+	my ( $self, $Statement, $Document ) = @_;
 	# my $self      = shift;
 	# my $Statement = _INSTANCE(shift, 'PPI::Statement') or die "Bad param 1";
 
@@ -631,7 +635,7 @@ sub _lex_statement {
 
 	# Begin processing tokens
 	my $Token;
-	while ( ref( $Token = $self->_get_token ) ) {
+	while ( ref( $Token = $self->_get_token($Document) ) ) {
 		# Delay whitespace and comment tokens
 		unless ( $Token->significant ) {
 			push @{$self->{delayed}}, $Token;
@@ -675,12 +679,12 @@ sub _lex_statement {
 
 		# Determine the class for the structure and create it
 		my $method    = $RESOLVE{$Token->content};
-		my $Structure = $self->$method($Statement)->new($Token);
+		my $Structure = $self->$method( $Statement, $Document )->new($Token);
 
 		# Move the lexing down into the Structure
 		$self->_add_delayed( $Statement );
 		$self->_add_element( $Statement, $Structure );
-		$self->_lex_structure( $Structure );
+		$self->_lex_structure( $Structure, $Document );
 	}
 
 	# Was it an error in the tokenizer?
@@ -1130,7 +1134,7 @@ my @CURLY_LOOKAHEAD_CLASSES = (
 # Given a parent element, and a { token to open a structure, determine
 # the class that the structure should be.
 sub _curly {
-	my ($self, $Parent) = @_;
+	my ( $self, $Parent, $Document ) = @_;
 	# my $self   = shift;
 	# my $Parent = _INSTANCE(shift, 'PPI::Node') or die "Bad param 1";
 
@@ -1230,7 +1234,7 @@ sub _curly {
 	my $Next;
 	my $position = 0;
 	my @delayed;
-	while ( $Next = $self->_get_token ) {
+	while ( $Next = $self->_get_token($Document) ) {
 		unless ( $Next->significant ) {
 			push @delayed, $Next;
 			next;
@@ -1263,13 +1267,13 @@ sub _curly {
 
 
 sub _lex_structure {
-	my ($self, $Structure) = @_;
+	my ( $self, $Structure, $Document ) = @_;
 	# my $self      = shift;
 	# my $Structure = _INSTANCE(shift, 'PPI::Structure') or die "Bad param 1";
 
 	# Start the processing loop
 	my $Token;
-	while ( ref($Token = $self->_get_token) ) {
+	while ( ref($Token = $self->_get_token($Document)) ) {
 		# Is this a direct type token
 		unless ( $Token->significant ) {
 			push @{$self->{delayed}}, $Token;
@@ -1284,11 +1288,11 @@ sub _lex_structure {
 			$self->_add_delayed( $Structure );
 
 			# Determine the class for the Statement and create it
-			my $Statement = $self->_statement($Structure, $Token)->new($Token);
+			my $Statement = $self->_statement($Structure, $Token, $Document)->new($Token);
 
 			# Move the lexing down into the Statement
 			$self->_add_element( $Structure, $Statement );
-			$self->_lex_statement( $Statement );
+			$self->_lex_statement( $Statement, $Document );
 
 			next;
 		}
@@ -1299,7 +1303,7 @@ sub _lex_structure {
 			$self->_rollback( $Token );
 			my $Statement = PPI::Statement->new;
 			$self->_add_element( $Structure, $Statement );
-			$self->_lex_statement( $Statement );
+			$self->_lex_statement( $Statement, $Document );
 			next;
 		}
 
@@ -1363,7 +1367,7 @@ sub _lex_structure {
 
 # Get the next token for processing, handling buffering
 sub _get_token {
-	shift(@{$_[0]->{buffer}}) or $_[0]->{Tokenizer}->get_token;
+	shift(@{$_[0]->{buffer}}) or $_[0]->{Tokenizer}->get_token($_[1]);
 }
 
 # Old long version of the above

lib/PPI/Statement/Include.pm

@@ -236,6 +236,38 @@ sub arguments {
 	return @args;
 }
 
+sub feature_mods {
+	my ($self) = @_;
+
+	my %known = ( signatures => 1 );
+
+	return if $self->type eq "require";
+
+	if ( my $perl_version = $self->version ) {
+		## crude proof of concept hack due to above
+		return { signatures => 1 } if $perl_version >= 5.035;
+
+		# # tried using feature.pm here, but it is impossible to install
+		# # future versions of it, so e.g. a 5.20 install cannot know about
+		# # 5.36 features
+		# $perl_version = join ".",    #
+		#   ( split /\./, $perl_version )[0],
+		#   0 + ( split /\./, $perl_version )[1];
+		# my $bundle = $feature::feature_bundle{$perl_version};
+		# return { map +( $_ => 1 ), %{$bundle} };
+	}
+
+	if ( $self->module eq "feature" ) {
+		my @features = grep $known{$_}, map $_->literal,
+		  map $_->isa("PPI::Structure::List") ? $_->children : $_,
+		  $self->arguments;
+		my $on_or_off = $self->type eq "use" ? 1 : 0;
+		return { map +( $_ => $on_or_off ), @features } if @features;
+	}
+
+	return;
+}
+
 1;
 
 =pod

lib/PPI/Token.pm

@@ -70,6 +70,7 @@ use PPI::Token::Separator             ();
 use PPI::Token::Data                  ();
 use PPI::Token::End                   ();
 use PPI::Token::Prototype             ();
+use PPI::Token::Signature             ();
 use PPI::Token::Attribute             ();
 use PPI::Token::Unknown               ();
 

lib/PPI/Token/ArrayIndex.pm

@@ -39,7 +39,7 @@ our @ISA = "PPI::Token";
 # Tokenizer Methods
 
 sub __TOKENIZER__on_char {
-	my $t = $_[1];
+	my ( undef, $t, $Document ) = @_;
 
 	# Suck in till the end of the arrayindex
 	pos $t->{line} = $t->{line_cursor};
@@ -49,7 +49,7 @@ sub __TOKENIZER__on_char {
 	}
 
 	# End of token
-	$t->_finalize_token->__TOKENIZER__on_char( $t );
+	$t->_finalize_token->__TOKENIZER__on_char( $t, $Document );
 }
 
 1;

lib/PPI/Token/Attribute.pm

@@ -88,14 +88,13 @@ sub parameters {
 # Tokenizer Methods
 
 sub __TOKENIZER__on_char {
-	my $class = shift;
-	my $t     = shift;
+	my ( $class, $t, $Document ) = @_;
 	my $char  = substr( $t->{line}, $t->{line_cursor}, 1 );
 
 	# Unless this is a '(', we are finished.
 	unless ( $char eq '(' ) {
 		# Finalise and recheck
-		return $t->_finalize_token->__TOKENIZER__on_char( $t );
+		return $t->_finalize_token->__TOKENIZER__on_char( $t, $Document );
 	}
 
 	# This is a bar(...) style attribute.
@@ -111,7 +110,7 @@ sub __TOKENIZER__on_char {
 
 	# Found the end of the attribute
 	$t->{token}->{content} .= $string;
-	$t->_finalize_token->__TOKENIZER__on_char( $t );
+	$t->_finalize_token->__TOKENIZER__on_char( $t, $Document );
 }
 
 # Scan for a close braced, and take into account both escaping,