From c2100fabe78c6a2a5a592a8c9a5b8364051b5b0c Mon Sep 17 00:00:00 2001 From: Christian Garbs Date: Fri, 12 Nov 2010 21:23:48 +0100 Subject: [PATCH 1/6] don't search tweets from blocked accounts Regularly poll the list of blocked accounts and filter search and subscription results by this list. This is a very close copy of the friends list handling. --- twirssi.pl | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/twirssi.pl b/twirssi.pl index f338386..f39da8f 100644 --- a/twirssi.pl +++ b/twirssi.pl @@ -34,8 +34,10 @@ my $poll; my $last_poll; my $last_friends_poll = 0; +my $last_blocks_poll = 0; my %nicks; my %friends; +my %blocks; my %tweet_cache; my %state; my $failstatus = 0; @@ -554,7 +556,7 @@ sub cmd_login { return; } - %friends = %nicks = (); + %blocks = %friends = %nicks = (); my $service; if ( $user =~ /^(.*)@(twitter|identica)$/ ) { @@ -758,10 +760,13 @@ sub verify_twitter_object { Irssi::timeout_remove($poll) if $poll; $poll = Irssi::timeout_add( &get_poll_time * 1000, \&get_updates, "" ); ¬ice( [ "tweet", "$user\@$service" ], - "Logged in as $user\@$service, loading friends list..." ); + "Logged in as $user\@$service, loading friends list and blocks..." ); &load_friends(); ¬ice( [ "tweet", "$user\@$service" ], "loaded friends: " . scalar keys %friends ); + &load_blocks(); + ¬ice( [ "tweet", "$user\@$service" ], + "loaded blocks: " . scalar keys %blocks ); %nicks = %friends; $nicks{$user} = 0; @@ -1139,6 +1144,48 @@ sub load_friends { return ( $added, $removed ); } +sub load_blocks { + my $fh = shift; + my $page = 1; + my %new_blocks; + eval { + while ( $page < 11 ) + { + print $fh "type:debug Loading blocks page $page...\n" + if ( $fh and &debug ); + my $blocks; + $blocks = $twit->blocking( { page => $page } ); + last unless $blocks; + $new_blocks{ $_->{screen_name} } = time foreach @$blocks; + $page++; + } + }; + + if ($@) { + print $fh "type:debug Error during blocks list update. Aborted.\n" + if $fh; + return; + } + + my ( $added, $removed ) = ( 0, 0 ); + print $fh "type:debug Scanning for new blocks...\n" if ( $fh and &debug ); + foreach ( keys %new_blocks ) { + next if exists $blocks{$_}; + $blocks{$_} = time; + $added++; + } + + print $fh "type:debug Scanning for removed blocks...\n" + if ( $fh and &debug ); + foreach ( keys %blocks ) { + next if exists $new_blocks{$_}; + delete $blocks{$_}; + $removed++; + } + + return ( $added, $removed ); +} + sub get_updates { print scalar localtime, " - get_updates starting" if &debug; @@ -1206,6 +1253,23 @@ sub get_updates { print $fh "$_ $friends{$_}\n"; } + print $fh "__blocks__\n"; + if ( time - $last_blocks_poll > $settings{blocks_poll} ) { + print $fh "__updated ", time, "\n"; + my ( $added, $removed ) = &load_blocks($fh); + if ( $added + $removed ) { + print $fh "type:debug %R***%n Blocks list updated: ", + join( ", ", + sprintf( "%d added", $added ), + sprintf( "%d removed", $removed ) ), + "\n"; + } + } + + foreach ( sort keys %blocks ) { + print $fh "$_ $blocks{$_}\n"; + } + if ($error) { print $fh "type:debug Update encountered errors. Aborted\n"; print $fh "-- $last_poll"; @@ -1421,6 +1485,7 @@ sub do_updates { $search->{max_id}, $username, $topic; foreach my $t ( reverse @{ $search->{results} } ) { + next if exists $blocks{ $t->{from_user} }; my $text = &get_text( $t, $obj ); printf $fh "id:%s account:%s nick:%s type:search topic:%s %s\n", $t->{id}, $username, $t->{from_user}, $topic, $text; @@ -1457,6 +1522,8 @@ sub do_updates { # TODO: consider applying ignore-settings to search results my @results = @{ $search->{results} }; + + @results = grep { not exists $blocks{ $_->{from_user} } } @results; if ( $max_results > 0 ) { splice @results, $max_results; } @@ -1697,12 +1764,25 @@ sub monitor_child { %friends = (); while () { + last if /^__blocks__/; if (/^__updated (\d+)$/) { $last_friends_poll = $1; print "Friend list updated" if &debug; next; } + my ( $f, $t ) = split ' ', $_; + $nicks{$f} = $friends{$f} = $t; + } + + %blocks = (); + while () { + if (/^__updated (\d+)$/) { + $last_blocks_poll = $1; + print "Block list updated" if &debug; + next; + } + if (/^-- (\d+)$/) { $new_last_poll = $1; if ( $new_last_poll >= $last_poll ) { @@ -1715,8 +1795,8 @@ sub monitor_child { next; } } - my ( $f, $t ) = split ' ', $_; - $nicks{$f} = $friends{$f} = $t; + my ( $b, $t ) = split ' ', $_; + $blocks{$b} = $t; } if ($new_last_poll) { @@ -2307,6 +2387,7 @@ sub window_to_account { map { "u: $_->{username}\@" . ref($_) } values %twits; print "selected: $user\@$defservice"; print "friends: ", join ", ", sort keys %friends; + print "blocks: ", join ", ", sort keys %blocks; print "nicks: ", join ", ", sort keys %nicks; print "searches: ", Dumper \%{ $state{__searches} }; print "windows: ", Dumper \%{ $state{__windows} }; From 041300d5f5fdc3129b754605c93f7d0d1b09c978 Mon Sep 17 00:00:00 2001 From: Christian Garbs Date: Mon, 22 Nov 2010 15:59:42 +0100 Subject: [PATCH 2/6] fix block lists for multiple accounts Only the block list for the current account was read. This has been changed to querying all accounts for their block lists. Every block applies globally, e.g. it filters results from _any_ account. Seems reasonable to me because I use blocks to remove unwanted results (spammers etc.) and these results don't become wanted just because another account stumbles upon them. --- twirssi.pl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/twirssi.pl b/twirssi.pl index f39da8f..3df2d66 100644 --- a/twirssi.pl +++ b/twirssi.pl @@ -1146,19 +1146,19 @@ sub load_friends { sub load_blocks { my $fh = shift; - my $page = 1; my %new_blocks; eval { - while ( $page < 11 ) - { - print $fh "type:debug Loading blocks page $page...\n" - if ( $fh and &debug ); - my $blocks; - $blocks = $twit->blocking( { page => $page } ); - last unless $blocks; - $new_blocks{ $_->{screen_name} } = time foreach @$blocks; - $page++; - } + foreach my $t ( keys %twits ) { + foreach my $page (1..10) + { + print $fh "type:debug Loading blocks page $page...\n" + if ( $fh and &debug ); + my $blocks; + $blocks = $twits{$t}->blocking( { page => $page } ); + last unless $blocks; + $new_blocks{ $_->{screen_name} } = time foreach @$blocks; + } + } }; if ($@) { From 8788c3e4bf8fe63852050655660b2043e178714c Mon Sep 17 00:00:00 2001 From: Christian Garbs Date: Mon, 22 Nov 2010 18:17:31 +0100 Subject: [PATCH 3/6] remove paging from load_blocks() Gedge found out that Twitter does not honor the page attribute to the blocks/blocking method which makes us download the same block list 10 times for every account. Talk about The Big Freeze on startup... --- twirssi.pl | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/twirssi.pl b/twirssi.pl index 3df2d66..f6d325c 100644 --- a/twirssi.pl +++ b/twirssi.pl @@ -1149,15 +1149,11 @@ sub load_blocks { my %new_blocks; eval { foreach my $t ( keys %twits ) { - foreach my $page (1..10) - { - print $fh "type:debug Loading blocks page $page...\n" - if ( $fh and &debug ); - my $blocks; - $blocks = $twits{$t}->blocking( { page => $page } ); - last unless $blocks; - $new_blocks{ $_->{screen_name} } = time foreach @$blocks; - } + print $fh "type:debug Loading blocks page ALL...\n" + if ( $fh and &debug ); + my $blocks; + $blocks = $twits{$t}->blocking(); + $new_blocks{ $_->{screen_name} } = time foreach @$blocks; } }; From 1b8118885b5d6e2f63ea45c43b20c6eb11f1584f Mon Sep 17 00:00:00 2001 From: Christian Garbs Date: Mon, 22 Nov 2010 23:03:49 +0100 Subject: [PATCH 4/6] document block list poll intervall The existing twitter_friends_poll setting is used for both friends and blocks. --- html/using.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/html/using.html b/html/using.html index 485d50e..3159de1 100644 --- a/html/using.html +++ b/html/using.html @@ -135,8 +135,8 @@

Settings

requested from twitter (in seconds). Default is every 5 minutes. Keep in mind that twitter will rate limit your user (or IP) if you hit them too often. Hardcoded minimum is one minute. -
  • twitter_friends_poll - How often should the updated friends - list be retrieved from the server. By default, only once every 10 +
  • twitter_friends_poll - How often should the updated friends and block + lists be retrieved from the server. By default, only once every 10 minutes
  • twitter_timeout - How long should we wait before giving up on a twitter operation. Should help avoid irssi pinging out of IRC From c569ad8163e306c59aac742041adeda48cf7f546 Mon Sep 17 00:00:00 2001 From: Christian Garbs Date: Wed, 1 Dec 2010 23:00:58 +0100 Subject: [PATCH 5/6] store and compare blocked accounts in lowercase This should fix a case-related bug that seems to pop up sometime (don't know why, though). Account names are case insensitive so using all lowercase can't hurt. --- twirssi.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/twirssi.pl b/twirssi.pl index f6d325c..13f8c59 100644 --- a/twirssi.pl +++ b/twirssi.pl @@ -1153,7 +1153,7 @@ sub load_blocks { if ( $fh and &debug ); my $blocks; $blocks = $twits{$t}->blocking(); - $new_blocks{ $_->{screen_name} } = time foreach @$blocks; + $new_blocks{ lc $_->{screen_name} } = time foreach @$blocks; } }; @@ -1481,7 +1481,7 @@ sub do_updates { $search->{max_id}, $username, $topic; foreach my $t ( reverse @{ $search->{results} } ) { - next if exists $blocks{ $t->{from_user} }; + next if exists $blocks{ lc $t->{from_user} }; my $text = &get_text( $t, $obj ); printf $fh "id:%s account:%s nick:%s type:search topic:%s %s\n", $t->{id}, $username, $t->{from_user}, $topic, $text; @@ -1519,7 +1519,7 @@ sub do_updates { # TODO: consider applying ignore-settings to search results my @results = @{ $search->{results} }; - @results = grep { not exists $blocks{ $_->{from_user} } } @results; + @results = grep { not exists $blocks{ lc $_->{from_user} } } @results; if ( $max_results > 0 ) { splice @results, $max_results; } From 3d59a8dde41315c56b180ad9d7638a056c0bb479 Mon Sep 17 00:00:00 2001 From: Christian Garbs Date: Fri, 4 Feb 2011 21:29:46 +0100 Subject: [PATCH 6/6] fix block list polling As Gedge pointed out, $settings{blocks_poll} does not exist. Use $settings{friends_poll} instead, as documented in html/using.html. I don't see much sense in polling the friends and blocks lists at different intervals, so just use the same setting for both lists for now. --- twirssi.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twirssi.pl b/twirssi.pl index 88c07d9..9ba7027 100644 --- a/twirssi.pl +++ b/twirssi.pl @@ -1250,7 +1250,7 @@ sub get_updates { } print $fh "__blocks__\n"; - if ( time - $last_blocks_poll > $settings{blocks_poll} ) { + if ( time - $last_blocks_poll > $settings{friends_poll} ) { print $fh "__updated ", time, "\n"; my ( $added, $removed ) = &load_blocks($fh); if ( $added + $removed ) {