Retrieve Twitter profiles using guest_token (#2462)

README.md

@@ -109,10 +109,6 @@ $ export HOMOCHECKER_DB_SSLCERT=/path/to/sslcert
 $ export HOMOCHECKER_DB_SSLKEY=/path/to/sslkey
 $ export HOMOCHECKER_DB_SSLROOTCERT=/path/to/sslrootcert
 $ export HOMOCHECKER_LOG_LEVEL=debug
-$ export HOMOCHECKER_TWITTER_CONSUMER_KEY=
-$ export HOMOCHECKER_TWITTER_CONSUMER_SECRET=
-$ export HOMOCHECKER_TWITTER_TOKEN=
-$ export HOMOCHECKER_TWITTER_TOKEN_SECRET=
 ```
 
 ### ビルド

api/composer.json

@@ -3,7 +3,6 @@
     "description": "HomoChecker API",
     "require": {
         "guzzlehttp/guzzle": "^7",
-        "guzzlehttp/oauth-subscriber": "^0",
         "illuminate/config": "^10",
         "illuminate/database": "^10",
         "illuminate/events": "^10",

api/src/Providers/HomoProfileServiceProvider.php

@@ -6,7 +6,6 @@
 use GuzzleHttp\Client;
 use GuzzleHttp\ClientInterface;
 use GuzzleHttp\HandlerStack;
-use GuzzleHttp\Subscriber\Oauth\Oauth1;
 use HomoChecker\Http\RequestSigner;
 use HomoChecker\Service\Profile\MastodonProfileService;
 use HomoChecker\Service\Profile\TwitterProfileService;
@@ -28,15 +27,7 @@ public function register()
             return new Client($config);
         });
 
-        $this->app->singleton('twitter.client', function (Container $app) {
-            $handler = HandlerStack::create();
-            $handler->push($app->make('twitter.oauth'));
-
-            $config = $app->make('config')->get('twitter.client');
-            $config['handler'] = $handler;
-
-            return new Client($config);
-        });
+        $this->app->singleton('twitter.client', fn (Container $app) => new Client($app->make('config')->get('twitter.client')));
 
         $this->app->singleton('mastodon.signer', RequestSigner::class);
         $this->app->when(RequestSigner::class)
@@ -49,11 +40,6 @@ public function register()
                 return \file_get_contents($actor['private_key']);
             });
 
-        $this->app->singleton('twitter.oauth', Oauth1::class);
-        $this->app->when(Oauth1::class)
-            ->needs('$config')
-            ->giveConfig('twitter.oauth');
-
         $this->app->when(MastodonProfileService::class)
             ->needs(ClientInterface::class)
             ->give('mastodon.client');
@@ -76,8 +62,9 @@ public function provides()
     {
         return [
             'profiles',
+            'mastodon.client',
+            'mastodon.signer',
             'twitter.client',
-            'twitter.oauth',
         ];
     }
 }

api/src/Service/Profile/TwitterProfileService.php

@@ -13,14 +13,68 @@
 
 class TwitterProfileService implements ProfileServiceContract
 {
-    public const CACHE_EXPIRE = 60 * 60 * 24 * 30;
+    public const CACHE_EXPIRE = 180;
+
+    public const TWITTER_API_GRAPHQL_ROOT = 'https://twitter.com/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/';
+    public const TOKEN = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
+
+    protected ?string $guestToken = null;
 
     public function __construct(
         protected ClientInterface $client,
         protected ProfileRepositoryContract $repository,
         protected Counter $profileErrorCounter,
     ) {}
 
+    protected function getGuestToken(): PromiseInterface
+    {
+        return Coroutine::of(function () {
+            if ($this->guestToken) {
+                return yield $this->guestToken;
+            }
+
+            $target = 'guest/activate.json';
+            $response = yield $this->client->postAsync($target, [
+                'headers' => [
+                    'Authorization' => static::TOKEN,
+                ],
+            ]);
+            $guest = json_decode((string) $response->getBody());
+            if (!$guest->guest_token) {
+                throw new \RuntimeException('Error issuing guest_token');
+            }
+
+            $this->guestToken = $guest->guest_token;
+            return yield $this->guestToken;
+        });
+    }
+
+    protected function generateHeaders(): PromiseInterface
+    {
+        return Coroutine::of(function () {
+            $guestToken = yield $this->getGuestToken();
+            $csrfToken = uniqid();
+            $cookie = implode('; ', [
+                'guest_id=' . urlencode("v1:{$guestToken}"),
+                'ct0=' . urlencode($csrfToken),
+            ]);
+
+            return yield [
+                'Accept' => '*/*',
+                'Authorization' => static::TOKEN,
+                'Content-Type' => 'application/x-www-form-urlencoded',
+                'Cookie' => $cookie,
+                'Dnt' => '1',
+                'Origin' => 'https://twitter.com',
+                'Referer' => 'https://twitter.com',
+                'X-Csrf-Token' => $csrfToken,
+                'X-Guest-Token' => $guestToken,
+                'X-Twitter-Active-User' => 'yes',
+                'X-Twitter-Client-Language' => 'en',
+            ];
+        });
+    }
+
     /**
      * Get the URL of profile image of the user.
      * @param  string           $screen_name The screen_name of the user.
@@ -30,10 +84,26 @@ public function getIconAsync(string $screen_name): PromiseInterface
     {
         return Coroutine::of(function () use ($screen_name) {
             try {
-                $target = "users/show.json?screen_name={$screen_name}";
-                $response = yield $this->client->getAsync($target);
+                $headers = yield $this->generateHeaders();
+                $variables = urlencode(json_encode([
+                    'screen_name' => $screen_name,
+                ]));
+                $features = urlencode(json_encode([
+                    'blue_business_profile_image_shape_enabled' => true,
+                    'responsive_web_graphql_exclude_directive_enabled' => true,
+                    'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => true,
+                    'responsive_web_graphql_timeline_navigation_enabled' => true,
+                    'verified_phone_label_enabled' => true,
+                ]));
+                $target = static::TWITTER_API_GRAPHQL_ROOT . "UserByScreenName?variables={$variables}&features={$features}";
+                $response = yield $this->client->getAsync($target, [
+                    'headers' => $headers,
+                ]);
                 $user = json_decode((string) $response->getBody());
-                $url = str_replace('_normal', '_200x200', $user->profile_image_url_https);
+                if (!isset($user->data->user)) {
+                    throw new \RuntimeException('User not found');
+                }
+                $url = str_replace('_normal', '_200x200', $user->data->user->result->legacy->profile_image_url_https);
 
                 $this->repository->save(
                     $screen_name,

api/src/config.php

@@ -65,13 +65,6 @@
     'twitter.client' => [
         'timeout' => 5,
         'base_uri' => 'https://api.twitter.com/1.1/',
-        'auth' => 'oauth',
-    ],
-    'twitter.oauth' => [
-        'consumer_key' => env('HOMOCHECKER_TWITTER_CONSUMER_KEY'),
-        'consumer_secret' => env('HOMOCHECKER_TWITTER_CONSUMER_SECRET'),
-        'token' => env('HOMOCHECKER_TWITTER_TOKEN'),
-        'token_secret' => env('HOMOCHECKER_TWITTER_TOKEN_SECRET'),
     ],
     'regex' => '/https?:\/\/twitter\.com\/mpyw\/?/',
 ];

api/tests/Case/Service/Profile/TwitterProfileServiceTest.php

@@ -27,25 +27,31 @@ public function testGetIconAsync(): void
         $screen_name = 'example';
         $url = 'https://pbs.twimg.com/profile_images/114514/example_bigger.jpg';
         $handler = HandlerStack::create(new MockHandler([
-            new Response(200, [], "
+            new Response(200, [], '{}'),
+            new Response(200, [], '
                 {
-                    \"id\": 114514,
-                    \"id_str\": \"114514\",
-                    \"name\": \"test\",
-                    \"screen_name\": \"test\",
-                    \"profile_image_url_https\": \"{$url}\"
+                    "guest_token": "1145141919"
                 }
-            "),
-            new Response(404, [], '
+            '),
+            new Response(200, [], '
                 {
-                    "errors": [
-                        {
-                            "code": 50,
-                            "message": "User not found."
+                    "data": {
+                        "user": {
+                            "result": {
+                                "__typename": "User",
+                                "id": "VXNlcjoxMTQ1MTQ=",
+                                "rest_id": "114514",
+                                "legacy": {
+                                    "name": "test",
+                                    "screen_name": "test",
+                                    "profile_image_url_https": "https://pbs.twimg.com/profile_images/114514/example_bigger.jpg"
+                                }
+                            }
                         }
-                    ]
+                    }
                 }
             '),
+            new Response(200, [], '{"data": {}}'),
             new RequestException('Connection problem occurred', new Request('GET', '')),
         ]));
 
@@ -69,8 +75,16 @@ public function testGetIconAsync(): void
 
         $profile = new TwitterProfileService($client, $repository, $profileErrorCounter);
 
+        // (1) Retrieving guest_token fails
+        $this->assertEquals($profile->getDefaultUrl(), $profile->getIconAsync($screen_name)->wait());
+
+        // (2) Retrieving guest_token succeeds and user is retrieved
         $this->assertEquals($url, $profile->getIconAsync($screen_name)->wait());
+
+        // (3) Using the cached guest_token and retrieved user is empty
         $this->assertEquals($profile->getDefaultUrl(), $profile->getIconAsync($screen_name)->wait());
+
+        // (4) Using the cached guest_token and retrieving user fails
         $this->assertEquals($profile->getDefaultUrl(), $profile->getIconAsync($screen_name)->wait());
     }
 }

compose.yaml

@@ -13,10 +13,6 @@ services:
       HOMOCHECKER_DB_USERNAME: homo
       HOMOCHECKER_DB_PASSWORD: homo
       HOMOCHECKER_LOG_LEVEL: debug
-      HOMOCHECKER_TWITTER_CONSUMER_KEY:
-      HOMOCHECKER_TWITTER_CONSUMER_SECRET:
-      HOMOCHECKER_TWITTER_TOKEN:
-      HOMOCHECKER_TWITTER_TOKEN_SECRET:
     configs:
       - source: activity_pub_actor_public_key
         target: activity_pub_actor.pub