From e75b7a4ea9d43b6e3b0ca4fbe1d822ae7c50647b Mon Sep 17 00:00:00 2001 From: Chitoku Date: Sun, 19 Nov 2023 15:11:01 +0900 Subject: [PATCH] Retrieve Twitter profiles using guest_token (#2462) --- README.md | 4 - api/composer.json | 1 - api/composer.lock | 61 +-------------- .../Providers/HomoProfileServiceProvider.php | 19 +---- .../Service/Profile/TwitterProfileService.php | 78 ++++++++++++++++++- api/src/config.php | 7 -- .../Profile/TwitterProfileServiceTest.php | 40 ++++++---- compose.yaml | 4 - 8 files changed, 105 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 00d2a43b9..539199bff 100644 --- a/README.md +++ b/README.md @@ -109,10 +109,6 @@ $ export HOMOCHECKER_DB_SSLCERT=/path/to/sslcert $ export HOMOCHECKER_DB_SSLKEY=/path/to/sslkey $ export HOMOCHECKER_DB_SSLROOTCERT=/path/to/sslrootcert $ export HOMOCHECKER_LOG_LEVEL=debug -$ export HOMOCHECKER_TWITTER_CONSUMER_KEY= -$ export HOMOCHECKER_TWITTER_CONSUMER_SECRET= -$ export HOMOCHECKER_TWITTER_TOKEN= -$ export HOMOCHECKER_TWITTER_TOKEN_SECRET= ``` ### ビルド diff --git a/api/composer.json b/api/composer.json index 967c099e1..e55d6bdd1 100644 --- a/api/composer.json +++ b/api/composer.json @@ -3,7 +3,6 @@ "description": "HomoChecker API", "require": { "guzzlehttp/guzzle": "^7", - "guzzlehttp/oauth-subscriber": "^0", "illuminate/config": "^10", "illuminate/database": "^10", "illuminate/events": "^10", diff --git a/api/composer.lock b/api/composer.lock index cc4855815..5a5e87474 100644 --- a/api/composer.lock +++ b/api/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "049f93acc5e8d7158c5ec916d82cad3a", + "content-hash": "e2992ab4c29d04dd2e4dddc8781853c7", "packages": [ { "name": "brick/math", @@ -396,65 +396,6 @@ ], "time": "2023-08-27T10:20:53+00:00" }, - { - "name": "guzzlehttp/oauth-subscriber", - "version": "0.6.0", - "source": { - "type": "git", - "url": "https://github.com/guzzle/oauth-subscriber.git", - "reference": "8d6cab29f8397e5712d00a383eeead36108a3c1f" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/guzzle/oauth-subscriber/zipball/8d6cab29f8397e5712d00a383eeead36108a3c1f", - "reference": "8d6cab29f8397e5712d00a383eeead36108a3c1f", - "shasum": "" - }, - "require": { - "guzzlehttp/guzzle": "^6.5|^7.2", - "guzzlehttp/psr7": "^1.7|^2.0", - "php": ">=5.5.0" - }, - "require-dev": { - "phpunit/phpunit": "~4.0|^9.3.3" - }, - "suggest": { - "ext-openssl": "Required to sign using RSA-SHA1" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "0.6-dev" - } - }, - "autoload": { - "psr-4": { - "GuzzleHttp\\Subscriber\\Oauth\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Michael Dowling", - "email": "mtdowling@gmail.com", - "homepage": "https://github.com/mtdowling" - } - ], - "description": "Guzzle OAuth 1.0 subscriber", - "homepage": "http://guzzlephp.org/", - "keywords": [ - "Guzzle", - "oauth" - ], - "support": { - "issues": "https://github.com/guzzle/oauth-subscriber/issues", - "source": "https://github.com/guzzle/oauth-subscriber/tree/0.6.0" - }, - "time": "2021-07-13T12:01:32+00:00" - }, { "name": "guzzlehttp/promises", "version": "2.0.1", diff --git a/api/src/Providers/HomoProfileServiceProvider.php b/api/src/Providers/HomoProfileServiceProvider.php index fc6342e06..aa64eb467 100644 --- a/api/src/Providers/HomoProfileServiceProvider.php +++ b/api/src/Providers/HomoProfileServiceProvider.php @@ -6,7 +6,6 @@ use GuzzleHttp\Client; use GuzzleHttp\ClientInterface; use GuzzleHttp\HandlerStack; -use GuzzleHttp\Subscriber\Oauth\Oauth1; use HomoChecker\Http\RequestSigner; use HomoChecker\Service\Profile\MastodonProfileService; use HomoChecker\Service\Profile\TwitterProfileService; @@ -28,15 +27,7 @@ public function register() return new Client($config); }); - $this->app->singleton('twitter.client', function (Container $app) { - $handler = HandlerStack::create(); - $handler->push($app->make('twitter.oauth')); - - $config = $app->make('config')->get('twitter.client'); - $config['handler'] = $handler; - - return new Client($config); - }); + $this->app->singleton('twitter.client', fn (Container $app) => new Client($app->make('config')->get('twitter.client'))); $this->app->singleton('mastodon.signer', RequestSigner::class); $this->app->when(RequestSigner::class) @@ -49,11 +40,6 @@ public function register() return \file_get_contents($actor['private_key']); }); - $this->app->singleton('twitter.oauth', Oauth1::class); - $this->app->when(Oauth1::class) - ->needs('$config') - ->giveConfig('twitter.oauth'); - $this->app->when(MastodonProfileService::class) ->needs(ClientInterface::class) ->give('mastodon.client'); @@ -76,8 +62,9 @@ public function provides() { return [ 'profiles', + 'mastodon.client', + 'mastodon.signer', 'twitter.client', - 'twitter.oauth', ]; } } diff --git a/api/src/Service/Profile/TwitterProfileService.php b/api/src/Service/Profile/TwitterProfileService.php index eff2ba534..7cac6169b 100644 --- a/api/src/Service/Profile/TwitterProfileService.php +++ b/api/src/Service/Profile/TwitterProfileService.php @@ -13,7 +13,12 @@ class TwitterProfileService implements ProfileServiceContract { - public const CACHE_EXPIRE = 60 * 60 * 24 * 30; + public const CACHE_EXPIRE = 180; + + public const TWITTER_API_GRAPHQL_ROOT = 'https://twitter.com/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/'; + public const TOKEN = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; + + protected ?string $guestToken = null; public function __construct( protected ClientInterface $client, @@ -21,6 +26,55 @@ public function __construct( protected Counter $profileErrorCounter, ) {} + protected function getGuestToken(): PromiseInterface + { + return Coroutine::of(function () { + if ($this->guestToken) { + return yield $this->guestToken; + } + + $target = 'guest/activate.json'; + $response = yield $this->client->postAsync($target, [ + 'headers' => [ + 'Authorization' => static::TOKEN, + ], + ]); + $guest = json_decode((string) $response->getBody()); + if (!$guest->guest_token) { + throw new \RuntimeException('Error issuing guest_token'); + } + + $this->guestToken = $guest->guest_token; + return yield $this->guestToken; + }); + } + + protected function generateHeaders(): PromiseInterface + { + return Coroutine::of(function () { + $guestToken = yield $this->getGuestToken(); + $csrfToken = uniqid(); + $cookie = implode('; ', [ + 'guest_id=' . urlencode("v1:{$guestToken}"), + 'ct0=' . urlencode($csrfToken), + ]); + + return yield [ + 'Accept' => '*/*', + 'Authorization' => static::TOKEN, + 'Content-Type' => 'application/x-www-form-urlencoded', + 'Cookie' => $cookie, + 'Dnt' => '1', + 'Origin' => 'https://twitter.com', + 'Referer' => 'https://twitter.com', + 'X-Csrf-Token' => $csrfToken, + 'X-Guest-Token' => $guestToken, + 'X-Twitter-Active-User' => 'yes', + 'X-Twitter-Client-Language' => 'en', + ]; + }); + } + /** * Get the URL of profile image of the user. * @param string $screen_name The screen_name of the user. @@ -30,10 +84,26 @@ public function getIconAsync(string $screen_name): PromiseInterface { return Coroutine::of(function () use ($screen_name) { try { - $target = "users/show.json?screen_name={$screen_name}"; - $response = yield $this->client->getAsync($target); + $headers = yield $this->generateHeaders(); + $variables = urlencode(json_encode([ + 'screen_name' => $screen_name, + ])); + $features = urlencode(json_encode([ + 'blue_business_profile_image_shape_enabled' => true, + 'responsive_web_graphql_exclude_directive_enabled' => true, + 'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => true, + 'responsive_web_graphql_timeline_navigation_enabled' => true, + 'verified_phone_label_enabled' => true, + ])); + $target = static::TWITTER_API_GRAPHQL_ROOT . "UserByScreenName?variables={$variables}&features={$features}"; + $response = yield $this->client->getAsync($target, [ + 'headers' => $headers, + ]); $user = json_decode((string) $response->getBody()); - $url = str_replace('_normal', '_200x200', $user->profile_image_url_https); + if (!isset($user->data->user)) { + throw new \RuntimeException('User not found'); + } + $url = str_replace('_normal', '_200x200', $user->data->user->result->legacy->profile_image_url_https); $this->repository->save( $screen_name, diff --git a/api/src/config.php b/api/src/config.php index 05e318db3..2a911bd05 100644 --- a/api/src/config.php +++ b/api/src/config.php @@ -65,13 +65,6 @@ 'twitter.client' => [ 'timeout' => 5, 'base_uri' => 'https://api.twitter.com/1.1/', - 'auth' => 'oauth', - ], - 'twitter.oauth' => [ - 'consumer_key' => env('HOMOCHECKER_TWITTER_CONSUMER_KEY'), - 'consumer_secret' => env('HOMOCHECKER_TWITTER_CONSUMER_SECRET'), - 'token' => env('HOMOCHECKER_TWITTER_TOKEN'), - 'token_secret' => env('HOMOCHECKER_TWITTER_TOKEN_SECRET'), ], 'regex' => '/https?:\/\/twitter\.com\/mpyw\/?/', ]; diff --git a/api/tests/Case/Service/Profile/TwitterProfileServiceTest.php b/api/tests/Case/Service/Profile/TwitterProfileServiceTest.php index 7067f23a1..746e3b4e7 100644 --- a/api/tests/Case/Service/Profile/TwitterProfileServiceTest.php +++ b/api/tests/Case/Service/Profile/TwitterProfileServiceTest.php @@ -27,25 +27,31 @@ public function testGetIconAsync(): void $screen_name = 'example'; $url = 'https://pbs.twimg.com/profile_images/114514/example_bigger.jpg'; $handler = HandlerStack::create(new MockHandler([ - new Response(200, [], " + new Response(200, [], '{}'), + new Response(200, [], ' { - \"id\": 114514, - \"id_str\": \"114514\", - \"name\": \"test\", - \"screen_name\": \"test\", - \"profile_image_url_https\": \"{$url}\" + "guest_token": "1145141919" } - "), - new Response(404, [], ' + '), + new Response(200, [], ' { - "errors": [ - { - "code": 50, - "message": "User not found." + "data": { + "user": { + "result": { + "__typename": "User", + "id": "VXNlcjoxMTQ1MTQ=", + "rest_id": "114514", + "legacy": { + "name": "test", + "screen_name": "test", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/114514/example_bigger.jpg" + } + } } - ] + } } '), + new Response(200, [], '{"data": {}}'), new RequestException('Connection problem occurred', new Request('GET', '')), ])); @@ -69,8 +75,16 @@ public function testGetIconAsync(): void $profile = new TwitterProfileService($client, $repository, $profileErrorCounter); + // (1) Retrieving guest_token fails + $this->assertEquals($profile->getDefaultUrl(), $profile->getIconAsync($screen_name)->wait()); + + // (2) Retrieving guest_token succeeds and user is retrieved $this->assertEquals($url, $profile->getIconAsync($screen_name)->wait()); + + // (3) Using the cached guest_token and retrieved user is empty $this->assertEquals($profile->getDefaultUrl(), $profile->getIconAsync($screen_name)->wait()); + + // (4) Using the cached guest_token and retrieving user fails $this->assertEquals($profile->getDefaultUrl(), $profile->getIconAsync($screen_name)->wait()); } } diff --git a/compose.yaml b/compose.yaml index e76196a39..83b4b8fa9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -13,10 +13,6 @@ services: HOMOCHECKER_DB_USERNAME: homo HOMOCHECKER_DB_PASSWORD: homo HOMOCHECKER_LOG_LEVEL: debug - HOMOCHECKER_TWITTER_CONSUMER_KEY: - HOMOCHECKER_TWITTER_CONSUMER_SECRET: - HOMOCHECKER_TWITTER_TOKEN: - HOMOCHECKER_TWITTER_TOKEN_SECRET: configs: - source: activity_pub_actor_public_key target: activity_pub_actor.pub