diff --git a/SampleAgent.js b/SampleAgent.js index af2f2d5..f6968c6 100644 --- a/SampleAgent.js +++ b/SampleAgent.js @@ -37,20 +37,14 @@ async function main() { // }, // }, // ); - // console.log(await scraper.getTweet('id')); - // const tweet = await scraper.getTweetV2('1856441982811529619', { - // expansions: ['attachments.poll_ids'], - // pollFields: ['options', 'end_datetime'], - // }); + // console.log(await scraper.getTweet('1856441982811529619')); + // const tweet = await scraper.getTweetV2('1856441982811529619'); + // console.log({ tweet }); // console.log('tweet', tweet); - // const tweets = await scraper.getTweetsV2( - // ['1856441982811529619', '1856429655215260130'], - // { - // expansions: ['attachments.poll_ids', 'attachments.media_keys'], - // pollFields: ['options', 'end_datetime'], - // mediaFields: ['url', 'preview_image_url'], - // }, - // ); + // const tweets = await scraper.getTweetsV2([ + // '1856441982811529619', + // '1856429655215260130', + // ]); // console.log('tweets', tweets); } diff --git a/package.json b/package.json index ed6a8de..4311a98 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "agent-twitter-client", "description": "A twitter client for agents", + "type": "module", "keywords": [ "x", "twitter", diff --git a/src/scraper.ts b/src/scraper.ts index 76bc546..fae70fc 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -49,7 +49,14 @@ import { } from './tweets'; import { parseTimelineTweetsV2, TimelineV2 } from './timeline-v2'; import { fetchHomeTimeline } from './timeline-home'; -import { TweetV2 } from 'twitter-api-v2'; +import { + TTweetv2Expansion, + TTweetv2MediaField, + TTweetv2PlaceField, + TTweetv2PollField, + TTweetv2TweetField, + TTweetv2UserField, +} from 'twitter-api-v2'; const twUrl = 'https://twitter.com'; const UserTweetsUrl = @@ -557,15 +564,15 @@ export class Scraper { */ async getTweetV2( id: string, - options?: { - expansions?: string[]; - tweetFields?: string[]; - pollFields?: string[]; - mediaFields?: string[]; - userFields?: string[]; - placeFields?: string[]; + options: { + expansions?: TTweetv2Expansion[]; + tweetFields?: TTweetv2TweetField[]; + pollFields?: TTweetv2PollField[]; + mediaFields?: TTweetv2MediaField[]; + userFields?: TTweetv2UserField[]; + placeFields?: TTweetv2PlaceField[]; }, - ): Promise { + ): Promise { return await getTweetV2(id, this.auth, options); } @@ -585,15 +592,15 @@ export class Scraper { */ async getTweetsV2( ids: string[], - options?: { - expansions?: string[]; - tweetFields?: string[]; - pollFields?: string[]; - mediaFields?: string[]; - userFields?: string[]; - placeFields?: string[]; + options: { + expansions?: TTweetv2Expansion[]; + tweetFields?: TTweetv2TweetField[]; + pollFields?: TTweetv2PollField[]; + mediaFields?: TTweetv2MediaField[]; + userFields?: TTweetv2UserField[]; + placeFields?: TTweetv2PlaceField[]; }, - ): Promise { + ): Promise { return await getTweetsV2(ids, this.auth, options); } diff --git a/src/tweets.ts b/src/tweets.ts index a13d6c5..613050c 100644 --- a/src/tweets.ts +++ b/src/tweets.ts @@ -14,8 +14,99 @@ import { getTweetTimeline } from './timeline-async'; import { apiRequestFactory } from './api-data'; import { ListTimeline, parseListTimelineTweets } from './timeline-list'; import { updateCookieJar } from './requests'; -import { TweetV2 } from 'twitter-api-v2'; - +import { + ApiV2Includes, + MediaObjectV2, + PlaceV2, + PollV2, + TTweetv2Expansion, + TTweetv2MediaField, + TTweetv2PlaceField, + TTweetv2PollField, + TTweetv2TweetField, + TTweetv2UserField, + TweetV2, + UserV2, +} from 'twitter-api-v2'; + +const defaultOptions = { + expansions: [ + 'attachments.poll_ids', + 'attachments.media_keys', + 'author_id', + 'referenced_tweets.id', + 'in_reply_to_user_id', + 'edit_history_tweet_ids', + 'geo.place_id', + 'entities.mentions.username', + 'referenced_tweets.id.author_id', + ] as TTweetv2Expansion[], + tweetFields: [ + 'attachments', + 'author_id', + 'context_annotations', + 'conversation_id', + 'created_at', + 'entities', + 'geo', + 'id', + 'in_reply_to_user_id', + 'lang', + 'public_metrics', + 'edit_controls', + 'possibly_sensitive', + 'referenced_tweets', + 'reply_settings', + 'source', + 'text', + 'withheld', + 'note_tweet', + ] as TTweetv2TweetField[], + pollFields: [ + 'duration_minutes', + 'end_datetime', + 'id', + 'options', + 'voting_status', + ] as TTweetv2PollField[], + mediaFields: [ + 'duration_ms', + 'height', + 'media_key', + 'preview_image_url', + 'type', + 'url', + 'width', + 'public_metrics', + 'alt_text', + 'variants', + ] as TTweetv2MediaField[], + userFields: [ + 'created_at', + 'description', + 'entities', + 'id', + 'location', + 'name', + 'profile_image_url', + 'protected', + 'public_metrics', + 'url', + 'username', + 'verified', + 'withheld', + ] as TTweetv2UserField[], + placeFields: [ + 'contained_within', + 'country', + 'country_code', + 'full_name', + 'geo', + 'id', + 'name', + 'place_type', + ] as TTweetv2PlaceField[], +}; export interface Mention { id: string; username?: string; @@ -47,21 +138,18 @@ export interface PlaceRaw { }; } -export interface PollOption { - label: string; - votes?: number; -} - export interface PollData { + id: string; + end_datetime: string; + voting_status: string; + duration_minutes: number; options: PollOption[]; - durationMinutes: number; } -export interface PollResult { - id: string; - options: PollOption[]; - totalVotes: number; - endDateTime: Date; +export interface PollOption { + position: number; + label: string; + votes?: number; } /** @@ -102,6 +190,7 @@ export interface Tweet { videos: Video[]; views?: number; sensitiveContent?: boolean; + poll?: PollV2 | null; } export type TweetQuery = @@ -199,7 +288,7 @@ export async function createCreateTweetRequestV2( text, poll: { options: poll?.options.map((option) => option.label) ?? [], - duration_minutes: poll?.durationMinutes ?? 60, + duration_minutes: poll?.duration_minutes ?? 60, }, }; } else if (tweetId) { @@ -211,8 +300,139 @@ export async function createCreateTweetRequestV2( }; } const tweetResponse = await v2client.v2.tweet(tweetConfig); - // TODO: extract poll results from response - return await getTweet(tweetResponse.data.id, auth); + let optionsConfig = {}; + if (options?.poll) { + optionsConfig = { + expansions: ['attachments.poll_ids'], + pollFields: [ + 'options', + 'duration_minutes', + 'end_datetime', + 'voting_status', + ], + }; + } + return await getTweetV2(tweetResponse.data.id, auth, optionsConfig); +} + +export function parseTweetV2ToV1( + tweetV2: TweetV2, + includes?: ApiV2Includes, + defaultTweetData?: Tweet | null, +): Tweet { + let parsedTweet; + if (defaultTweetData != null) { + parsedTweet = defaultTweetData; + } + parsedTweet = { + id: tweetV2.id, + text: tweetV2.text ?? defaultTweetData?.text ?? '', + hashtags: + tweetV2.entities?.hashtags?.map((tag) => tag.tag) ?? + defaultTweetData?.hashtags ?? + [], + mentions: + tweetV2.entities?.mentions?.map((mention) => ({ + id: mention.id, + username: mention.username, + })) ?? + defaultTweetData?.mentions ?? + [], + urls: + tweetV2.entities?.urls?.map((url) => url.url) ?? + defaultTweetData?.urls ?? + [], + likes: tweetV2.public_metrics?.like_count ?? defaultTweetData?.likes ?? 0, + retweets: + tweetV2.public_metrics?.retweet_count ?? defaultTweetData?.retweets ?? 0, + replies: + tweetV2.public_metrics?.reply_count ?? defaultTweetData?.replies ?? 0, + views: + tweetV2.public_metrics?.impression_count ?? defaultTweetData?.views ?? 0, + userId: tweetV2.author_id ?? defaultTweetData?.userId, + conversationId: tweetV2.conversation_id ?? defaultTweetData?.conversationId, + photos: defaultTweetData?.photos ?? [], + videos: defaultTweetData?.videos ?? [], + poll: defaultTweetData?.poll ?? null, + username: defaultTweetData?.username ?? '', + name: defaultTweetData?.name ?? '', + place: defaultTweetData?.place, + thread: defaultTweetData?.thread ?? [], + }; + + // Process Polls + if (includes?.polls?.length) { + const poll = includes.polls[0]; + parsedTweet.poll = { + id: poll.id, + end_datetime: poll.end_datetime + ? poll.end_datetime + : defaultTweetData?.poll?.end_datetime + ? defaultTweetData?.poll?.end_datetime + : undefined, + options: poll.options.map((option) => ({ + position: option.position, + label: option.label, + votes: option.votes, + })), + voting_status: + poll.voting_status ?? defaultTweetData?.poll?.voting_status, + }; + } + + // Process Media (photos and videos) + if (includes?.media?.length) { + includes.media.forEach((media: MediaObjectV2) => { + if (media.type === 'photo') { + parsedTweet.photos.push({ + id: media.media_key, + url: media.url ?? '', + alt_text: media.alt_text ?? '', + }); + } else if (media.type === 'video' || media.type === 'animated_gif') { + parsedTweet.videos.push({ + id: media.media_key, + preview: media.preview_image_url ?? '', + url: + media.variants?.find( + (variant) => variant.content_type === 'video/mp4', + )?.url ?? '', + }); + } + }); + } + + // Process User (for author info) + if (includes?.users?.length) { + const user = includes.users.find( + (user: UserV2) => user.id === tweetV2.author_id, + ); + if (user) { + parsedTweet.username = user.username ?? defaultTweetData?.username ?? ''; + parsedTweet.name = user.name ?? defaultTweetData?.name ?? ''; + } + } + + // Process Place (if any) + if (tweetV2?.geo?.place_id && includes?.places?.length) { + const place = includes.places.find( + (place: PlaceV2) => place.id === tweetV2?.geo?.place_id, + ); + if (place) { + parsedTweet.place = { + id: place.id, + full_name: place.full_name ?? defaultTweetData?.place?.full_name ?? '', + country: place.country ?? defaultTweetData?.place?.country ?? '', + country_code: + place.country_code ?? defaultTweetData?.place?.country_code ?? '', + name: place.name ?? defaultTweetData?.place?.name ?? '', + place_type: place.place_type ?? defaultTweetData?.place?.place_type, + }; + } + } + + // TODO: Process Thread (referenced tweets) and remove reference to v1 + return parsedTweet; } export async function createCreateTweetRequest( @@ -521,15 +741,15 @@ export async function getTweet( export async function getTweetV2( id: string, auth: TwitterAuth, - options?: { - expansions?: string[]; - tweetFields?: string[]; - pollFields?: string[]; - mediaFields?: string[]; - userFields?: string[]; - placeFields?: string[]; - }, -): Promise { + options: { + expansions?: TTweetv2Expansion[]; + tweetFields?: TTweetv2TweetField[]; + pollFields?: TTweetv2PollField[]; + mediaFields?: TTweetv2MediaField[]; + userFields?: TTweetv2UserField[]; + placeFields?: TTweetv2PlaceField[]; + } = defaultOptions, +): Promise { const v2client = auth.getV2Client(); if (!v2client) { throw new Error('V2 client is not initialized'); @@ -550,7 +770,15 @@ export async function getTweetV2( return null; } - return tweetData.data; + const defaultTweetData = await getTweet(tweetData.data.id, auth); + // Extract primary tweet data + const parsedTweet = parseTweetV2ToV1( + tweetData.data, + tweetData?.includes, + defaultTweetData, + ); + + return parsedTweet; } catch (error) { console.error(`Error fetching tweet ${id}:`, error); return null; @@ -560,15 +788,15 @@ export async function getTweetV2( export async function getTweetsV2( ids: string[], auth: TwitterAuth, - options?: { - expansions?: string[]; - tweetFields?: string[]; - pollFields?: string[]; - mediaFields?: string[]; - userFields?: string[]; - placeFields?: string[]; - }, -): Promise { + options: { + expansions?: TTweetv2Expansion[]; + tweetFields?: TTweetv2TweetField[]; + pollFields?: TTweetv2PollField[]; + mediaFields?: TTweetv2MediaField[]; + userFields?: TTweetv2UserField[]; + placeFields?: TTweetv2PlaceField[]; + } = defaultOptions, +): Promise { const v2client = auth.getV2Client(); if (!v2client) { return []; @@ -583,12 +811,18 @@ export async function getTweetsV2( 'user.fields': options?.userFields, 'place.fields': options?.placeFields, }); - - if (!tweetData?.data) { + const tweetsV2 = tweetData.data; + if (tweetsV2.length === 0) { console.warn(`No tweet data found for IDs: ${ids.join(', ')}`); return []; } - return tweetData?.data; + return ( + await Promise.all( + tweetsV2.map( + async (tweet) => await getTweetV2(tweet.id, auth, options), + ), + ) + ).filter((tweet) => tweet !== null); } catch (error) { console.error(`Error fetching tweets for IDs: ${ids.join(', ')}`, error); return [];