From 19adc658c317c71e671201385350343add6275c8 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 2 Feb 2023 12:31:49 +0100 Subject: [PATCH] Replace tweet endpoint with GraphQL --- src/api.nim | 15 ++++-- src/consts.nim | 32 ++++++++++++ src/parser.nim | 120 ++++++++++++++++++++++++-------------------- src/parserutils.nim | 4 -- src/tokens.nim | 3 +- src/types.nim | 2 +- src/views/tweet.nim | 2 +- 7 files changed, 111 insertions(+), 67 deletions(-) diff --git a/src/api.nim b/src/api.nim index dd94b69..dfcf413 100644 --- a/src/api.nim +++ b/src/api.nim @@ -101,16 +101,21 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = except InternalError: return Result[T](beginning: true, query: query) -proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} = - let url = tweet / (id & ".json") ? genParams(cursor=after) - result = parseConversation(await fetch(url, Api.tweet), id) +proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = tweetVariables % [id, cursor] + params = {"variables": variables, "features": tweetFeatures} + js = await fetch(graphTweet ? params, Api.tweetDetail) + result = parseGraphConversation(js, id) proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} = - result = (await getTweetImpl(id, after)).replies + result = (await getGraphTweet(id, after)).replies result.beginning = after.len == 0 proc getTweet*(id: string; after=""): Future[Conversation] {.async.} = - result = await getTweetImpl(id) + result = await getGraphTweet(id) if after.len > 0: result.replies = await getReplies(id, after) diff --git a/src/consts.nim b/src/consts.nim index 2d3ea56..18ecb2c 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -19,6 +19,7 @@ const tweet* = timelineApi / "conversation" graphql = api / "graphql" + graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail" graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName" graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId" graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List" @@ -58,3 +59,34 @@ const ## user: "result_filter: user" ## photos: "result_filter: photos" ## videos: "result_filter: videos" + + tweetVariables* = """{ + "focalTweetId": "$1", + $2 + "includePromotedContent": false, + "withBirdwatchNotes": false, + "withDownvotePerspective": false, + "withReactionsMetadata": false, + "withReactionsPerspective": false, + "withSuperFollowsTweetFields": false, + "withSuperFollowsUserFields": false, + "withVoice": false, + "withV2Timeline": true +}""" + + tweetFeatures* = """{ + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false, + "responsive_web_graphql_timeline_navigation_enabled": false, + "standardized_nudges_misinfo": false, + "verified_phone_label_enabled": false, + "responsive_web_twitter_blue_verified_badge_is_enabled": false, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false, + "view_counts_everywhere_api_enabled": false, + "responsive_web_edit_tweet_api_enabled": false, + "tweetypie_unmention_optimization_enabled": false, + "vibe_api_enabled": false, + "longform_notetweets_consumption_enabled": false, + "responsive_web_text_conversations_enabled": false, + "responsive_web_enhance_cards_enabled": false, + "interactive_text_enabled": false +}""" diff --git a/src/parser.nim b/src/parser.nim index 2b0c6a4..544f943 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -72,8 +72,8 @@ proc parseGif(js: JsonNode): Gif = proc parseVideo(js: JsonNode): Video = result = Video( thumb: js{"media_url_https"}.getImageStr, - views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr, - available: js{"ext_media_availability", "status"}.getStr == "available", + views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt), + available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available", title: js{"ext_alt_text"}.getStr, durationMs: js{"video_info", "duration_millis"}.getInt # playbackType: mp4 @@ -185,7 +185,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card = result.url.len == 0 or result.url.startsWith("card://"): result.url = getPicUrl(result.image) -proc parseTweet(js: JsonNode): Tweet = +proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = if js.isNull: return result = Tweet( id: js{"id_str"}.getId, @@ -193,7 +193,6 @@ proc parseTweet(js: JsonNode): Tweet = replyId: js{"in_reply_to_status_id_str"}.getId, text: js{"full_text"}.getStr, time: js{"created_at"}.getTime, - source: getSource(js), hasThread: js{"self_thread"}.notNull, available: true, user: User(id: js{"user_id_str"}.getStr), @@ -218,7 +217,7 @@ proc parseTweet(js: JsonNode): Tweet = result.retweet = some Tweet(id: rt.getId) return - with jsCard, js{"card"}: + if jsCard.kind != JNull: let name = jsCard{"name"}.getStr if "poll" in name: if "image" in name: @@ -295,64 +294,17 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects = result.users[k] = parseUser(v, k) for k, v in tweets: - var tweet = parseTweet(v) + var tweet = parseTweet(v, v{"card"}) if tweet.user.id in result.users: tweet.user = result.users[tweet.user.id] result.tweets[k] = tweet -proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] = - result.thread = Chain() - - let thread = js{"content", "item", "content", "conversationThread"} - with cursor, thread{"showMoreCursor"}: - result.thread.cursor = cursor{"value"}.getStr - result.thread.hasMore = true - - for t in thread{"conversationComponents"}: - let content = t{"conversationTweetComponent", "tweet"} - - if content{"displayType"}.getStr == "SelfThread": - result.self = true - - var tweet = finalizeTweet(global, content{"id"}.getStr) - if not tweet.available: - tweet.tombstone = getTombstone(content{"tombstone"}) - result.thread.content.add tweet - -proc parseConversation*(js: JsonNode; tweetId: string): Conversation = - result = Conversation(replies: Result[Chain](beginning: true)) - let global = parseGlobalObjects(? js) - - let instructions = ? js{"timeline", "instructions"} - if instructions.len == 0: - return - - for e in instructions[0]{"addEntries", "entries"}: - let entry = e{"entryId"}.getStr - if "tweet" in entry or "tombstone" in entry: - let tweet = finalizeTweet(global, e.getEntryId) - if $tweet.id != tweetId: - result.before.content.add tweet - else: - result.tweet = tweet - elif "conversationThread" in entry: - let (thread, self) = parseThread(e, global) - if thread.content.len > 0: - if self: - result.after = thread - else: - result.replies.content.add thread - elif "cursor-showMore" in entry: - result.replies.bottom = e.getCursor - elif "cursor-bottom" in entry: - result.replies.bottom = e.getCursor - proc parseStatus*(js: JsonNode): Tweet = with e, js{"errors"}: if e.getError == tweetNotFound: return - result = parseTweet(js) + result = parseTweet(js, js{"card"}) if not result.isNil: result.user = parseUser(js{"user"}) @@ -409,7 +361,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline = proc parsePhotoRail*(js: JsonNode): PhotoRail = for tweet in js: let - t = parseTweet(tweet) + t = parseTweet(tweet, js{"card"}) url = if t.photos.len > 0: t.photos[0] elif t.video.isSome: get(t.video).thumb elif t.gif.isSome: get(t.gif).thumb @@ -418,3 +370,61 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail = if url.len == 0: continue result.add GalleryPhoto(url: url, tweetId: $t.id) + +proc parseGraphTweet(js: JsonNode): Tweet = + if js.kind == JNull: + return Tweet(available: false) + + var jsCard = copy(js{"card", "legacy"}) + if jsCard.kind != JNull: + var values = newJObject() + for val in jsCard["binding_values"]: + values[val["key"].getStr] = val["value"] + jsCard["binding_values"] = values + + result = parseTweet(js{"legacy"}, jsCard) + result.user = parseUser(js{"core", "user_results", "result", "legacy"}) + + if result.quote.isSome: + result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"})) + +proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] = + let thread = js{"content", "items"} + for t in js{"content", "items"}: + let entryId = t{"entryId"}.getStr + if "cursor-showmore" in entryId: + let cursor = t{"item", "itemContent", "value"} + result.thread.cursor = cursor.getStr + result.thread.hasMore = true + elif "tweet" in entryId: + let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"}) + result.thread.content.add tweet + + if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread": + result.self = true + +proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = + result = Conversation(replies: Result[Chain](beginning: true)) + + let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"} + if instructions.len == 0: + return + + for e in instructions[0]{"entries"}: + let entryId = e{"entryId"}.getStr + # echo entryId + if entryId.startsWith("tweet"): + let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"}) + + if $tweet.id == tweetId: + result.tweet = tweet + else: + result.before.content.add tweet + elif entryId.startsWith("conversationthread"): + let (thread, self) = parseGraphThread(e) + if self: + result.after = thread + else: + result.replies.content.add thread + elif entryId.startsWith("cursor-bottom"): + result.replies.bottom = e{"content", "itemContent", "value"}.getStr diff --git a/src/parserutils.nim b/src/parserutils.nim index 4b89236..af4d062 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -133,10 +133,6 @@ proc getTombstone*(js: JsonNode): string = result = js{"tombstoneInfo", "richText", "text"}.getStr result.removeSuffix(" Learn more") -proc getSource*(js: JsonNode): string = - let src = js{"source"}.getStr - result = src.substr(src.find('>') + 1, src.rfind('<') - 1) - proc getMp4Resolution*(url: string): int = # parses the height out of a URL like this one: # https://video.twimg.com/ext_tw_video//pu/vid/720x1280/.mp4 diff --git a/src/tokens.nim b/src/tokens.nim index e3b916a..e6a4449 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -41,7 +41,8 @@ proc getPoolJson*(): JsonNode = let maxReqs = case api - of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId, Api.userScreenName: 500 + of Api.listMembers, Api.listBySlug, Api.list, + Api.userRestId, Api.userScreenName, Api.tweetDetail: 500 of Api.timeline: 187 else: 180 reqs = maxReqs - token.apis[api].remaining diff --git a/src/types.nim b/src/types.nim index 07f9bf7..087acb2 100644 --- a/src/types.nim +++ b/src/types.nim @@ -9,6 +9,7 @@ type InternalError* = object of CatchableError Api* {.pure.} = enum + tweetDetail userShow timeline search @@ -176,7 +177,6 @@ type available*: bool tombstone*: string location*: string - source*: string stats*: TweetStats retweet*: Option[Tweet] attribution*: Option[User] diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 69335fa..ea94e28 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -347,7 +347,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; renderQuote(tweet.quote.get(), prefs, path) if mainTweet: - p(class="tweet-published"): text &"{getTime(tweet)} ยท {tweet.source}" + p(class="tweet-published"): text &"{getTime(tweet)}" if tweet.mediaTags.len > 0: renderMediaTags(tweet.mediaTags)