From 36c72f98603e6387431224b3fb65ada765e9ab65 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 2 Feb 2023 12:38:58 +0100 Subject: [PATCH] Revert "Replace tweet endpoint with GraphQL" This reverts commit 19adc658c317c71e671201385350343add6275c8. --- src/api.nim | 15 ++---- src/consts.nim | 32 ------------ src/parser.nim | 120 ++++++++++++++++++++------------------------ src/parserutils.nim | 4 ++ src/tokens.nim | 3 +- src/types.nim | 2 +- src/views/tweet.nim | 2 +- 7 files changed, 67 insertions(+), 111 deletions(-) diff --git a/src/api.nim b/src/api.nim index dfcf413..dd94b69 100644 --- a/src/api.nim +++ b/src/api.nim @@ -101,21 +101,16 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = except InternalError: return Result[T](beginning: true, query: query) -proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} = - if id.len == 0: return - let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = tweetVariables % [id, cursor] - params = {"variables": variables, "features": tweetFeatures} - js = await fetch(graphTweet ? params, Api.tweetDetail) - result = parseGraphConversation(js, id) +proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} = + let url = tweet / (id & ".json") ? genParams(cursor=after) + result = parseConversation(await fetch(url, Api.tweet), id) proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} = - result = (await getGraphTweet(id, after)).replies + result = (await getTweetImpl(id, after)).replies result.beginning = after.len == 0 proc getTweet*(id: string; after=""): Future[Conversation] {.async.} = - result = await getGraphTweet(id) + result = await getTweetImpl(id) if after.len > 0: result.replies = await getReplies(id, after) diff --git a/src/consts.nim b/src/consts.nim index 18ecb2c..2d3ea56 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -19,7 +19,6 @@ const tweet* = timelineApi / "conversation" graphql = api / "graphql" - graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail" graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName" graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId" graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List" @@ -59,34 +58,3 @@ const ## user: "result_filter: user" ## photos: "result_filter: photos" ## videos: "result_filter: videos" - - tweetVariables* = """{ - "focalTweetId": "$1", - $2 - "includePromotedContent": false, - "withBirdwatchNotes": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withSuperFollowsTweetFields": false, - "withSuperFollowsUserFields": false, - "withVoice": false, - "withV2Timeline": true -}""" - - tweetFeatures* = """{ - "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false, - "responsive_web_graphql_timeline_navigation_enabled": false, - "standardized_nudges_misinfo": false, - "verified_phone_label_enabled": false, - "responsive_web_twitter_blue_verified_badge_is_enabled": false, - "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false, - "view_counts_everywhere_api_enabled": false, - "responsive_web_edit_tweet_api_enabled": false, - "tweetypie_unmention_optimization_enabled": false, - "vibe_api_enabled": false, - "longform_notetweets_consumption_enabled": false, - "responsive_web_text_conversations_enabled": false, - "responsive_web_enhance_cards_enabled": false, - "interactive_text_enabled": false -}""" diff --git a/src/parser.nim b/src/parser.nim index 544f943..2b0c6a4 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -72,8 +72,8 @@ proc parseGif(js: JsonNode): Gif = proc parseVideo(js: JsonNode): Video = result = Video( thumb: js{"media_url_https"}.getImageStr, - views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt), - available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available", + views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr, + available: js{"ext_media_availability", "status"}.getStr == "available", title: js{"ext_alt_text"}.getStr, durationMs: js{"video_info", "duration_millis"}.getInt # playbackType: mp4 @@ -185,7 +185,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card = result.url.len == 0 or result.url.startsWith("card://"): result.url = getPicUrl(result.image) -proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = +proc parseTweet(js: JsonNode): Tweet = if js.isNull: return result = Tweet( id: js{"id_str"}.getId, @@ -193,6 +193,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = replyId: js{"in_reply_to_status_id_str"}.getId, text: js{"full_text"}.getStr, time: js{"created_at"}.getTime, + source: getSource(js), hasThread: js{"self_thread"}.notNull, available: true, user: User(id: js{"user_id_str"}.getStr), @@ -217,7 +218,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = result.retweet = some Tweet(id: rt.getId) return - if jsCard.kind != JNull: + with jsCard, js{"card"}: let name = jsCard{"name"}.getStr if "poll" in name: if "image" in name: @@ -294,17 +295,64 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects = result.users[k] = parseUser(v, k) for k, v in tweets: - var tweet = parseTweet(v, v{"card"}) + var tweet = parseTweet(v) if tweet.user.id in result.users: tweet.user = result.users[tweet.user.id] result.tweets[k] = tweet +proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] = + result.thread = Chain() + + let thread = js{"content", "item", "content", "conversationThread"} + with cursor, thread{"showMoreCursor"}: + result.thread.cursor = cursor{"value"}.getStr + result.thread.hasMore = true + + for t in thread{"conversationComponents"}: + let content = t{"conversationTweetComponent", "tweet"} + + if content{"displayType"}.getStr == "SelfThread": + result.self = true + + var tweet = finalizeTweet(global, content{"id"}.getStr) + if not tweet.available: + tweet.tombstone = getTombstone(content{"tombstone"}) + result.thread.content.add tweet + +proc parseConversation*(js: JsonNode; tweetId: string): Conversation = + result = Conversation(replies: Result[Chain](beginning: true)) + let global = parseGlobalObjects(? js) + + let instructions = ? js{"timeline", "instructions"} + if instructions.len == 0: + return + + for e in instructions[0]{"addEntries", "entries"}: + let entry = e{"entryId"}.getStr + if "tweet" in entry or "tombstone" in entry: + let tweet = finalizeTweet(global, e.getEntryId) + if $tweet.id != tweetId: + result.before.content.add tweet + else: + result.tweet = tweet + elif "conversationThread" in entry: + let (thread, self) = parseThread(e, global) + if thread.content.len > 0: + if self: + result.after = thread + else: + result.replies.content.add thread + elif "cursor-showMore" in entry: + result.replies.bottom = e.getCursor + elif "cursor-bottom" in entry: + result.replies.bottom = e.getCursor + proc parseStatus*(js: JsonNode): Tweet = with e, js{"errors"}: if e.getError == tweetNotFound: return - result = parseTweet(js, js{"card"}) + result = parseTweet(js) if not result.isNil: result.user = parseUser(js{"user"}) @@ -361,7 +409,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline = proc parsePhotoRail*(js: JsonNode): PhotoRail = for tweet in js: let - t = parseTweet(tweet, js{"card"}) + t = parseTweet(tweet) url = if t.photos.len > 0: t.photos[0] elif t.video.isSome: get(t.video).thumb elif t.gif.isSome: get(t.gif).thumb @@ -370,61 +418,3 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail = if url.len == 0: continue result.add GalleryPhoto(url: url, tweetId: $t.id) - -proc parseGraphTweet(js: JsonNode): Tweet = - if js.kind == JNull: - return Tweet(available: false) - - var jsCard = copy(js{"card", "legacy"}) - if jsCard.kind != JNull: - var values = newJObject() - for val in jsCard["binding_values"]: - values[val["key"].getStr] = val["value"] - jsCard["binding_values"] = values - - result = parseTweet(js{"legacy"}, jsCard) - result.user = parseUser(js{"core", "user_results", "result", "legacy"}) - - if result.quote.isSome: - result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"})) - -proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] = - let thread = js{"content", "items"} - for t in js{"content", "items"}: - let entryId = t{"entryId"}.getStr - if "cursor-showmore" in entryId: - let cursor = t{"item", "itemContent", "value"} - result.thread.cursor = cursor.getStr - result.thread.hasMore = true - elif "tweet" in entryId: - let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"}) - result.thread.content.add tweet - - if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread": - result.self = true - -proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = - result = Conversation(replies: Result[Chain](beginning: true)) - - let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"} - if instructions.len == 0: - return - - for e in instructions[0]{"entries"}: - let entryId = e{"entryId"}.getStr - # echo entryId - if entryId.startsWith("tweet"): - let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"}) - - if $tweet.id == tweetId: - result.tweet = tweet - else: - result.before.content.add tweet - elif entryId.startsWith("conversationthread"): - let (thread, self) = parseGraphThread(e) - if self: - result.after = thread - else: - result.replies.content.add thread - elif entryId.startsWith("cursor-bottom"): - result.replies.bottom = e{"content", "itemContent", "value"}.getStr diff --git a/src/parserutils.nim b/src/parserutils.nim index af4d062..4b89236 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -133,6 +133,10 @@ proc getTombstone*(js: JsonNode): string = result = js{"tombstoneInfo", "richText", "text"}.getStr result.removeSuffix(" Learn more") +proc getSource*(js: JsonNode): string = + let src = js{"source"}.getStr + result = src.substr(src.find('>') + 1, src.rfind('<') - 1) + proc getMp4Resolution*(url: string): int = # parses the height out of a URL like this one: # https://video.twimg.com/ext_tw_video//pu/vid/720x1280/.mp4 diff --git a/src/tokens.nim b/src/tokens.nim index e6a4449..e3b916a 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -41,8 +41,7 @@ proc getPoolJson*(): JsonNode = let maxReqs = case api - of Api.listMembers, Api.listBySlug, Api.list, - Api.userRestId, Api.userScreenName, Api.tweetDetail: 500 + of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId, Api.userScreenName: 500 of Api.timeline: 187 else: 180 reqs = maxReqs - token.apis[api].remaining diff --git a/src/types.nim b/src/types.nim index 087acb2..07f9bf7 100644 --- a/src/types.nim +++ b/src/types.nim @@ -9,7 +9,6 @@ type InternalError* = object of CatchableError Api* {.pure.} = enum - tweetDetail userShow timeline search @@ -177,6 +176,7 @@ type available*: bool tombstone*: string location*: string + source*: string stats*: TweetStats retweet*: Option[Tweet] attribution*: Option[User] diff --git a/src/views/tweet.nim b/src/views/tweet.nim index ea94e28..69335fa 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -347,7 +347,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; renderQuote(tweet.quote.get(), prefs, path) if mainTweet: - p(class="tweet-published"): text &"{getTime(tweet)}" + p(class="tweet-published"): text &"{getTime(tweet)} ยท {tweet.source}" if tweet.mediaTags.len > 0: renderMediaTags(tweet.mediaTags)