Replace profile timeline with GraphQL endpoint

src/api.nim

@@ -7,12 +7,9 @@ import experimental/parser as newParser
 proc getGraphUser*(username: string): Future[User] {.async.} =
   if username.len == 0: return
   let
-    variables = """{
-      "screen_name": "$1",
-      "withSafetyModeUserFields": false,
-      "withSuperFollowsUserFields": false
-    }""" % [username]
-    js = await fetchRaw(graphUser ? {"variables": variables}, Api.userScreenName)
+    variables = %*{"screen_name": username}
+    params = {"variables": $variables, "features": userFeatures}
+    js = await fetchRaw(graphUser ? params, Api.userScreenName)
   result = parseGraphUser(js)
 
 proc getGraphUserById*(id: string): Future[User] {.async.} =
@@ -22,6 +19,16 @@ proc getGraphUserById*(id: string): Future[User] {.async.} =
     js = await fetchRaw(graphUserById ? {"variables": variables}, Api.userRestId)
   result = parseGraphUser(js)
 
+proc getGraphUserTweets*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = userTweetsVariables % [id, cursor]
+    params = {"variables": variables, "features": userTweetsFeatures}
+    url = if replies: graphUserTweetsAndReplies else: graphUserTweets
+    js = await fetch(url ? params, Api.tweetDetail)
+  result = parseGraphTimeline(js, after)
+
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
     variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}

src/consts.nim

@@ -19,8 +19,10 @@ const
   tweet* = timelineApi / "conversation"
 
   graphql = api / "graphql"
+  graphUserTweets* = graphql / "9rys0A7w1EyqVd2ME0QCJg/UserTweets"
+  graphUserTweetsAndReplies* = graphql / "ehMCHF3Mkgjsfz_aImqOsg/UserTweetsAndReplies"
   graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
-  graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
+  graphUser* = graphql / "nZjSkpOpSL5rWyIVdsKeLA/UserByScreenName"
   graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
   graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
   graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
@@ -35,6 +37,7 @@ const
     "include_mute_edge": "0",
     "include_can_dm": "0",
     "include_can_media_tag": "1",
+    "include_ext_is_blue_verified": "true",
     "skip_status": "1",
     "cards_platform": "Web-12",
     "include_cards": "1",
@@ -60,6 +63,40 @@ const
   ## photos: "result_filter: photos"
   ## videos: "result_filter: videos"
 
+  userTweetsVariables* = """{
+  "userId": "$1",
+  $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false,
+  "withV2Timeline": true
+}"""
+
+  userTweetsFeatures* = """{
+  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+  "responsive_web_graphql_exclude_directive_enabled": false,
+  "verified_phone_label_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": false,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "tweetypie_unmention_optimization_enabled": false,
+  "vibe_api_enabled": false,
+  "responsive_web_edit_tweet_api_enabled": false,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+  "view_counts_everywhere_api_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
+  "tweet_awards_web_tipping_enabled": false,
+  "freedom_of_speech_not_reach_fetch_enabled": false,
+  "standardized_nudges_misinfo": false,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+  "interactive_text_enabled": false,
+  "responsive_web_text_conversations_enabled": false,
+  "longform_notetweets_richtext_consumption_enabled": false,
+  "responsive_web_enhance_cards_enabled": false
+}"""
+
   tweetVariables* = """{
   "focalTweetId": "$1",
   $2
@@ -79,7 +116,7 @@ const
   "responsive_web_graphql_timeline_navigation_enabled": false,
   "standardized_nudges_misinfo": false,
   "verified_phone_label_enabled": false,
-  "responsive_web_twitter_blue_verified_badge_is_enabled": false,
+  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
   "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
   "view_counts_everywhere_api_enabled": false,
   "responsive_web_edit_tweet_api_enabled": false,
@@ -90,3 +127,11 @@ const
   "responsive_web_enhance_cards_enabled": false,
   "interactive_text_enabled": false
 }"""
+
+  userFeatures* = """{
+  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+  "verified_phone_label_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": false,
+  "responsive_web_graphql_exclude_directive_enabled": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": true
+}"""

src/experimental/parser/graphql.nim

@@ -11,6 +11,7 @@ proc parseGraphUser*(json: string): User =
 
   result = toUser raw.data.user.result.legacy
   result.id = raw.data.user.result.restId
+  result.verified = result.verified or raw.data.user.result.isBlueVerified
 
 proc parseGraphListMembers*(json, cursor: string): Result[User] =
   result = Result[User](

src/experimental/types/graphuser.nim

@@ -11,4 +11,5 @@ type
   UserResult = object
     legacy*: RawUser
     restId*: string
+    isBlueVerified*: bool
     reason*: Option[string]

src/parser.nim

@@ -4,6 +4,8 @@ import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
+proc parseGraphTweet(js: JsonNode): Tweet
+
 proc parseUser(js: JsonNode; id=""): User =
   if js.isNull: return
   result = User(
@@ -19,13 +21,20 @@ proc parseUser(js: JsonNode; id=""): User =
     tweets: js{"statuses_count"}.getInt,
     likes: js{"favourites_count"}.getInt,
     media: js{"media_count"}.getInt,
-    verified: js{"verified"}.getBool,
+    verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool,
     protected: js{"protected"}.getBool,
     joinDate: js{"created_at"}.getTime
   )
 
   result.expandUserEntities(js)
 
+proc parseGraphUser(js: JsonNode): User =
+  let user = ? js{"user_results", "result"}
+  result = parseUser(user{"legacy"})
+
+  if "is_blue_verified" in user:
+    result.verified = true
+
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 
@@ -213,10 +222,16 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
   if js{"is_quote_status"}.getBool:
     result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
 
+  # legacy
   with rt, js{"retweeted_status_id_str"}:
     result.retweet = some Tweet(id: rt.getId)
     return
 
+  # graphql
+  with rt, js{"retweeted_status_result", "result"}:
+    result.retweet = some parseGraphTweet(rt)
+    return
+
   if jsCard.kind != JNull:
     let name = jsCard{"name"}.getStr
     if "poll" in name:
@@ -237,7 +252,10 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       of "video":
         result.video = some(parseVideo(m))
         with user, m{"additional_media_info", "source_user"}:
-          result.attribution = some(parseUser(user))
+          if user{"id"}.getInt > 0:
+            result.attribution = some(parseUser(user))
+          else:
+            result.attribution = some(parseGraphUser(user))
       of "animated_gif":
         result.gif = some(parseGif(m))
       else: discard
@@ -384,7 +402,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
     jsCard["binding_values"] = values
 
   result = parseTweet(js{"legacy"}, jsCard)
-  result.user = parseUser(js{"core", "user_results", "result", "legacy"})
+  result.user = parseGraphUser(js{"core"})
 
   with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
     result.expandNoteTweetEntities(noteTweet)
@@ -435,3 +453,22 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
         result.replies.content.add thread
     elif entryId.startsWith("cursor-bottom"):
       result.replies.bottom = e{"content", "itemContent", "value"}.getStr
+
+proc parseGraphTimeline*(js: JsonNode; after=""): Timeline =
+  result = Timeline(beginning: after.len == 0)
+
+  let instructions = ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for e in instructions[instructions.len - 1]{"entries"}:
+    let entryId = e{"entryId"}.getStr
+    if entryId.startsWith("tweet"):
+      let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
+      if not tweet.available:
+        tweet.id = parseBiggestInt(entryId.getId())
+      result.content.add tweet
+    elif entryId.startsWith("cursor-bottom"):
+      result.bottom = e{"content", "value"}.getStr
+    elif "cursor-top" notin entryId:
+      echo e

src/routes/timeline.nim

@@ -47,8 +47,8 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
   let
     timeline =
       case query.kind
-      of posts: getTimeline(userId, after)
-      of replies: getTimeline(userId, after, replies=true)
+      of posts: getGraphUserTweets(userId, after)
+      of replies: getGraphUserTweets(userId, after, replies=true)
       of media: getMediaTimeline(userId, after)
       else: getSearch[Tweet](query, after)
 
@@ -64,6 +64,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
     let tweet = await getCachedTweet(user.pinnedTweet)
     if not tweet.isNil:
       tweet.pinned = true
+      tweet.user = user
       pinned = some tweet
 
   result = Profile(

src/tokens.nim

@@ -42,7 +42,7 @@ proc getPoolJson*(): JsonNode =
         maxReqs =
           case api
           of Api.listMembers, Api.listBySlug, Api.list,
-             Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
+             Api.userRestId, Api.userScreenName, Api.userTweets, Api.tweetDetail: 500
           of Api.timeline: 187
           else: 180
         reqs = maxReqs - token.apis[api].remaining

src/types.nim

@@ -19,6 +19,7 @@ type
     listMembers
     userRestId
     userScreenName
+    userTweets
     status
 
   RateLimit* = object

tests/test_profile.py

@@ -17,11 +17,6 @@ protected = [
 
 invalid = [['thisprofiledoesntexist'], ['%']]
 
-banner_color = [
-    ['nim_lang', '22, 25, 32'],
-    ['rustlang', '35, 31, 32']
-]
-
 banner_image = [
     ['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
 ]
@@ -74,12 +69,6 @@ class ProfileTest(BaseTestCase):
         self.open_nitter('user')
         self.assert_text('User "user" has been suspended')
 
-    @parameterized.expand(banner_color)
-    def test_banner_color(self, username, color):
-        self.open_nitter(username)
-        banner = self.find_element(Profile.banner + ' a')
-        self.assertIn(color, banner.value_of_css_property('background-color'))
-
     @parameterized.expand(banner_image)
     def test_banner_image(self, username, url):
         self.open_nitter(username)