mirror of https://github.com/zedeus/nitter
Replace profile timeline with GraphQL endpoint
This commit is contained in:
parent
ae03170286
commit
8fc3c3dec5
19
src/api.nim
19
src/api.nim
|
@ -7,12 +7,9 @@ import experimental/parser as newParser
|
||||||
proc getGraphUser*(username: string): Future[User] {.async.} =
|
proc getGraphUser*(username: string): Future[User] {.async.} =
|
||||||
if username.len == 0: return
|
if username.len == 0: return
|
||||||
let
|
let
|
||||||
variables = """{
|
variables = %*{"screen_name": username}
|
||||||
"screen_name": "$1",
|
params = {"variables": $variables, "features": userFeatures}
|
||||||
"withSafetyModeUserFields": false,
|
js = await fetchRaw(graphUser ? params, Api.userScreenName)
|
||||||
"withSuperFollowsUserFields": false
|
|
||||||
}""" % [username]
|
|
||||||
js = await fetchRaw(graphUser ? {"variables": variables}, Api.userScreenName)
|
|
||||||
result = parseGraphUser(js)
|
result = parseGraphUser(js)
|
||||||
|
|
||||||
proc getGraphUserById*(id: string): Future[User] {.async.} =
|
proc getGraphUserById*(id: string): Future[User] {.async.} =
|
||||||
|
@ -22,6 +19,16 @@ proc getGraphUserById*(id: string): Future[User] {.async.} =
|
||||||
js = await fetchRaw(graphUserById ? {"variables": variables}, Api.userRestId)
|
js = await fetchRaw(graphUserById ? {"variables": variables}, Api.userRestId)
|
||||||
result = parseGraphUser(js)
|
result = parseGraphUser(js)
|
||||||
|
|
||||||
|
proc getGraphUserTweets*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
|
||||||
|
if id.len == 0: return
|
||||||
|
let
|
||||||
|
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||||
|
variables = userTweetsVariables % [id, cursor]
|
||||||
|
params = {"variables": variables, "features": userTweetsFeatures}
|
||||||
|
url = if replies: graphUserTweetsAndReplies else: graphUserTweets
|
||||||
|
js = await fetch(url ? params, Api.tweetDetail)
|
||||||
|
result = parseGraphTimeline(js, after)
|
||||||
|
|
||||||
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
|
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
|
||||||
let
|
let
|
||||||
variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
|
variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
|
||||||
|
|
|
@ -19,8 +19,10 @@ const
|
||||||
tweet* = timelineApi / "conversation"
|
tweet* = timelineApi / "conversation"
|
||||||
|
|
||||||
graphql = api / "graphql"
|
graphql = api / "graphql"
|
||||||
|
graphUserTweets* = graphql / "9rys0A7w1EyqVd2ME0QCJg/UserTweets"
|
||||||
|
graphUserTweetsAndReplies* = graphql / "ehMCHF3Mkgjsfz_aImqOsg/UserTweetsAndReplies"
|
||||||
graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
|
graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
|
||||||
graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
|
graphUser* = graphql / "nZjSkpOpSL5rWyIVdsKeLA/UserByScreenName"
|
||||||
graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
|
graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
|
||||||
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
|
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
|
||||||
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
|
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
|
||||||
|
@ -35,6 +37,7 @@ const
|
||||||
"include_mute_edge": "0",
|
"include_mute_edge": "0",
|
||||||
"include_can_dm": "0",
|
"include_can_dm": "0",
|
||||||
"include_can_media_tag": "1",
|
"include_can_media_tag": "1",
|
||||||
|
"include_ext_is_blue_verified": "true",
|
||||||
"skip_status": "1",
|
"skip_status": "1",
|
||||||
"cards_platform": "Web-12",
|
"cards_platform": "Web-12",
|
||||||
"include_cards": "1",
|
"include_cards": "1",
|
||||||
|
@ -60,6 +63,40 @@ const
|
||||||
## photos: "result_filter: photos"
|
## photos: "result_filter: photos"
|
||||||
## videos: "result_filter: videos"
|
## videos: "result_filter: videos"
|
||||||
|
|
||||||
|
userTweetsVariables* = """{
|
||||||
|
"userId": "$1",
|
||||||
|
$2
|
||||||
|
"count": 20,
|
||||||
|
"includePromotedContent": false,
|
||||||
|
"withDownvotePerspective": false,
|
||||||
|
"withReactionsMetadata": false,
|
||||||
|
"withReactionsPerspective": false,
|
||||||
|
"withVoice": false,
|
||||||
|
"withV2Timeline": true
|
||||||
|
}"""
|
||||||
|
|
||||||
|
userTweetsFeatures* = """{
|
||||||
|
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
|
||||||
|
"responsive_web_graphql_exclude_directive_enabled": false,
|
||||||
|
"verified_phone_label_enabled": false,
|
||||||
|
"responsive_web_graphql_timeline_navigation_enabled": false,
|
||||||
|
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
|
||||||
|
"tweetypie_unmention_optimization_enabled": false,
|
||||||
|
"vibe_api_enabled": false,
|
||||||
|
"responsive_web_edit_tweet_api_enabled": false,
|
||||||
|
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
|
||||||
|
"view_counts_everywhere_api_enabled": false,
|
||||||
|
"longform_notetweets_consumption_enabled": true,
|
||||||
|
"tweet_awards_web_tipping_enabled": false,
|
||||||
|
"freedom_of_speech_not_reach_fetch_enabled": false,
|
||||||
|
"standardized_nudges_misinfo": false,
|
||||||
|
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
|
||||||
|
"interactive_text_enabled": false,
|
||||||
|
"responsive_web_text_conversations_enabled": false,
|
||||||
|
"longform_notetweets_richtext_consumption_enabled": false,
|
||||||
|
"responsive_web_enhance_cards_enabled": false
|
||||||
|
}"""
|
||||||
|
|
||||||
tweetVariables* = """{
|
tweetVariables* = """{
|
||||||
"focalTweetId": "$1",
|
"focalTweetId": "$1",
|
||||||
$2
|
$2
|
||||||
|
@ -79,7 +116,7 @@ const
|
||||||
"responsive_web_graphql_timeline_navigation_enabled": false,
|
"responsive_web_graphql_timeline_navigation_enabled": false,
|
||||||
"standardized_nudges_misinfo": false,
|
"standardized_nudges_misinfo": false,
|
||||||
"verified_phone_label_enabled": false,
|
"verified_phone_label_enabled": false,
|
||||||
"responsive_web_twitter_blue_verified_badge_is_enabled": false,
|
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
|
||||||
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
|
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
|
||||||
"view_counts_everywhere_api_enabled": false,
|
"view_counts_everywhere_api_enabled": false,
|
||||||
"responsive_web_edit_tweet_api_enabled": false,
|
"responsive_web_edit_tweet_api_enabled": false,
|
||||||
|
@ -90,3 +127,11 @@ const
|
||||||
"responsive_web_enhance_cards_enabled": false,
|
"responsive_web_enhance_cards_enabled": false,
|
||||||
"interactive_text_enabled": false
|
"interactive_text_enabled": false
|
||||||
}"""
|
}"""
|
||||||
|
|
||||||
|
userFeatures* = """{
|
||||||
|
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
|
||||||
|
"verified_phone_label_enabled": false,
|
||||||
|
"responsive_web_graphql_timeline_navigation_enabled": false,
|
||||||
|
"responsive_web_graphql_exclude_directive_enabled": true,
|
||||||
|
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": true
|
||||||
|
}"""
|
||||||
|
|
|
@ -11,6 +11,7 @@ proc parseGraphUser*(json: string): User =
|
||||||
|
|
||||||
result = toUser raw.data.user.result.legacy
|
result = toUser raw.data.user.result.legacy
|
||||||
result.id = raw.data.user.result.restId
|
result.id = raw.data.user.result.restId
|
||||||
|
result.verified = result.verified or raw.data.user.result.isBlueVerified
|
||||||
|
|
||||||
proc parseGraphListMembers*(json, cursor: string): Result[User] =
|
proc parseGraphListMembers*(json, cursor: string): Result[User] =
|
||||||
result = Result[User](
|
result = Result[User](
|
||||||
|
|
|
@ -11,4 +11,5 @@ type
|
||||||
UserResult = object
|
UserResult = object
|
||||||
legacy*: RawUser
|
legacy*: RawUser
|
||||||
restId*: string
|
restId*: string
|
||||||
|
isBlueVerified*: bool
|
||||||
reason*: Option[string]
|
reason*: Option[string]
|
||||||
|
|
|
@ -4,6 +4,8 @@ import packedjson, packedjson/deserialiser
|
||||||
import types, parserutils, utils
|
import types, parserutils, utils
|
||||||
import experimental/parser/unifiedcard
|
import experimental/parser/unifiedcard
|
||||||
|
|
||||||
|
proc parseGraphTweet(js: JsonNode): Tweet
|
||||||
|
|
||||||
proc parseUser(js: JsonNode; id=""): User =
|
proc parseUser(js: JsonNode; id=""): User =
|
||||||
if js.isNull: return
|
if js.isNull: return
|
||||||
result = User(
|
result = User(
|
||||||
|
@ -19,13 +21,20 @@ proc parseUser(js: JsonNode; id=""): User =
|
||||||
tweets: js{"statuses_count"}.getInt,
|
tweets: js{"statuses_count"}.getInt,
|
||||||
likes: js{"favourites_count"}.getInt,
|
likes: js{"favourites_count"}.getInt,
|
||||||
media: js{"media_count"}.getInt,
|
media: js{"media_count"}.getInt,
|
||||||
verified: js{"verified"}.getBool,
|
verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool,
|
||||||
protected: js{"protected"}.getBool,
|
protected: js{"protected"}.getBool,
|
||||||
joinDate: js{"created_at"}.getTime
|
joinDate: js{"created_at"}.getTime
|
||||||
)
|
)
|
||||||
|
|
||||||
result.expandUserEntities(js)
|
result.expandUserEntities(js)
|
||||||
|
|
||||||
|
proc parseGraphUser(js: JsonNode): User =
|
||||||
|
let user = ? js{"user_results", "result"}
|
||||||
|
result = parseUser(user{"legacy"})
|
||||||
|
|
||||||
|
if "is_blue_verified" in user:
|
||||||
|
result.verified = true
|
||||||
|
|
||||||
proc parseGraphList*(js: JsonNode): List =
|
proc parseGraphList*(js: JsonNode): List =
|
||||||
if js.isNull: return
|
if js.isNull: return
|
||||||
|
|
||||||
|
@ -213,10 +222,16 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
|
||||||
if js{"is_quote_status"}.getBool:
|
if js{"is_quote_status"}.getBool:
|
||||||
result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
|
result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
|
||||||
|
|
||||||
|
# legacy
|
||||||
with rt, js{"retweeted_status_id_str"}:
|
with rt, js{"retweeted_status_id_str"}:
|
||||||
result.retweet = some Tweet(id: rt.getId)
|
result.retweet = some Tweet(id: rt.getId)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# graphql
|
||||||
|
with rt, js{"retweeted_status_result", "result"}:
|
||||||
|
result.retweet = some parseGraphTweet(rt)
|
||||||
|
return
|
||||||
|
|
||||||
if jsCard.kind != JNull:
|
if jsCard.kind != JNull:
|
||||||
let name = jsCard{"name"}.getStr
|
let name = jsCard{"name"}.getStr
|
||||||
if "poll" in name:
|
if "poll" in name:
|
||||||
|
@ -237,7 +252,10 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
|
||||||
of "video":
|
of "video":
|
||||||
result.video = some(parseVideo(m))
|
result.video = some(parseVideo(m))
|
||||||
with user, m{"additional_media_info", "source_user"}:
|
with user, m{"additional_media_info", "source_user"}:
|
||||||
result.attribution = some(parseUser(user))
|
if user{"id"}.getInt > 0:
|
||||||
|
result.attribution = some(parseUser(user))
|
||||||
|
else:
|
||||||
|
result.attribution = some(parseGraphUser(user))
|
||||||
of "animated_gif":
|
of "animated_gif":
|
||||||
result.gif = some(parseGif(m))
|
result.gif = some(parseGif(m))
|
||||||
else: discard
|
else: discard
|
||||||
|
@ -384,7 +402,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
|
||||||
jsCard["binding_values"] = values
|
jsCard["binding_values"] = values
|
||||||
|
|
||||||
result = parseTweet(js{"legacy"}, jsCard)
|
result = parseTweet(js{"legacy"}, jsCard)
|
||||||
result.user = parseUser(js{"core", "user_results", "result", "legacy"})
|
result.user = parseGraphUser(js{"core"})
|
||||||
|
|
||||||
with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
|
with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
|
||||||
result.expandNoteTweetEntities(noteTweet)
|
result.expandNoteTweetEntities(noteTweet)
|
||||||
|
@ -435,3 +453,22 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
|
||||||
result.replies.content.add thread
|
result.replies.content.add thread
|
||||||
elif entryId.startsWith("cursor-bottom"):
|
elif entryId.startsWith("cursor-bottom"):
|
||||||
result.replies.bottom = e{"content", "itemContent", "value"}.getStr
|
result.replies.bottom = e{"content", "itemContent", "value"}.getStr
|
||||||
|
|
||||||
|
proc parseGraphTimeline*(js: JsonNode; after=""): Timeline =
|
||||||
|
result = Timeline(beginning: after.len == 0)
|
||||||
|
|
||||||
|
let instructions = ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
|
||||||
|
if instructions.len == 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
for e in instructions[instructions.len - 1]{"entries"}:
|
||||||
|
let entryId = e{"entryId"}.getStr
|
||||||
|
if entryId.startsWith("tweet"):
|
||||||
|
let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
|
||||||
|
if not tweet.available:
|
||||||
|
tweet.id = parseBiggestInt(entryId.getId())
|
||||||
|
result.content.add tweet
|
||||||
|
elif entryId.startsWith("cursor-bottom"):
|
||||||
|
result.bottom = e{"content", "value"}.getStr
|
||||||
|
elif "cursor-top" notin entryId:
|
||||||
|
echo e
|
||||||
|
|
|
@ -47,8 +47,8 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
|
||||||
let
|
let
|
||||||
timeline =
|
timeline =
|
||||||
case query.kind
|
case query.kind
|
||||||
of posts: getTimeline(userId, after)
|
of posts: getGraphUserTweets(userId, after)
|
||||||
of replies: getTimeline(userId, after, replies=true)
|
of replies: getGraphUserTweets(userId, after, replies=true)
|
||||||
of media: getMediaTimeline(userId, after)
|
of media: getMediaTimeline(userId, after)
|
||||||
else: getSearch[Tweet](query, after)
|
else: getSearch[Tweet](query, after)
|
||||||
|
|
||||||
|
@ -64,6 +64,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
|
||||||
let tweet = await getCachedTweet(user.pinnedTweet)
|
let tweet = await getCachedTweet(user.pinnedTweet)
|
||||||
if not tweet.isNil:
|
if not tweet.isNil:
|
||||||
tweet.pinned = true
|
tweet.pinned = true
|
||||||
|
tweet.user = user
|
||||||
pinned = some tweet
|
pinned = some tweet
|
||||||
|
|
||||||
result = Profile(
|
result = Profile(
|
||||||
|
|
|
@ -42,7 +42,7 @@ proc getPoolJson*(): JsonNode =
|
||||||
maxReqs =
|
maxReqs =
|
||||||
case api
|
case api
|
||||||
of Api.listMembers, Api.listBySlug, Api.list,
|
of Api.listMembers, Api.listBySlug, Api.list,
|
||||||
Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
|
Api.userRestId, Api.userScreenName, Api.userTweets, Api.tweetDetail: 500
|
||||||
of Api.timeline: 187
|
of Api.timeline: 187
|
||||||
else: 180
|
else: 180
|
||||||
reqs = maxReqs - token.apis[api].remaining
|
reqs = maxReqs - token.apis[api].remaining
|
||||||
|
|
|
@ -19,6 +19,7 @@ type
|
||||||
listMembers
|
listMembers
|
||||||
userRestId
|
userRestId
|
||||||
userScreenName
|
userScreenName
|
||||||
|
userTweets
|
||||||
status
|
status
|
||||||
|
|
||||||
RateLimit* = object
|
RateLimit* = object
|
||||||
|
|
|
@ -17,11 +17,6 @@ protected = [
|
||||||
|
|
||||||
invalid = [['thisprofiledoesntexist'], ['%']]
|
invalid = [['thisprofiledoesntexist'], ['%']]
|
||||||
|
|
||||||
banner_color = [
|
|
||||||
['nim_lang', '22, 25, 32'],
|
|
||||||
['rustlang', '35, 31, 32']
|
|
||||||
]
|
|
||||||
|
|
||||||
banner_image = [
|
banner_image = [
|
||||||
['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
|
['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
|
||||||
]
|
]
|
||||||
|
@ -74,12 +69,6 @@ class ProfileTest(BaseTestCase):
|
||||||
self.open_nitter('user')
|
self.open_nitter('user')
|
||||||
self.assert_text('User "user" has been suspended')
|
self.assert_text('User "user" has been suspended')
|
||||||
|
|
||||||
@parameterized.expand(banner_color)
|
|
||||||
def test_banner_color(self, username, color):
|
|
||||||
self.open_nitter(username)
|
|
||||||
banner = self.find_element(Profile.banner + ' a')
|
|
||||||
self.assertIn(color, banner.value_of_css_property('background-color'))
|
|
||||||
|
|
||||||
@parameterized.expand(banner_image)
|
@parameterized.expand(banner_image)
|
||||||
def test_banner_image(self, username, url):
|
def test_banner_image(self, username, url):
|
||||||
self.open_nitter(username)
|
self.open_nitter(username)
|
||||||
|
|
Loading…
Reference in New Issue