lasers/nitter
1
0
Fork 0
mirror of https://github.com/zedeus/nitter synced 2024-11-21 17:35:22 +01:00
Code Issues Projects Releases Wiki Activity

GraphQL timeline (#812)

Browse Source 
* Update deps

* Replace profile timeline with GraphQL endpoint

* Update GraphQL endpoint versions

* Use GraphQL for profile media tab

* Fix UserByRestId request

* Improve routing, fixes #814

* Fix token pool JSON

* Deduplicate GraphQL timeline endpoints

* Update list endpoints

* Use GraphQL for list tweets

* Remove debug leftover

* Replace old pinned tweet endpoint with GraphQL

* Validate tweet ID

* Minor token handling fix

* Hide US-only commerce cards

* Update config example

* Remove http pool and gzip from token pool

* Support tombstoned tweets in threads

* Retry GraphQL timeout errors

* Remove unnecessary 401 retry

* Remove broken timeout retry

* Update karax, use new bool attribute feature

* Update card test

* Fix odd edgecase with broken retweets

* Replace search endpoints, switch Bearer token

* Only parse user search if it's a list

* Fix quoted tweet crash

* Fix empty search query handling

* Fix invalid user search errors again
This commit is contained in:
Zed 2023-04-21 12:41:30 +00:00 committed by GitHub
parent e2560dc1f1
commit 1ac389e7c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 405 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
nitter.example.conf
View File

@ -1,11 +1,11 @@
[Server]
hostname = "nitter.net" # for generating links, change this to your own domain/ip
title = "nitter"
address = "0.0.0.0"
port = 8080
https = false # disable to enable cookies when not using https
httpMaxConnections = 100
staticDir = "./public"
title = "nitter"
hostname = "nitter.net"
[Cache]
listMinutes = 240 # how long to cache list info (not the tweets, so keep it high)
@ -13,9 +13,9 @@ rssMinutes = 10 # how long to cache rss queries
redisHost = "localhost" # Change to "nitter-redis" if using docker-compose
redisPort = 6379
redisPassword = ""
redisConnections = 20 # connection pool size
redisConnections = 20 # minimum open connections in pool
redisMaxConnections = 30
# max, new connections are opened when none are available, but if the pool size
# new connections are opened when none are available, but if the pool size
# goes above this, they're closed when released. don't worry about this unless
# you receive tons of requests per second
@ -23,15 +23,15 @@ redisMaxConnections = 30
hmacKey = "secretkey" # random key for cryptographic signing of video urls
base64Media = false # use base64 encoding for proxied media urls
enableRSS = true # set this to false to disable RSS feeds
enableDebug = false # enable request logs and debug endpoints
enableDebug = false # enable request logs and debug endpoints (/.tokens)
proxy = "" # http/https url, SOCKS proxies are not supported
proxyAuth = ""
tokenCount = 10
# minimum amount of usable tokens. tokens are used to authorize API requests,
# but they expire after ~1 hour, and have a limit of 187 requests.
# the limit gets reset every 15 minutes, and the pool is filled up so there's
# always at least $tokenCount usable tokens. again, only increase this if
# you receive major bursts all the time
# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
# the limits reset every 15 minutes, and the pool is filled up so there's
# always at least `tokenCount` usable tokens. only increase this if you receive
# major bursts all the time and don't have a rate limiting setup via e.g. nginx
# Change default preferences here, see src/prefs_impl.nim for a complete list
[Preferences]

2
nitter.nimble
View File

@ -12,7 +12,7 @@ bin = @["nitter"]
requires "nim >= 1.4.8"
requires "jester#baca3f"
requires "karax#9ee695b"
requires "karax#5cf360c"
requires "sass#7dfdd03"
requires "nimcrypto#4014ef9"
requires "markdown#158efe3"

156
src/api.nim
View File

@ -7,106 +7,82 @@ import experimental/parser as newParser
proc getGraphUser*(username: string): Future[User] {.async.} =
if username.len == 0: return
let
variables = """{
"screen_name": "$1",
"withSafetyModeUserFields": false,
"withSuperFollowsUserFields": false
}""" % [username]
js = await fetchRaw(graphUser ? {"variables": variables}, Api.userScreenName)
variables = %*{"screen_name": username}
params = {"variables": $variables, "features": gqlFeatures}
js = await fetchRaw(graphUser ? params, Api.userScreenName)
result = parseGraphUser(js)
proc getGraphUserById*(id: string): Future[User] {.async.} =
if id.len == 0 or id.any(c => not c.isDigit): return
let
variables = """{"userId": "$1", "withSuperFollowsUserFields": true}""" % [id]
js = await fetchRaw(graphUserById ? {"variables": variables}, Api.userRestId)
variables = %*{"userId": id}
params = {"variables": $variables, "features": gqlFeatures}
js = await fetchRaw(graphUserById ? params, Api.userRestId)
result = parseGraphUser(js)
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = userTweetsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
(url, apiId) = case kind
of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
of TimelineKind.media: (graphUserMedia, Api.userMedia)
js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, "user", after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = listTweetsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphListTweets ? params, Api.listTweets)
result = parseGraphTimeline(js, "list", after)
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
url = graphListBySlug ? {"variables": $variables}
result = parseGraphList(await fetch(url, Api.listBySlug))
variables = %*{"screenName": name, "listSlug": list}
params = {"variables": $variables, "features": gqlFeatures}
result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
proc getGraphList*(id: string): Future[List] {.async.} =
let
variables = %*{"listId": id, "withHighlightedLabel": false}
url = graphList ? {"variables": $variables}
result = parseGraphList(await fetch(url, Api.list))
variables = %*{"listId": id}
params = {"variables": $variables, "features": gqlFeatures}
result = parseGraphList(await fetch(graphListById ? params, Api.list))
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
if list.id.len == 0: return
var
variables = %*{
"listId": list.id,
"withSuperFollowsUserFields": false,
"withBirdwatchPivots": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withSuperFollowsTweetFields": false
"withReactionsPerspective": false
}
if after.len > 0:
variables["cursor"] = % after
let url = graphListMembers ? {"variables": $variables}
let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
if id.len == 0: return
let
ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
url = listTimeline ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)
proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
if id.len == 0: return
let
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
url = timeline / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)
proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
result = parseTimeline(await fetch(url, Api.timeline), after)
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let
ps = genParams({"screen_name": name, "trim_user": "true"},
count="18", ext=false)
url = photoRail ? ps
result = parsePhotoRail(await fetch(url, Api.timeline))
proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
when T is User:
const
searchMode = ("result_filter", "user")
parse = parseUsers
fetchFunc = fetchRaw
else:
const
searchMode = ("tweet_search_mode", "live")
parse = parseTimeline
fetchFunc = fetch
let q = genQueryParam(query)
if q.len == 0 or q == emptyQuery:
return Result[T](beginning: true, query: query)
let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
try:
result = parse(await fetchFunc(url, Api.search), after)
result.query = query
except InternalError:
return Result[T](beginning: true, query: query)
variables = tweetResultVariables % id
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweetResult ? params, Api.tweetResult)
result = parseGraphTweetResult(js)
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = tweetVariables % [id, cursor]
params = {"variables": variables, "features": tweetFeatures}
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweet ? params, Api.tweetDetail)
result = parseGraphConversation(js, id)
@ -119,9 +95,51 @@ proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
if after.len > 0:
result.replies = await getReplies(id, after)
proc getStatus*(id: string): Future[Tweet] {.async.} =
let url = status / (id & ".json") ? genParams()
result = parseStatus(await fetch(url, Api.status))
proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} =
let q = genQueryParam(query)
if q.len == 0 or q == emptyQuery:
return Result[Tweet](query: query, beginning: true)
var
variables = %*{
"rawQuery": q,
"count": 20,
"product": "Latest",
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
}
if after.len > 0:
variables["cursor"] = % after
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphSearch(await fetch(url, Api.search), after)
result.query = query
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
if query.text.len == 0:
return Result[User](query: query, beginning: true)
var url = userSearch ? {
"q": query.text,
"skip_status": "1",
"count": "20",
"page": page
}
result = parseUsers(await fetchRaw(url, Api.userSearch))
result.query = query
if page.len == 0:
result.bottom = "2"
elif page.allCharsInSet(Digits):
result.bottom = $(parseInt(page) + 1)
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let
ps = genParams({"screen_name": name, "trim_user": "true"},
count="18", ext=false)
url = photoRail ? ps
result = parsePhotoRail(await fetch(url, Api.timeline))
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
let client = newAsyncHttpClient(maxRedirects=0)

20
src/apiutils.nim
View File

@ -17,8 +17,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
result &= p
if ext:
result &= ("ext", "mediaStats")
result &= ("include_ext_alt_text", "true")
result &= ("include_ext_media_availability", "true")
result &= ("include_ext_alt_text", "1")
result &= ("include_ext_media_availability", "1")
if count.len > 0:
result &= ("count", count)
if cursor.len > 0:
@ -44,7 +44,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
})
template updateToken() =
if api != Api.search and resp.headers.hasKey(rlRemaining):
if resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
@ -67,14 +67,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
getContent()
# Twitter randomly returns 401 errors with an empty body quite often.
# Retrying the request usually works.
if resp.status == "401 Unauthorized" and result.len == 0:
getContent()
if resp.status == $Http503:
badClient = true
raise newException(InternalError, result)
if resp.status == $Http503:
badClient = true
raise newException(BadClientError, "Bad client")
if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip":
@ -90,6 +85,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
raise newException(InternalError, $url)
except InternalError as e:
raise e
except BadClientError as e:
release(token, used=true)
raise e
except Exception as e:
echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
if "length" notin e.msg and "descriptor" notin e.msg:

129
src/consts.nim
View File

@ -1,30 +1,28 @@
# SPDX-License-Identifier: AGPL-3.0-only
import uri, sequtils
import uri, sequtils, strutils
const
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
api = parseUri("https://api.twitter.com")
activate* = $(api / "1.1/guest/activate.json")
userShow* = api / "1.1/users/show.json"
photoRail* = api / "1.1/statuses/media_timeline.json"
status* = api / "1.1/statuses/show"
search* = api / "2/search/adaptive.json"
timelineApi = api / "2/timeline"
timeline* = timelineApi / "profile"
mediaTimeline* = timelineApi / "media"
listTimeline* = timelineApi / "list.json"
tweet* = timelineApi / "conversation"
userSearch* = api / "1.1/users/search.json"
graphql = api / "graphql"
graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
graphUser* = graphql / "8mPfHBetXOg-EHAyeVxUoA/UserByScreenName"
graphUserById* = graphql / "nI8WydSd-X-lQIVo6bdktQ/UserByRestId"
graphUserTweets* = graphql / "9rys0A7w1EyqVd2ME0QCJg/UserTweets"
graphUserTweetsAndReplies* = graphql / "ehMCHF3Mkgjsfz_aImqOsg/UserTweetsAndReplies"
graphUserMedia* = graphql / "MA_EP2a21zpzNWKRkaPBMg/UserMedia"
graphTweet* = graphql / "6I7Hm635Q6ftv69L8VrSeQ/TweetDetail"
graphTweetResult* = graphql / "rt-rHeSJ-2H9O9gxWQcPcg/TweetResultByRestId"
graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"
timelineParams* = {
"include_profile_interstitial_type": "0",
@ -35,58 +33,85 @@ const
"include_mute_edge": "0",
"include_can_dm": "0",
"include_can_media_tag": "1",
"include_ext_is_blue_verified": "1",
"skip_status": "1",
"cards_platform": "Web-12",
"include_cards": "1",
"include_composer_source": "false",
"include_composer_source": "0",
"include_reply_count": "1",
"tweet_mode": "extended",
"include_entities": "true",
"include_user_entities": "true",
"include_ext_media_color": "false",
"send_error_codes": "true",
"simple_quoted_tweet": "true",
"include_quote_count": "true"
"include_entities": "1",
"include_user_entities": "1",
"include_ext_media_color": "0",
"send_error_codes": "1",
"simple_quoted_tweet": "1",
"include_quote_count": "1"
}.toSeq
searchParams* = {
"query_source": "typed_query",
"pc": "1",
"spelling_corrections": "1"
}.toSeq
## top: nothing
## latest: "tweet_search_mode: live"
## user: "result_filter: user"
## photos: "result_filter: photos"
## videos: "result_filter: videos"
gqlFeatures* = """{
"blue_business_profile_image_shape_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": false,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
"interactive_text_enabled": false,
"longform_notetweets_consumption_enabled": true,
"longform_notetweets_richtext_consumption_enabled": true,
"longform_notetweets_rich_text_read_enabled": false,
"responsive_web_edit_tweet_api_enabled": false,
"responsive_web_enhance_cards_enabled": false,
"responsive_web_graphql_exclude_directive_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": false,
"responsive_web_text_conversations_enabled": false,
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"spaces_2022_h2_clipping": true,
"spaces_2022_h2_spaces_communities": true,
"standardized_nudges_misinfo": false,
"tweet_awards_web_tipping_enabled": false,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"tweetypie_unmention_optimization_enabled": false,
"verified_phone_label_enabled": false,
"vibe_api_enabled": false,
"view_counts_everywhere_api_enabled": false
}""".replace(" ", "").replace("\n", "")
tweetVariables* = """{
"focalTweetId": "$1",
$2
"includePromotedContent": false,
"withBirdwatchNotes": false,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false
}"""
tweetResultVariables* = """{
"tweetId": "$1",
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false,
"withCommunity": false
}"""
userTweetsVariables* = """{
"userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withSuperFollowsTweetFields": false,
"withSuperFollowsUserFields": false,
"withVoice": false,
"withV2Timeline": true
}"""
tweetFeatures* = """{
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": false,
"standardized_nudges_misinfo": false,
"verified_phone_label_enabled": false,
"responsive_web_twitter_blue_verified_badge_is_enabled": false,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"view_counts_everywhere_api_enabled": false,
"responsive_web_edit_tweet_api_enabled": false,
"tweetypie_unmention_optimization_enabled": false,
"vibe_api_enabled": false,
"longform_notetweets_consumption_enabled": true,
"responsive_web_text_conversations_enabled": false,
"responsive_web_enhance_cards_enabled": false,
"interactive_text_enabled": false
listTweetsVariables* = """{
"listId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false
}"""

4
src/experimental/parser.nim
View File

@ -1,2 +1,2 @@
import parser/[user, graphql, timeline]
export user, graphql, timeline
import parser/[user, graphql]
export user, graphql

1
src/experimental/parser/graphql.nim
View File

@ -11,6 +11,7 @@ proc parseGraphUser*(json: string): User =
result = toUser raw.data.user.result.legacy
result.id = raw.data.user.result.restId
result.verified = result.verified or raw.data.user.result.isBlueVerified
proc parseGraphListMembers*(json, cursor: string): Result[User] =
result = Result[User](

30
src/experimental/parser/timeline.nim
View File

@ -1,30 +0,0 @@
import std/[strutils, tables]
import jsony
import user, ../types/timeline
from ../../types import Result, User
proc getId(id: string): string {.inline.} =
let start = id.rfind("-")
if start < 0: return id
id[start + 1 ..< id.len]
proc parseUsers*(json: string; after=""): Result[User] =
result = Result[User](beginning: after.len == 0)
let raw = json.fromJson(Search)
if raw.timeline.instructions.len == 0:
return
for i in raw.timeline.instructions:
if i.addEntries.entries.len > 0:
for e in i.addEntries.entries:
let id = e.entryId.getId
if e.entryId.startsWith("user"):
if id in raw.globalObjects.users:
result.content.add toUser raw.globalObjects.users[id]
elif e.entryId.startsWith("cursor"):
let cursor = e.content.operation.cursor
if cursor.cursorType == "Top":
result.top = cursor.value
elif cursor.cursorType == "Bottom":
result.bottom = cursor.value

2
src/experimental/parser/unifiedcard.nim
View File

@ -84,6 +84,8 @@ proc parseUnifiedCard*(json: string): Card =
component.parseMedia(card, result)
of buttonGroup:
discard
of ComponentType.hidden:
result.kind = CardKind.hidden
of ComponentType.unknown:
echo "ERROR: Unknown component type: ", json

11
src/experimental/parser/user.nim
View File

@ -2,7 +2,7 @@ import std/[algorithm, unicode, re, strutils, strformat, options, nre]
import jsony
import utils, slices
import ../types/user as userType
from ../../types import User, Error
from ../../types import Result, User, Error
let
unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
@ -76,3 +76,12 @@ proc parseUser*(json: string; username=""): User =
else: echo "[error - parseUser]: ", error
result = toUser json.fromJson(RawUser)
proc parseUsers*(json: string; after=""): Result[User] =
result = Result[User](beginning: after.len == 0)
# starting with '{' means it's an error
if json[0] == '[':
let raw = json.fromJson(seq[RawUser])
for user in raw:
result.content.add user.toUser

1
src/experimental/types/graphuser.nim
View File

@ -11,4 +11,5 @@ type
UserResult = object
legacy*: RawUser
restId*: string
isBlueVerified*: bool
reason*: Option[string]

6
src/experimental/types/unifiedcard.nim
View File

@ -17,6 +17,7 @@ type
twitterListDetails
communityDetails
mediaWithDetailsHorizontal
hidden
unknown
Component* = object
@ -71,11 +72,11 @@ type
Text = object
content: string
HasTypeField = Component | Destination | MediaEntity | AppStoreData
TypeField = Component | Destination | MediaEntity | AppStoreData
converter fromText*(text: Text): string = text.content
proc renameHook*(v: var HasTypeField; fieldName: var string) =
proc renameHook*(v: var TypeField; fieldName: var string) =
if fieldName == "type":
fieldName = "kind"
@ -89,6 +90,7 @@ proc enumHook*(s: string; v: var ComponentType) =
of "twitter_list_details": twitterListDetails
of "community_details": communityDetails
of "media_with_details_horizontal": mediaWithDetailsHorizontal
of "commerce_drop_details": hidden
else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
proc enumHook*(s: string; v: var AppType) =

6
src/http_pool.nim
View File

@ -42,5 +42,11 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
except ProtocolError:
# Twitter closed the connection, retry
body
except BadClientError:
# Twitter returned 503, we need a new client
pool.release(c, true)
badClient = false
c = pool.acquire(heads)
body
finally:
pool.release(c, badClient)

14
src/nitter.nim
View File

@ -85,19 +85,23 @@ routes:
resp Http500, showError(
&"An error occurred, please {link} with the URL you tried to visit.", cfg)
error BadClientError:
echo error.exc.name, ": ", error.exc.msg
resp Http500, showError("Network error occured, please try again.", cfg)
error RateLimitError:
const link = a("another instance", href = instancesUrl)
resp Http429, showError(
&"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
extend unsupported, ""
extend preferences, ""
extend resolver, ""
extend rss, ""
extend status, ""
extend search, ""
extend timeline, ""
extend list, ""
extend status, ""
extend media, ""
extend list, ""
extend preferences, ""
extend resolver, ""
extend embed, ""
extend debug, ""
extend unsupported, ""

134
src/parser.nim
View File

@ -4,6 +4,8 @@ import packedjson, packedjson/deserialiser
import types, parserutils, utils
import experimental/parser/unifiedcard
proc parseGraphTweet(js: JsonNode): Tweet
proc parseUser(js: JsonNode; id=""): User =
if js.isNull: return
result = User(
@ -19,13 +21,20 @@ proc parseUser(js: JsonNode; id=""): User =
tweets: js{"statuses_count"}.getInt,
likes: js{"favourites_count"}.getInt,
media: js{"media_count"}.getInt,
verified: js{"verified"}.getBool,
verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool,
protected: js{"protected"}.getBool,
joinDate: js{"created_at"}.getTime
)
result.expandUserEntities(js)
proc parseGraphUser(js: JsonNode): User =
let user = ? js{"user_results", "result"}
result = parseUser(user{"legacy"})
if "is_blue_verified" in user:
result.verified = true
proc parseGraphList*(js: JsonNode): List =
if js.isNull: return
@ -38,11 +47,11 @@ proc parseGraphList*(js: JsonNode): List =
result = List(
id: list{"id_str"}.getStr,
name: list{"name"}.getStr,
username: list{"user", "legacy", "screen_name"}.getStr,
userId: list{"user", "rest_id"}.getStr,
username: list{"user_results", "result", "legacy", "screen_name"}.getStr,
userId: list{"user_results", "result", "rest_id"}.getStr,
description: list{"description"}.getStr,
members: list{"member_count"}.getInt,
banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
banner: list{"custom_banner_media", "media_info", "original_img_url"}.getImageStr
)
proc parsePoll(js: JsonNode): Poll =
@ -213,10 +222,18 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
if js{"is_quote_status"}.getBool:
result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
# legacy
with rt, js{"retweeted_status_id_str"}:
result.retweet = some Tweet(id: rt.getId)
return
# graphql
with rt, js{"retweeted_status_result", "result"}:
# needed due to weird edgecase where the actual tweet data isn't included
if "legacy" in rt:
result.retweet = some parseGraphTweet(rt)
return
if jsCard.kind != JNull:
let name = jsCard{"name"}.getStr
if "poll" in name:
@ -237,7 +254,10 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
of "video":
result.video = some(parseVideo(m))
with user, m{"additional_media_info", "source_user"}:
result.attribution = some(parseUser(user))
if user{"id"}.getInt > 0:
result.attribution = some(parseUser(user))
else:
result.attribution = some(parseGraphUser(user))
of "animated_gif":
result.gif = some(parseGif(m))
else: discard
@ -299,19 +319,6 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet
proc parseStatus*(js: JsonNode): Tweet =
with e, js{"errors"}:
if e.getError in {tweetNotFound, tweetUnavailable, tweetCensored, doesntExist,
tweetNotAuthorized, suspended}:
return
result = parseTweet(js, js{"card"})
if not result.isNil:
result.user = parseUser(js{"user"})
with quote, js{"quoted_status"}:
result.quote = some parseStatus(js{"quoted_status"})
proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
if js.kind != JArray or js.len == 0:
return
@ -352,7 +359,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
result.top = e.getCursor
elif "cursor-bottom" in entry:
result.bottom = e.getCursor
elif entry.startsWith("sq-C"):
elif entry.startsWith("sq-cursor"):
with cursor, e{"content", "operation", "cursor"}:
if cursor{"cursorType"}.getStr == "Bottom":
result.bottom = cursor{"value"}.getStr
@ -373,9 +380,20 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =
result.add GalleryPhoto(url: url, tweetId: $t.id)
proc parseGraphTweet(js: JsonNode): Tweet =
if js.kind == JNull or js{"__typename"}.getStr == "TweetUnavailable":
if js.kind == JNull:
return Tweet(available: false)
case js{"__typename"}.getStr
of "TweetUnavailable":
return Tweet(available: false)
of "TweetTombstone":
return Tweet(
available: false,
text: js{"tombstone", "text"}.getTombstone
)
of "TweetWithVisibilityResults":
return parseGraphTweet(js{"tweet"})
var jsCard = copy(js{"card", "legacy"})
if jsCard.kind != JNull:
var values = newJObject()
@ -384,7 +402,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
jsCard["binding_values"] = values
result = parseTweet(js{"legacy"}, jsCard)
result.user = parseUser(js{"core", "user_results", "result", "legacy"})
result.user = parseGraphUser(js{"core"})
with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
result.expandNoteTweetEntities(noteTweet)
@ -407,10 +425,14 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
result.self = true
proc parseGraphTweetResult*(js: JsonNode): Tweet =
with tweet, js{"data", "tweetResult", "result"}:
result = parseGraphTweet(tweet)
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
let instructions = ? js{"data", "threaded_conversation_with_injections", "instructions"}
if instructions.len == 0:
return
@ -418,12 +440,25 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
let entryId = e{"entryId"}.getStr
# echo entryId
if entryId.startsWith("tweet"):
let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
if $tweet.id == tweetId:
if $tweet.id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
elif entryId.startsWith("tombstone"):
let id = entryId.getId()
let tweet = Tweet(
id: parseBiggestInt(id),
available: false,
text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
)
if id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
@ -435,3 +470,50 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
result.replies.content.add thread
elif entryId.startsWith("cursor-bottom"):
result.replies.bottom = e{"content", "itemContent", "value"}.getStr
proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
let instructions =
if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"}
else: ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
if instructions.len == 0:
return
for i in instructions:
if i{"type"}.getStr == "TimelineAddEntries":
for e in i{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.content.add tweet
elif entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr
proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
if instructions.len == 0:
return
for instruction in instructions:
let typ = instruction{"type"}.getStr
if typ == "TimelineAddEntries":
for e in instructions[0]{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.content.add tweet
elif entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr
elif typ == "TimelineReplaceEntry":
if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
result.bottom = instruction{"entry", "content", "value"}.getStr

2
src/parserutils.nim
View File

@ -130,7 +130,7 @@ proc getBanner*(js: JsonNode): string =
return
proc getTombstone*(js: JsonNode): string =
result = js{"tombstoneInfo", "richText", "text"}.getStr
result = js{"text"}.getStr
result.removeSuffix(" Learn more")
proc getMp4Resolution*(url: string): int =

2
src/redis_cache.nim
View File

@ -153,7 +153,7 @@ proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
if tweet != redisNil:
tweet.deserialize(Tweet)
else:
result = await getStatus($id)
result = await getGraphTweetResult($id)
if not result.isNil:
await cache(result)

3
src/routes/list.nim
View File

@ -6,7 +6,6 @@ import jester
import router_utils
import ".."/[types, redis_cache, api]
import ../views/[general, timeline, list]
export getListTimeline, getGraphList
template respList*(list, timeline, title, vnode: typed) =
if list.id.len == 0 or list.name.len == 0:
@ -39,7 +38,7 @@ proc createListRouter*(cfg: Config) =
let
prefs = cookiePrefs()
list = await getCachedList(id=(@"id"))
timeline = await getListTimeline(list.id, getCursor())
timeline = await getGraphListTweets(list.id, getCursor())
vnode = renderTimelineTweets(timeline, prefs, request.path)
respList(list, timeline, list.title, vnode)

6
src/routes/rss.nim
View File

@ -28,7 +28,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
var q = query
q.fromUser = names
profile = Profile(
tweets: await getSearch[Tweet](q, after),
tweets: await getGraphSearch(q, after),
# this is kinda dumb
user: User(
username: name,
@ -78,7 +78,7 @@ proc createRssRouter*(cfg: Config) =
if rss.cursor.len > 0:
respRss(rss, "Search")
let tweets = await getSearch[Tweet](query, cursor)
let tweets = await getGraphSearch(query, cursor)
rss.cursor = tweets.bottom
rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
@ -159,7 +159,7 @@ proc createRssRouter*(cfg: Config) =
let
list = await getCachedList(id=id)
timeline = await getListTimeline(list.id, cursor)
timeline = await getGraphListTweets(list.id, cursor)
rss.cursor = timeline.bottom
rss.feed = renderListRss(timeline.content, list, cfg)

8
src/routes/search.nim
View File

@ -27,11 +27,15 @@ proc createSearchRouter*(cfg: Config) =
of users:
if "," in q:
redirect("/" & q)
let users = await getSearch[User](query, getCursor())
var users: Result[User]
try:
users = await getUserSearch(query, getCursor())
except InternalError:
users = Result[User](beginning: true, query: query)
resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
of tweets:
let
tweets = await getSearch[Tweet](query, getCursor())
tweets = await getGraphSearch(query, getCursor())
rss = "/search/rss?" & genQueryUrl(query)
resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
request, cfg, prefs, title, rss=rss)

10
src/routes/status.nim
View File

@ -16,17 +16,21 @@ proc createStatusRouter*(cfg: Config) =
router status:
get "/@name/status/@id/?":
cond '.' notin @"name"
cond not @"id".any(c => not c.isDigit)
let id = @"id"
if id.len > 19 or id.any(c => not c.isDigit):
resp Http404, showError("Invalid tweet ID", cfg)
let prefs = cookiePrefs()
# used for the infinite scroll feature
if @"scroll".len > 0:
let replies = await getReplies(@"id", getCursor())
let replies = await getReplies(id, getCursor())
if replies.content.len == 0:
resp Http404, ""
resp $renderReplies(replies, prefs, getPath())
let conv = await getTweet(@"id", getCursor())
let conv = await getTweet(id, getCursor())
if conv == nil:
echo "nil conv"

15
src/routes/timeline.nim
View File

@ -47,10 +47,10 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
let
timeline =
case query.kind
of posts: getTimeline(userId, after)
of replies: getTimeline(userId, after, replies=true)
of media: getMediaTimeline(userId, after)
else: getSearch[Tweet](query, after)
of posts: getGraphUserTweets(userId, TimelineKind.tweets, after)
of replies: getGraphUserTweets(userId, TimelineKind.replies, after)
of media: getGraphUserTweets(userId, TimelineKind.media, after)
else: getGraphSearch(query, after)
rail =
skipIf(skipRail or query.kind == media, @[]):
@ -64,6 +64,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
let tweet = await getCachedTweet(user.pinnedTweet)
if not tweet.isNil:
tweet.pinned = true
tweet.user = user
pinned = some tweet
result = Profile(
@ -82,7 +83,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
rss, after: string): Future[string] {.async.} =
if query.fromUser.len != 1:
let
timeline = await getSearch[Tweet](query, after)
timeline = await getGraphSearch(query, after)
html = renderTweetSearch(timeline, prefs, getPath())
return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
@ -123,7 +124,7 @@ proc createTimelineRouter*(cfg: Config) =
get "/@name/?@tab?/?":
cond '.' notin @"name"
cond @"name" notin ["pic", "gif", "video"]
cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
cond @"tab" in ["with_replies", "media", "search", ""]
let
prefs = cookiePrefs()
@ -137,7 +138,7 @@ proc createTimelineRouter*(cfg: Config) =
# used for the infinite scroll feature
if @"scroll".len > 0:
if query.fromUser.len != 1:
var timeline = await getSearch[Tweet](query, after)
var timeline = await getGraphSearch(query, after)
if timeline.content.len == 0: resp Http404
timeline.beginning = true
resp $renderTweetSearch(timeline, prefs, getPath())

29
src/tokens.nim
View File

@ -1,8 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, httpclient, times, sequtils, json, random
import strutils, tables
import zippy
import types, consts, http_pool
import types, consts
const
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
@ -11,11 +10,12 @@ const
failDelay = initDuration(minutes=30)
var
clientPool: HttpPool
tokenPool: seq[Token]
lastFailed: Time
enableLogging = false
let headers = newHttpHeaders({"authorization": auth})
template log(str) =
if enableLogging: echo "[tokens] ", str
@ -41,10 +41,12 @@ proc getPoolJson*(): JsonNode =
let
maxReqs =
case api
of Api.listMembers, Api.listBySlug, Api.list,
Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
of Api.timeline: 187
else: 180
of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
Api.userRestId, Api.userScreenName,
Api.tweetDetail, Api.tweetResult, Api.search: 500
of Api.userSearch: 900
reqs = maxReqs - token.apis[api].remaining
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
@ -65,18 +67,12 @@ proc fetchToken(): Future[Token] {.async.} =
if getTime() - lastFailed < failDelay:
raise rateLimitError()
let headers = newHttpHeaders({
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.5",
"connection": "keep-alive",
"authorization": auth
})
let client = newAsyncHttpClient(headers=headers)
try:
let
resp = clientPool.use(headers): await c.postContent(activate)
tokNode = parseJson(uncompress(resp))["guest_token"]
resp = await client.postContent(activate)
tokNode = parseJson(resp)["guest_token"]
tok = tokNode.getStr($(tokNode.getInt))
time = getTime()
@ -86,6 +82,8 @@ proc fetchToken(): Future[Token] {.async.} =
if "Try again" notin e.msg:
echo "[tokens] fetching tokens paused, resuming in 30 minutes"
lastFailed = getTime()
finally:
client.close()
proc expired(token: Token): bool =
let time = getTime()
@ -158,7 +156,6 @@ proc poolTokens*(amount: int) {.async.} =
tokenPool.add newToken
proc initTokenPool*(cfg: Config) {.async.} =
clientPool = HttpPool()
enableLogging = cfg.enableDebug
while true:

18
src/types.nim
View File

@ -7,19 +7,28 @@ genPrefsType()
type
RateLimitError* = object of CatchableError
InternalError* = object of CatchableError
BadClientError* = object of CatchableError
TimelineKind* {.pure.} = enum
tweets
replies
media
Api* {.pure.} = enum
tweetDetail
userShow
tweetResult
timeline
search
tweet
userSearch
list
listBySlug
listMembers
listTweets
userRestId
userScreenName
status
userTweets
userTweetsAndReplies
userMedia
RateLimit* = object
remaining*: int
@ -36,8 +45,10 @@ type
null = 0
noUserMatches = 17
protectedUser = 22
missingParams = 25
couldntAuth = 32
doesntExist = 34
invalidParam = 47
userNotFound = 50
suspended = 63
rateLimited = 88
@ -150,6 +161,7 @@ type
imageDirectMessage = "image_direct_message"
audiospace = "audiospace"
newsletterPublication = "newsletter_publication"
hidden
unknown
Card* = object

25
src/views/renderutils.nim
View File

@ -59,8 +59,7 @@ proc buttonReferer*(action, text, path: string; class=""; `method`="post"): VNod
proc genCheckbox*(pref, label: string; state: bool): VNode =
buildHtml(label(class="pref-group checkbox-container")):
text label
if state: input(name=pref, `type`="checkbox", checked="")
else: input(name=pref, `type`="checkbox")
input(name=pref, `type`="checkbox", checked=state)
span(class="checkbox")
proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
@ -68,20 +67,15 @@ proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true
buildHtml(tdiv(class=("pref-group pref-input " & class))):
if label.len > 0:
label(`for`=pref): text label
if autofocus and state.len == 0:
input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
else:
input(name=pref, `type`="text", placeholder=p, value=state)
input(name=pref, `type`="text", placeholder=p, value=state, autofocus=(autofocus and state.len == 0))
proc genSelect*(pref, label, state: string; options: seq[string]): VNode =
buildHtml(tdiv(class="pref-group pref-input")):
label(`for`=pref): text label
select(name=pref):
for opt in options:
if opt == state:
option(value=opt, selected=""): text opt
else:
option(value=opt): text opt
option(value=opt, selected=(opt == state)):
text opt
proc genDate*(pref, state: string): VNode =
buildHtml(span(class="date-input")):
@ -93,12 +87,9 @@ proc genImg*(url: string; class=""): VNode =
img(src=getPicUrl(url), class=class, alt="")
proc getTabClass*(query: Query; tab: QueryKind): string =
result = "tab-item"
if query.kind == tab:
result &= " active"
if query.kind == tab: "tab-item active"
else: "tab-item"
proc getAvatarClass*(prefs: Prefs): string =
if prefs.squareAvatars:
"avatar"
else:
"avatar round"
if prefs.squareAvatars: "avatar"
else: "avatar round"

10
src/views/search.nim
View File

@ -63,12 +63,10 @@ proc renderSearchPanel*(query: Query): VNode =
hiddenField("f", "tweets")
genInput("q", "", query.text, "Enter search...", class="pref-inline")
button(`type`="submit"): icon "search"
if isPanelOpen(query):
input(id="search-panel-toggle", `type`="checkbox", checked="")
else:
input(id="search-panel-toggle", `type`="checkbox")
label(`for`="search-panel-toggle"):
icon "down"
input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
label(`for`="search-panel-toggle"): icon "down"
tdiv(class="search-panel"):
for f in @["filter", "exclude"]:
span(class="search-title"): text capitalize(f)

23
src/views/tweet.nim
View File

@ -106,14 +106,10 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
else: vidUrl
case playbackType
of mp4:
if prefs.muteVideos:
video(poster=thumb, controls="", muted=""):
source(src=source, `type`="video/mp4")
else:
video(poster=thumb, controls=""):
source(src=source, `type`="video/mp4")
video(poster=thumb, controls="", muted=prefs.muteVideos):
source(src=source, `type`="video/mp4")
of m3u8, vmap:
video(poster=thumb, data-url=source, data-autoload="false")
video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
tdiv(class="overlay-circle"): span(class="overlay-triangle")
verbatim "</div>"
@ -127,14 +123,9 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
buildHtml(tdiv(class="attachments media-gif")):
tdiv(class="gallery-gif", style={maxHeight: "unset"}):
tdiv(class="attachment"):
let thumb = getSmallPic(gif.thumb)
let url = getPicUrl(gif.url)
if prefs.autoplayGifs:
video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""):
source(src=url, `type`="video/mp4")
else:
video(class="gif", poster=thumb, controls="", muted="", loop=""):
source(src=url, `type`="video/mp4")
video(class="gif", poster=getSmallPic(gif.thumb), autoplay=prefs.autoplayGifs,
controls="", muted="", loop=""):
source(src=getPicUrl(gif.url), `type`="video/mp4")
proc renderPoll(poll: Poll): VNode =
buildHtml(tdiv(class="poll")):
@ -328,7 +319,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
if tweet.attribution.isSome:
renderAttribution(tweet.attribution.get(), prefs)
if tweet.card.isSome:
if tweet.card.isSome and tweet.card.get().kind != hidden:
renderCard(tweet.card.get(), prefs, path)
if tweet.photos.len > 0:

10
tests/test_card.py
View File

@ -3,11 +3,6 @@ from parameterized import parameterized
card = [
['Thom_Wolf/status/1122466524860702729',
'facebookresearch/fairseq',
'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
'github.com', True],
['nim_lang/status/1136652293510717440',
'Version 0.20.0 released',
'We are very proud to announce Nim version 0.20. This is a massive release, both literally and figuratively. It contains more than 1,000 commits and it marks our release candidate for version 1.0!',
@ -25,6 +20,11 @@ card = [
]
no_thumb = [
['Thom_Wolf/status/1122466524860702729',
'facebookresearch/fairseq',
'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
'github.com'],
['brent_p/status/1088857328680488961',
'Hts Nim Sugar',
'hts-nim is a library that allows one to use htslib via the nim programming language. Nim is a garbage-collected language that compiles to C and often has similar performance. I have become very...',

11
tests/test_profile.py
View File

@ -17,11 +17,6 @@ protected = [
invalid = [['thisprofiledoesntexist'], ['%']]
banner_color = [
['nim_lang', '22, 25, 32'],
['rustlang', '35, 31, 32']
]
banner_image = [
['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
]
@ -74,12 +69,6 @@ class ProfileTest(BaseTestCase):
self.open_nitter('user')
self.assert_text('User "user" has been suspended')
@parameterized.expand(banner_color)
def test_banner_color(self, username, color):
self.open_nitter(username)
banner = self.find_element(Profile.banner + ' a')
self.assertIn(color, banner.value_of_css_property('background-color'))
@parameterized.expand(banner_image)
def test_banner_image(self, username, url):
self.open_nitter(username)