lasers
/
nitter
mirror of https://github.com/zedeus/nitter
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add support for displaying Community Notes

This commit is contained in:
Aaron Hill 2023-09-17 12:15:15 -04:00
parent 14f9a092d8
commit 2966e75f6a
No known key found for this signature in database
GPG Key ID: B4087E510E98B164
10 changed files with 214 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/api.nim
View File

@ -31,7 +31,7 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
of TimelineKind.media: (graphUserMedia, Api.userMedia)
js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, "user", after)
result = await parseGraphTimeline(js, "user", after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
@ -40,7 +40,7 @@ proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
variables = listTweetsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphListTweets ? params, Api.listTweets)
result = parseGraphTimeline(js, "list", after).tweets
result = (await parseGraphTimeline(js, "list", after)).tweets
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
@ -59,7 +59,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
var
variables = %*{
"listId": list.id,
"withBirdwatchPivots": false,
"withBirdwatchPivots": true,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
@ -75,16 +75,17 @@ proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
variables = """{"rest_id": "$1"}""" % id
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweetResult ? params, Api.tweetResult)
result = parseGraphTweetResult(js)
result = await parseGraphTweetResult(js)
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
cursor = if after.len > 0: "\"cursorp\":\"$1\"," % after else: ""
variables = tweetVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweet ? params, Api.tweetDetail)
result = parseGraphConversation(js, id)
result = await parseGraphConversation(js, id)
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
result = (await getGraphTweet(id, after)).replies
@ -112,7 +113,7 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
if after.len > 0:
variables["cursor"] = % after
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphSearch(await fetch(url, Api.search), after)
result = await parseGraphSearch(await fetch(url, Api.search), after)
result.query = query
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
@ -138,10 +139,10 @@ proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
ps = genParams({"screen_name": name, "trim_user": "true"},
count="18", ext=false)
url = photoRail ? ps
result = parsePhotoRail(await fetch(url, Api.photoRail))
result = await parsePhotoRail(await fetch(url, Api.photoRail))
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
let client = newAsyncHttpClient(maxRedirects=0)
let client = newAsyncHttpClient(maxRedirects=5)
try:
let resp = await client.request(url, HttpHead)
result = resp.headers["location"].replaceUrls(prefs)

50
src/apiutils.nim
View File

@ -46,12 +46,16 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
return getOauth1RequestHeader(params)["authorization"]
proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
let header = getOauthHeader(url, oauthToken, oauthTokenSecret)
proc genHeaders*(url: string, account: GuestAccount, browserapi: bool): HttpHeaders =
let authorization = if browserapi:
"Bearer " & bearerToken
else:
getOauthHeader(url, account.oauthToken, account.oauthSecret)
result = newHttpHeaders({
"connection": "keep-alive",
"authorization": header,
"authorization": authorization,
"content-type": "application/json",
"x-twitter-active-user": "yes",
"authority": "api.twitter.com",
@ -61,6 +65,9 @@ proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
"DNT": "1"
})
if browserapi:
result["x-guest-token"] = account.guestToken
template fetchImpl(result, fetchBody) {.dirty.} =
once:
pool = HttpPool()
@ -72,7 +79,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
try:
var resp: AsyncResponse
pool.use(genHeaders($url, account.oauthToken, account.oauthSecret)):
pool.use(genHeaders($url, account, browserApi)):
template getContent =
resp = await c.get($url)
result = await resp.body
@ -133,7 +140,16 @@ template retry(bod) =
echo "[accounts] Rate limited, retrying ", api, " request..."
bod
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
# `fetch` and `fetchRaw` operate in two modes:
# 1. `browserApi` is false (the normal mode). We call 'https://api.twitter.com' endpoints,
# using the oauth token for a particular GuestAccount. This is used for everything
# except Community Notes.
# 2. `browserApi` is true. We call 'https://twitter.com/i/api' endpoints,
# using a hardcoded Bearer token, and an 'x-guest-token' header for a particular GuestAccount.
# This is currently only used for retrieving Community Notes, which do not seem to be available
# through any of the 'https://api.twitter.com' endpoints.
#
proc fetch*(url: Uri; api: Api, browserApi = false): Future[JsonNode] {.async.} =
retry:
var body: string
fetchImpl body:
@ -149,9 +165,31 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
invalidate(account)
raise rateLimitError()
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
proc fetchRaw*(url: Uri; api: Api, browserApi = false): Future[string] {.async.} =
retry:
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):
echo resp.status, ": ", result, " --- url: ", url
result.setLen(0)
proc parseCommunityNote(js: JsonNode): Option[CommunityNote] =
if js.isNull: return
var pivot = js{"data", "tweetResult", "result", "birdwatch_pivot"}
if pivot.isNull: return
result = some CommunityNote(
title: pivot{"title"}.getStr,
subtitle: expandCommunityNoteEntities(pivot{"subtitle"}),
footer: expandCommunityNoteEntities(pivot{"footer"}),
url: pivot{"destinationUrl"}.getStr
)
proc getCommunityNote*(id: string): Future[Option[CommunityNote]] {.async.} =
if id.len == 0: return
let
variables = browserApiTweetVariables % [id]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(browserGraphTweetResultByRestId ? params, Api.tweetResultByRestId, true)
result = parseCommunityNote(js)

14
src/consts.nim
View File

@ -4,8 +4,11 @@ import uri, sequtils, strutils
const
consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
bearerToken* = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
api = parseUri("https://api.twitter.com")
# This is the API accessed by the browser, which is different from the developer API
browserApi = parseUri("https://twitter.com/i/api")
activate* = $(api / "1.1/guest/activate.json")
photoRail* = api / "1.1/statuses/media_timeline.json"
@ -25,6 +28,8 @@ const
graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
browserGraphTweetResultByRestId* = browserApi / "/graphql/DJS3BdhUhcaEpZ7B7irJDg/TweetResultByRestId"
timelineParams* = {
"include_can_media_tag": "1",
"include_cards": "1",
@ -91,11 +96,18 @@ const
$2
"includeHasBirdwatchNotes": false,
"includePromotedContent": false,
"withBirdwatchNotes": false,
"withBirdwatchNotes": true,
"withVoice": false,
"withV2Timeline": true
}""".replace(" ", "").replace("\n", "")
browserApiTweetVariables* = """{
"tweetId": "$1",
"includePromotedContent": false,
"withCommunity": false,
"withVoice": false
}""".replace(" ", "").replace("\n", "")
# oldUserTweetsVariables* = """{
# "userId": "$1", $2
# "count": 20,

55
src/parser.nim
View File

@ -1,10 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, options, times, math
import asyncdispatch, strutils, options, times, math
import packedjson, packedjson/deserialiser
import types, parserutils, utils
import experimental/parser/unifiedcard
import apiutils
proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet
proc parseGraphTweet(js: JsonNode; isLegacy=false): Future[Tweet] {.async.}
proc parseUser(js: JsonNode; id=""): User =
if js.isNull: return
@ -200,7 +201,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
result.url.len == 0 or result.url.startsWith("card://"):
result.url = getPicUrl(result.image)
proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(), hasBirdwatchNotes = false): Future[Tweet] {.async.} =
if js.isNull: return
result = Tweet(
id: js{"id_str"}.getId,
@ -216,11 +217,14 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
retweets: js{"retweet_count"}.getInt,
likes: js{"favorite_count"}.getInt,
quotes: js{"quote_count"}.getInt
)
),
)
result.expandTweetEntities(js)
if hasBirdwatchNotes:
result.communityNote = await getCommunityNote(js{"id_str"}.getStr)
# fix for pinned threads
if result.hasThread and result.threadId == 0:
result.threadId = js{"self_thread", "id_str"}.getId
@ -239,7 +243,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
with rt, js{"retweeted_status_result", "result"}:
# needed due to weird edgecase where the actual tweet data isn't included
if "legacy" in rt:
result.retweet = some parseGraphTweet(rt)
result.retweet = some await parseGraphTweet(rt)
return
if jsCard.kind != JNull:
@ -289,14 +293,15 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
result.text.removeSuffix(" Learn more.")
result.available = false
proc parsePhotoRail*(js: JsonNode): PhotoRail =
proc parsePhotoRail*(js: JsonNode): Future[PhotoRail] {.async.} =
with error, js{"error"}:
if error.getStr == "Not authorized.":
return
for tweet in js:
let
t = parseTweet(tweet, js{"tweet_card"})
# We don't support community notes here (TODO: see if this is possible)
t = await parseTweet(tweet, js{"tweet_card"}, false)
url = if t.photos.len > 0: t.photos[0]
elif t.video.isSome: get(t.video).thumb
elif t.gif.isSome: get(t.gif).thumb
@ -306,7 +311,7 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =
if url.len == 0: continue
result.add GalleryPhoto(url: url, tweetId: $t.id)
proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
proc parseGraphTweet(js: JsonNode; isLegacy=false): Future[Tweet] {.async.} =
if js.kind == JNull:
return Tweet()
@ -322,7 +327,7 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
of "TweetPreviewDisplay":
return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
of "TweetWithVisibilityResults":
return parseGraphTweet(js{"tweet"}, isLegacy)
return await parseGraphTweet(js{"tweet"}, isLegacy)
if not js.hasKey("legacy"):
return Tweet()
@ -334,7 +339,7 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
values[val["key"].getStr] = val["value"]
jsCard["binding_values"] = values
result = parseTweet(js{"legacy"}, jsCard)
result = await parseTweet(js{"legacy"}, jsCard, js{"has_birdwatch_notes"}.getBool)
result.id = js{"rest_id"}.getId
result.user = parseGraphUser(js{"core"})
@ -342,9 +347,9 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
result.expandNoteTweetEntities(noteTweet)
if result.quote.isSome:
result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}, isLegacy))
result.quote = some(await parseGraphTweet(js{"quoted_status_result", "result"}, isLegacy))
proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
proc parseGraphThread(js: JsonNode): Future[tuple[thread: Chain; self: bool]] {.async.} =
for t in js{"content", "items"}:
let entryId = t{"entryId"}.getStr
if "cursor-showmore" in entryId:
@ -358,16 +363,16 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
else: ("content", "tweetResult")
with content, t{"item", contentKey}:
result.thread.content.add parseGraphTweet(content{resultKey, "result"}, isLegacy)
result.thread.content.add (await parseGraphTweet(content{resultKey, "result"}, isLegacy))
if content{"tweetDisplayType"}.getStr == "SelfThread":
result.self = true
proc parseGraphTweetResult*(js: JsonNode): Tweet =
proc parseGraphTweetResult*(js: JsonNode): Future[Tweet] {.async.} =
with tweet, js{"data", "tweet_result", "result"}:
result = parseGraphTweet(tweet, false)
result = await parseGraphTweet(tweet, false)
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
proc parseGraphConversation*(js: JsonNode; tweetId: string): Future[Conversation] {.async.} =
result = Conversation(replies: Result[Chain](beginning: true))
let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
@ -378,7 +383,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult, true)
let tweet = await parseGraphTweet(tweetResult, true)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
@ -400,7 +405,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
else:
result.before.content.add tweet
elif entryId.startsWith("conversationthread"):
let (thread, self) = parseGraphThread(e)
let (thread, self) = await parseGraphThread(e)
if self:
result.after = thread
else:
@ -408,7 +413,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
elif entryId.startsWith("cursor-bottom"):
result.replies.bottom = e{"content", "itemContent", "value"}.getStr
proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Future[Profile] {.async.} =
result = Profile(tweets: Timeline(beginning: after.len == 0))
let instructions =
@ -424,18 +429,18 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
with tweetResult, e{"content", "content", "tweetResult", "result"}:
let tweet = parseGraphTweet(tweetResult, false)
let tweet = await parseGraphTweet(tweetResult, false)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.tweets.content.add tweet
elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
let (thread, self) = parseGraphThread(e)
let (thread, self) = await parseGraphThread(e)
result.tweets.content.add thread.content
elif entryId.startsWith("cursor-bottom"):
result.tweets.bottom = e{"content", "value"}.getStr
if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry":
with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}:
let tweet = parseGraphTweet(tweetResult, false)
let tweet = await parseGraphTweet(tweetResult, false)
tweet.pinned = true
if not tweet.available and tweet.tombstone.len == 0:
let entryId = i{"entry", "entryId"}.getEntryId
@ -443,7 +448,7 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
tweet.id = parseBiggestInt(entryId)
result.pinned = some tweet
proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
proc parseGraphSearch*(js: JsonNode; after=""): Future[Timeline] {.async.} =
result = Timeline(beginning: after.len == 0)
let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
@ -457,7 +462,7 @@ proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
with tweetRes, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetRes, true)
let tweet = await parseGraphTweet(tweetRes, true)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.content.add tweet
@ -465,4 +470,4 @@ proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
result.bottom = e{"content", "value"}.getStr
elif typ == "TimelineReplaceEntry":
if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
result.bottom = instruction{"entry", "content", "value"}.getStr
result.bottom = instruction{"entry", "content", "value"}.getStr

33
src/parserutils.nim
View File

@ -308,3 +308,36 @@ proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
textSlice = 0..text.runeLen
tweet.expandTextEntities(entities, text, textSlice)
proc expandCommunityNoteEntities*(js: JsonNode): string =
var replacements = newSeq[ReplaceSlice]()
let
text = js{"text"}.getStr
entities = ? js{"entities"}
let runes = text.toRunes
for entity in entities:
# These are the only types that I've seen so far
if entity{"ref", "type"}.getStr != "TimelineUrl":
echo "Unknown community note entity type: " & entity{"ref", "type"}.getStr
continue
if entity{"ref", "urlType"}.getStr != "ExternalUrl":
echo "Unknown community note entity urlType: " & entity{"ref", "urlType"}.getStr
continue
let fromIndex = entity{"fromIndex"}.getInt
# Nim slices include the endpoint, while 'toIndex' excludes it
let toIndex = entity{"toIndex"}.getInt - 1
var slice = fromIndex .. toIndex
replacements.add ReplaceSlice(kind: rkUrl, url: entity{"ref", "url"}.getStr,
display: $runes[slice], slice: slice)
replacements.deduplicate
replacements.sort(cmp)
result = runes.replacedWith(replacements, 0 .. runes.len-1).strip(leading=false)

1
src/sass/tweet/_base.scss
View File

@ -7,6 +7,7 @@
@import 'card';
@import 'poll';
@import 'quote';
@import 'community-note';
.tweet-body {
flex: 1;

59
src/sass/tweet/community-note.scss Normal file
View File

@ -0,0 +1,59 @@
@import '_variables';
@import '_mixins';
.community-note {
margin: 5px 0;
pointer-events: all;
max-height: unset;
}
.community-note-container {
border-radius: 10px;
border-width: 1px;
border-style: solid;
border-color: var(--dark_grey);
background-color: var(--bg_elements);
overflow: hidden;
color: inherit;
display: flex;
flex-direction: row;
text-decoration: none !important;
&:hover {
border-color: var(--grey);
}
.attachments {
margin: 0;
border-radius: 0;
}
}
.community-note-content {
padding: 0.5em;
}
.community-note-title {
white-space: unset;
font-weight: bold;
font-size: 1.1em;
}
.community-note-destination {
color: var(--grey);
display: block;
}
.community-note-content-container {
color: unset;
overflow: auto;
&:hover {
text-decoration: none;
}
}
.large {
.community-note-container {
display: block;
}
}

3
src/tokens.nim
View File

@ -55,7 +55,7 @@ proc getPoolJson*(): JsonNode =
maxReqs =
case api
of Api.search: 50
of Api.tweetDetail: 150
of Api.tweetDetail, Api.tweetResultByRestId: 150
of Api.photoRail: 180
of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
Api.userRestId, Api.userScreenName,
@ -149,4 +149,5 @@ proc initAccountPool*(cfg: Config; accounts: JsonNode) =
id: account{"user", "id_str"}.getStr,
oauthToken: account{"oauth_token"}.getStr,
oauthSecret: account{"oauth_token_secret"}.getStr,
guestToken: account{"guest_token"}.getStr,
)

9
src/types.nim
View File

@ -17,6 +17,7 @@ type
Api* {.pure.} = enum
tweetDetail
tweetResult
tweetResultByRestId
photoRail
search
userSearch
@ -40,6 +41,7 @@ type
id*: string
oauthToken*: string
oauthSecret*: string
guestToken*: string
pending*: int
apis*: Table[Api, RateLimit]
@ -206,9 +208,16 @@ type
gif*: Option[Gif]
video*: Option[Video]
photos*: seq[string]
communityNote*: Option[CommunityNote]
Tweets* = seq[Tweet]
CommunityNote* = object
title*: string
subtitle*: string
footer*: string
url*: string
Result*[T] = object
content*: seq[T]
top*, bottom*: string

13
src/views/tweet.nim
View File

@ -203,6 +203,16 @@ proc renderAttribution(user: User; prefs: Prefs): VNode =
if user.verified:
icon "ok", class="verified-icon", title="Verified account"
proc renderCommunityNote*(note: CommunityNote): VNode =
buildHtml(tdiv(class=("community-note large"))):
tdiv(class="community-note-container"):
tdiv(class="community-note-content-container"):
buildHtml(tdiv(class="community-note-content")):
a(href=note.url, rel="noreferrer"): h2(class="community-note-title"):verbatim note.title
br()
verbatim note.subtitle.replace("\n", "<br>")
span(class="community-note-destination"): verbatim note.footer
proc renderMediaTags(tags: seq[User]): VNode =
buildHtml(tdiv(class="media-tag-block")):
icon "user"
@ -321,6 +331,9 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
if tweet.attribution.isSome:
renderAttribution(tweet.attribution.get(), prefs)
if tweet.communityNote.isSome:
renderCommunityNote(tweet.communityNote.get())
if tweet.card.isSome and tweet.card.get().kind != hidden:
renderCard(tweet.card.get(), prefs, path)