lasers
/
nitter
mirror of https://github.com/zedeus/nitter
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge 896d65fd21 into dcf73354ff

This commit is contained in:
PrivacyDevel 2023-07-01 14:16:31 -06:00 committed by GitHub
parent dcf73354ff 896d65fd21
commit 13528cd7d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 171 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
nitter.example.conf
View File

@ -33,6 +33,9 @@ tokenCount = 10
# always at least `tokenCount` usable tokens. only increase this if you receive
# major bursts all the time and don't have a rate limiting setup via e.g. nginx
#cookieHeader = "ct0=XXXXXXXXXXXXXXXXX; auth_token=XXXXXXXXXXXXXX" # authentication cookie of a logged in account, required for the likes tab and NSFW content
#xCsrfToken = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" # required for the likes tab and NSFW content
# Change default preferences here, see src/prefs_impl.nim for a complete list
[Preferences]
theme = "Nitter"

26
src/api.nim
View File

@ -3,6 +3,7 @@ import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
import packedjson
import types, query, formatters, consts, apiutils, parser
import experimental/parser as newParser
import config
proc getGraphUser*(username: string): Future[User] {.async.} =
if username.len == 0: return
@ -69,6 +70,13 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
proc getFavorites*(id: string; cfg: Config; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
ps = genParams({"userId": id}, after)
url = consts.favorites / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.favorites), after)
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
if id.len == 0: return
let
@ -86,6 +94,24 @@ proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
js = await fetch(graphTweet ? params, Api.tweetDetail)
result = parseGraphConversation(js, id)
proc getGraphFavoriters*(id: string; after=""): Future[UsersTimeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = reactorsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphFavoriters ? params, Api.favoriters)
result = parseGraphFavoritersTimeline(js, id)
proc getGraphRetweeters*(id: string; after=""): Future[UsersTimeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = reactorsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphRetweeters ? params, Api.retweeters)
result = parseGraphRetweetersTimeline(js, id)
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
result = (await getGraphTweet(id, after)).replies
result.beginning = after.len == 0

22
src/apiutils.nim
View File

@ -3,6 +3,7 @@ import httpclient, asyncdispatch, options, strutils, uri
import jsony, packedjson, zippy
import types, tokens, consts, parserutils, http_pool
import experimental/types/common
import config
const
rlRemaining = "x-rate-limit-remaining"
@ -50,7 +51,7 @@ template updateToken() =
reset = parseInt(resp.headers[rlReset])
token.setRateLimit(api, remaining, reset)
template fetchImpl(result, fetchBody) {.dirty.} =
template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
once:
pool = HttpPool()
@ -60,7 +61,10 @@ template fetchImpl(result, fetchBody) {.dirty.} =
try:
var resp: AsyncResponse
pool.use(genHeaders(token)):
var headers = genHeaders(token)
for key, value in additional_headers.pairs():
headers.add(key, value)
pool.use(headers):
template getContent =
resp = await c.get($url)
result = await resp.body
@ -94,9 +98,15 @@ template fetchImpl(result, fetchBody) {.dirty.} =
release(token, invalid=true)
raise rateLimitError()
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
if len(cfg.cookieHeader) != 0:
additional_headers.add("Cookie", cfg.cookieHeader)
if len(cfg.xCsrfToken) != 0:
additional_headers.add("x-csrf-token", cfg.xCsrfToken)
var body: string
fetchImpl body:
fetchImpl(body, additional_headers):
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
@ -111,8 +121,8 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
release(token, invalid=true)
raise rateLimitError()
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
fetchImpl result:
proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
fetchImpl(result, additional_headers):
if not (result.startsWith('{') or result.startsWith('[')):
echo resp.status, ": ", result, " --- url: ", url
result.setLen(0)

9
src/config.nim
View File

@ -1,6 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
import parsecfg except Config
import types, strutils
from os import getEnv
proc get*[T](config: parseCfg.Config; section, key: string; default: T): T =
let val = config.getSectionValue(section, key)
@ -40,7 +41,13 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
enableRss: cfg.get("Config", "enableRSS", true),
enableDebug: cfg.get("Config", "enableDebug", false),
proxy: cfg.get("Config", "proxy", ""),
proxyAuth: cfg.get("Config", "proxyAuth", "")
proxyAuth: cfg.get("Config", "proxyAuth", ""),
cookieHeader: cfg.get("Config", "cookieHeader", ""),
xCsrfToken: cfg.get("Config", "xCsrfToken", "")
)
return (conf, cfg)
let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
let (cfg*, fullCfg*) = getConfig(configPath)

11
src/consts.nim
View File

@ -8,6 +8,9 @@ const
activate* = $(api / "1.1/guest/activate.json")
photoRail* = api / "1.1/statuses/media_timeline.json"
timelineApi = api / "2/timeline"
favorites* = timelineApi / "favorites"
userSearch* = api / "1.1/users/search.json"
graphql = api / "graphql"
@ -23,6 +26,8 @@ const
graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"
graphFavoriters* = graphql / "mDc_nU8xGv0cLRWtTaIEug/Favoriters"
graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters"
timelineParams* = {
"include_profile_interstitial_type": "0",
@ -119,3 +124,9 @@ const
"withReactionsPerspective": false,
"withVoice": false
}"""
reactorsVariables* = """{
"tweetId" : "$1", $2
"count" : 20,
"includePromotedContent": false
}"""

4
src/nitter.nim
View File

@ -2,7 +2,6 @@
import asyncdispatch, strformat, logging
from net import Port
from htmlgen import a
from os import getEnv
import jester
@ -15,9 +14,6 @@ import routes/[
const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
const issuesUrl = "https://github.com/zedeus/nitter/issues"
let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
let (cfg, fullCfg) = getConfig(configPath)
if not cfg.enableDebug:
# Silence Jester's query warning
addHandler(newConsoleLogger())

27
src/parser.nim
View File

@ -497,6 +497,33 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline =
elif entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr
proc parseGraphUsersTimeline(js: JsonNode; root: string; key: string; after=""): UsersTimeline =
result = UsersTimeline(beginning: after.len == 0)
let instructions = ? js{"data", key, "timeline", "instructions"}
if instructions.len == 0:
return
for i in instructions:
if i{"type"}.getStr == "TimelineAddEntries":
for e in i{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("user"):
with graphUser, e{"content", "itemContent"}:
let user = parseGraphUser(graphUser)
result.content.add user
elif entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr
elif entryId.startsWith("cursor-top"):
result.top = e{"content", "value"}.getStr
proc parseGraphFavoritersTimeline*(js: JsonNode; root: string; after=""): UsersTimeline =
return parseGraphUsersTimeline(js, root, "favoriters_timeline", after)
proc parseGraphRetweetersTimeline*(js: JsonNode; root: string; after=""): UsersTimeline =
return parseGraphUsersTimeline(js, root, "retweeters_timeline", after)
proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)

7
src/query.nim
View File

@ -40,6 +40,13 @@ proc getMediaQuery*(name: string): Query =
sep: "OR"
)
proc getFavoritesQuery*(name: string): Query =
Query(
kind: favorites,
fromUser: @[name]
)
proc getReplyQuery*(name: string): Query =
Query(
kind: replies,

5
src/routes/rss.nim
View File

@ -23,7 +23,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
names = getNames(name)
if names.len == 1:
profile = await fetchProfile(after, query, skipRail=true, skipPinned=true)
profile = await fetchProfile(after, query, cfg, skipRail=true, skipPinned=true)
else:
var q = query
q.fromUser = names
@ -104,7 +104,7 @@ proc createRssRouter*(cfg: Config) =
get "/@name/@tab/rss":
cond cfg.enableRss
cond '.' notin @"name"
cond @"tab" in ["with_replies", "media", "search"]
cond @"tab" in ["with_replies", "media", "favorites", "search"]
let
name = @"name"
tab = @"tab"
@ -112,6 +112,7 @@ proc createRssRouter*(cfg: Config) =
case tab
of "with_replies": getReplyQuery(name)
of "media": getMediaQuery(name)
of "favorites": getFavoritesQuery(name)
of "search": initQuery(params(request), name=name)
else: Query(fromUser: @[name])

2
src/routes/search.nim
View File

@ -37,7 +37,7 @@ proc createSearchRouter*(cfg: Config) =
let
tweets = await getGraphSearch(query, getCursor())
rss = "/search/rss?" & genQueryUrl(query)
resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
resp renderMain(renderTweetSearch(tweets, cfg, prefs, getPath()),
request, cfg, prefs, title, rss=rss)
else:
resp Http404, showError("Invalid search", cfg)

25
src/routes/status.nim
View File

@ -5,7 +5,7 @@ import jester, karax/vdom
import router_utils
import ".."/[types, formatters, api]
import ../views/[general, status]
import ../views/[general, status, timeline, search]
export uri, sequtils, options, sugar
export router_utils
@ -14,6 +14,29 @@ export status
proc createStatusRouter*(cfg: Config) =
router status:
get "/@name/status/@id/@reactors":
cond '.' notin @"name"
let id = @"id"
if id.len > 19 or id.any(c => not c.isDigit):
resp Http404, showError("Invalid tweet ID", cfg)
let prefs = cookiePrefs()
# used for the infinite scroll feature
if @"scroll".len > 0:
let replies = await getReplies(id, getCursor())
if replies.content.len == 0:
resp Http404, ""
resp $renderReplies(replies, prefs, getPath())
if @"reactors" == "favoriters":
resp renderMain(renderUserList(await getGraphFavoriters(id, getCursor()), prefs),
request, cfg, prefs)
elif @"reactors" == "retweeters":
resp renderMain(renderUserList(await getGraphRetweeters(id, getCursor()), prefs),
request, cfg, prefs)
get "/@name/status/@id/?":
cond '.' notin @"name"
let id = @"id"

16
src/routes/timeline.nim
View File

@ -16,6 +16,7 @@ proc getQuery*(request: Request; tab, name: string): Query =
case tab
of "with_replies": getReplyQuery(name)
of "media": getMediaQuery(name)
of "favorites": getFavoritesQuery(name)
of "search": initQuery(params(request), name=name)
else: Query(fromUser: @[name])
@ -27,7 +28,7 @@ template skipIf[T](cond: bool; default; body: Future[T]): Future[T] =
else:
body
proc fetchProfile*(after: string; query: Query; skipRail=false;
proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
skipPinned=false): Future[Profile] {.async.} =
let
name = query.fromUser[0]
@ -50,6 +51,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
of posts: getGraphUserTweets(userId, TimelineKind.tweets, after)
of replies: getGraphUserTweets(userId, TimelineKind.replies, after)
of media: getGraphUserTweets(userId, TimelineKind.media, after)
of favorites: getFavorites(userId, cfg, after)
else: getGraphSearch(query, after)
rail =
@ -84,10 +86,10 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
if query.fromUser.len != 1:
let
timeline = await getGraphSearch(query, after)
html = renderTweetSearch(timeline, prefs, getPath())
html = renderTweetSearch(timeline, cfg, prefs, getPath())
return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins)
var profile = await fetchProfile(after, query, cfg, skipPinned=prefs.hidePins)
template u: untyped = profile.user
if u.suspended:
@ -95,7 +97,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
if profile.user.id.len == 0: return
let pHtml = renderProfile(profile, prefs, getPath())
let pHtml = renderProfile(profile, cfg, prefs, getPath())
result = renderMain(pHtml, request, cfg, prefs, pageTitle(u), pageDesc(u),
rss=rss, images = @[u.getUserPic("_400x400")],
banner=u.banner)
@ -125,7 +127,7 @@ proc createTimelineRouter*(cfg: Config) =
get "/@name/?@tab?/?":
cond '.' notin @"name"
cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
cond @"tab" in ["with_replies", "media", "search", ""]
cond @"tab" in ["with_replies", "media", "search", "favorites", ""]
let
prefs = cookiePrefs()
after = getCursor()
@ -141,9 +143,9 @@ proc createTimelineRouter*(cfg: Config) =
var timeline = await getGraphSearch(query, after)
if timeline.content.len == 0: resp Http404
timeline.beginning = true
resp $renderTweetSearch(timeline, prefs, getPath())
resp $renderTweetSearch(timeline, cfg, prefs, getPath())
else:
var profile = await fetchProfile(after, query, skipRail=true)
var profile = await fetchProfile(after, query, cfg, skipRail=true)
if profile.tweets.content.len == 0: resp Http404
profile.tweets.beginning = true
resp $renderTimelineTweets(profile.tweets, prefs, getPath())

1
src/sass/tweet/_base.scss
View File

@ -207,6 +207,7 @@
padding-top: 5px;
min-width: 1em;
margin-right: 0.8em;
pointer-events: all;
}
.show-thread {

2
src/tokens.nim
View File

@ -45,7 +45,7 @@ proc getPoolJson*(): JsonNode =
of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
Api.userRestId, Api.userScreenName,
Api.tweetDetail, Api.tweetResult, Api.search: 500
Api.tweetDetail, Api.tweetResult, Api.search, Api.favorites, Api.retweeters, Api.favoriters: 500
of Api.userSearch: 900
reqs = maxReqs - token.apis[api].remaining

9
src/types.nim
View File

@ -26,9 +26,12 @@ type
listTweets
userRestId
userScreenName
favorites
userTweets
userTweetsAndReplies
userMedia
favoriters
retweeters
RateLimit* = object
remaining*: int
@ -106,7 +109,7 @@ type
variants*: seq[VideoVariant]
QueryKind* = enum
posts, replies, media, users, tweets, userList
posts, replies, media, users, tweets, userList, favorites
Query* = object
kind*: QueryKind
@ -223,6 +226,7 @@ type
replies*: Result[Chain]
Timeline* = Result[Tweet]
UsersTimeline* = Result[User]
Profile* = object
user*: User
@ -269,6 +273,9 @@ type
redisMaxConns*: int
redisPassword*: string
cookieHeader*: string
xCsrfToken*: string
Rss* = object
feed*, cursor*: string

4
src/views/profile.nim
View File

@ -99,7 +99,7 @@ proc renderProtected(username: string): VNode =
h2: text "This account's tweets are protected."
p: text &"Only confirmed followers have access to @{username}'s tweets."
proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
proc renderProfile*(profile: var Profile; cfg: Config; prefs: Prefs; path: string): VNode =
profile.tweets.query.fromUser = @[profile.user.username]
buildHtml(tdiv(class="profile-tabs")):
@ -116,4 +116,4 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
if profile.user.protected:
renderProtected(profile.user.username)
else:
renderTweetSearch(profile.tweets, prefs, path, profile.pinned)
renderTweetSearch(profile.tweets, cfg, prefs, path, profile.pinned)

14
src/views/search.nim
View File

@ -29,7 +29,7 @@ proc renderSearch*(): VNode =
placeholder="Enter username...", dir="auto")
button(`type`="submit"): icon "search"
proc renderProfileTabs*(query: Query; username: string): VNode =
proc renderProfileTabs*(query: Query; username: string; cfg: Config): VNode =
let link = "/" & username
buildHtml(ul(class="tab")):
li(class=query.getTabClass(posts)):
@ -38,6 +38,9 @@ proc renderProfileTabs*(query: Query; username: string): VNode =
a(href=(link & "/with_replies")): text "Tweets & Replies"
li(class=query.getTabClass(media)):
a(href=(link & "/media")): text "Media"
if len(cfg.xCsrfToken) != 0 and len(cfg.cookieHeader) != 0:
li(class=query.getTabClass(favorites)):
a(href=(link & "/favorites")): text "Likes"
li(class=query.getTabClass(tweets)):
a(href=(link & "/search")): text "Search"
@ -88,7 +91,7 @@ proc renderSearchPanel*(query: Query): VNode =
span(class="search-title"): text "Near"
genInput("near", "", query.near, "Location...", autofocus=false)
proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
proc renderTweetSearch*(results: Result[Tweet]; cfg: Config; prefs: Prefs; path: string;
pinned=none(Tweet)): VNode =
let query = results.query
buildHtml(tdiv(class="timeline-container")):
@ -97,7 +100,7 @@ proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
text query.fromUser.join(" | ")
if query.fromUser.len > 0:
renderProfileTabs(query, query.fromUser.join(","))
renderProfileTabs(query, query.fromUser.join(","), cfg)
if query.fromUser.len == 0 or query.kind == tweets:
tdiv(class="timeline-header"):
@ -118,3 +121,8 @@ proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode =
renderSearchTabs(results.query)
renderTimelineUsers(results, prefs)
proc renderUserList*(results: Result[User]; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header")
renderTimelineUsers(results, prefs)

21
src/views/tweet.nim
View File

@ -181,14 +181,19 @@ func formatStat(stat: int): string =
if stat > 0: insertSep($stat, ',')
else: ""
proc renderStats(stats: TweetStats; views: string): VNode =
proc renderStats(stats: TweetStats; views: string; tweet: Tweet): VNode =
buildHtml(tdiv(class="tweet-stats")):
span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
span(class="tweet-stat"): icon "quote", formatStat(stats.quotes)
span(class="tweet-stat"): icon "heart", formatStat(stats.likes)
if views.len > 0:
span(class="tweet-stat"): icon "play", insertSep(views, ',')
a(href=getLink(tweet)):
span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
a(href=getLink(tweet, false) & "/retweeters"):
span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
a(href="/search?q=quoted_tweet_id:" & $tweet.id):
span(class="tweet-stat"): icon "quote", formatStat(stats.quotes)
a(href=getLink(tweet, false) & "/favoriters"):
span(class="tweet-stat"): icon "heart", formatStat(stats.likes)
a(href=getLink(tweet)):
if views.len > 0:
span(class="tweet-stat"): icon "play", insertSep(views, ',')
proc renderReply(tweet: Tweet): VNode =
buildHtml(tdiv(class="replying-to")):
@ -344,7 +349,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
renderMediaTags(tweet.mediaTags)
if not prefs.hideTweetStats:
renderStats(tweet.stats, views)
renderStats(tweet.stats, views, tweet)
if showThread:
a(class="show-thread", href=("/i/status/" & $tweet.threadId)):