From 7d2a558e89862072e204410836548f48a7212ac6 Mon Sep 17 00:00:00 2001 From: PrivacyDev Date: Tue, 4 Apr 2023 23:55:01 -0400 Subject: [PATCH 1/9] added favorites endpoint and added likes tab to profile pages --- nitter.example.conf | 3 +++ src/api.nim | 10 ++++++++++ src/apiutils.nim | 12 ++++++------ src/config.nim | 4 +++- src/consts.nim | 1 + src/query.nim | 7 +++++++ src/routes/rss.nim | 5 +++-- src/routes/search.nim | 2 +- src/routes/timeline.nim | 16 +++++++++------- src/types.nim | 6 +++++- src/views/profile.nim | 4 ++-- src/views/search.nim | 9 ++++++--- 12 files changed, 56 insertions(+), 23 deletions(-) diff --git a/nitter.example.conf b/nitter.example.conf index a7abea8..656e879 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -33,6 +33,9 @@ tokenCount = 10 # always at least $tokenCount usable tokens. again, only increase this if # you receive major bursts all the time +#cookieHeader = "ct0=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX; auth_token=XXXXXXXXXXXXXXXXXXXXXXX" # authentication cookie of a logged in account, required for the likes tab +#xCsrfToken = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" # required for the likes tab + # Change default preferences here, see src/prefs_impl.nim for a complete list [Preferences] theme = "Nitter" diff --git a/src/api.nim b/src/api.nim index dfcf413..8993e18 100644 --- a/src/api.nim +++ b/src/api.nim @@ -65,6 +65,16 @@ proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async url = timeline / (id & ".json") ? ps result = parseTimeline(await fetch(url, Api.timeline), after) +proc getFavorites*(id: string; cfg: Config; after=""): Future[Timeline] {.async.} = + if id.len == 0: return + let + ps = genParams({"userId": id}, after) + url = consts.favorites / (id & ".json") ? ps + headers = genHeaders() + headers.add("Cookie", cfg.cookieHeader) + headers.add("x-csrf-token", cfg.xCsrfToken) + result = parseTimeline(await fetch(url, Api.favorites, headers), after) + proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let url = mediaTimeline / (id & ".json") ? genParams(cursor=after) diff --git a/src/apiutils.nim b/src/apiutils.nim index 917932a..78c8c45 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -50,7 +50,7 @@ template updateToken() = reset = parseInt(resp.headers[rlReset]) token.setRateLimit(api, remaining, reset) -template fetchImpl(result, fetchBody) {.dirty.} = +template fetchImpl(result, headers, fetchBody) {.dirty.} = once: pool = HttpPool() @@ -60,7 +60,7 @@ template fetchImpl(result, fetchBody) {.dirty.} = try: var resp: AsyncResponse - pool.use(genHeaders(token)): + pool.use(headers): template getContent = resp = await c.get($url) result = await resp.body @@ -96,9 +96,9 @@ template fetchImpl(result, fetchBody) {.dirty.} = release(token, invalid=true) raise rateLimitError() -proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = +proc fetch*(url: Uri; api: Api; headers: HttpHeaders = genHeaders()): Future[JsonNode] {.async.} = var body: string - fetchImpl body: + fetchImpl(body, headers): if body.startsWith('{') or body.startsWith('['): result = parseJson(body) else: @@ -113,8 +113,8 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = release(token, invalid=true) raise rateLimitError() -proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} = - fetchImpl result: +proc fetchRaw*(url: Uri; api: Api; headers: HttpHeaders = genHeaders()): Future[string] {.async.} = + fetchImpl(result, headers): if not (result.startsWith('{') or result.startsWith('[')): echo resp.status, ": ", result, " --- url: ", url result.setLen(0) diff --git a/src/config.nim b/src/config.nim index 1b05ffe..47f0fc3 100644 --- a/src/config.nim +++ b/src/config.nim @@ -40,7 +40,9 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = enableRss: cfg.get("Config", "enableRSS", true), enableDebug: cfg.get("Config", "enableDebug", false), proxy: cfg.get("Config", "proxy", ""), - proxyAuth: cfg.get("Config", "proxyAuth", "") + proxyAuth: cfg.get("Config", "proxyAuth", ""), + cookieHeader: cfg.get("Config", "cookieHeader", ""), + xCsrfToken: cfg.get("Config", "xCsrfToken", "") ) return (conf, cfg) diff --git a/src/consts.nim b/src/consts.nim index bb4e1a3..c4e49d7 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -15,6 +15,7 @@ const timelineApi = api / "2/timeline" timeline* = timelineApi / "profile" mediaTimeline* = timelineApi / "media" + favorites* = timelineApi / "favorites" listTimeline* = timelineApi / "list.json" tweet* = timelineApi / "conversation" diff --git a/src/query.nim b/src/query.nim index d128f6f..49c5856 100644 --- a/src/query.nim +++ b/src/query.nim @@ -40,6 +40,13 @@ proc getMediaQuery*(name: string): Query = sep: "OR" ) + +proc getFavoritesQuery*(name: string): Query = + Query( + kind: favorites, + fromUser: @[name] + ) + proc getReplyQuery*(name: string): Query = Query( kind: replies, diff --git a/src/routes/rss.nim b/src/routes/rss.nim index 5da29b0..3b31671 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -23,7 +23,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async. names = getNames(name) if names.len == 1: - profile = await fetchProfile(after, query, skipRail=true, skipPinned=true) + profile = await fetchProfile(after, query, cfg, skipRail=true, skipPinned=true) else: var q = query q.fromUser = names @@ -104,7 +104,7 @@ proc createRssRouter*(cfg: Config) = get "/@name/@tab/rss": cond cfg.enableRss cond '.' notin @"name" - cond @"tab" in ["with_replies", "media", "search"] + cond @"tab" in ["with_replies", "media", "favorites", "search"] let name = @"name" tab = @"tab" @@ -112,6 +112,7 @@ proc createRssRouter*(cfg: Config) = case tab of "with_replies": getReplyQuery(name) of "media": getMediaQuery(name) + of "favorites": getFavoritesQuery(name) of "search": initQuery(params(request), name=name) else: Query(fromUser: @[name]) diff --git a/src/routes/search.nim b/src/routes/search.nim index b2fd718..70f5ca2 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -33,7 +33,7 @@ proc createSearchRouter*(cfg: Config) = let tweets = await getSearch[Tweet](query, getCursor()) rss = "/search/rss?" & genQueryUrl(query) - resp renderMain(renderTweetSearch(tweets, prefs, getPath()), + resp renderMain(renderTweetSearch(tweets, cfg, prefs, getPath()), request, cfg, prefs, title, rss=rss) else: resp Http404, showError("Invalid search", cfg) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index a0a6e21..906c5d4 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -16,6 +16,7 @@ proc getQuery*(request: Request; tab, name: string): Query = case tab of "with_replies": getReplyQuery(name) of "media": getMediaQuery(name) + of "favorites": getFavoritesQuery(name) of "search": initQuery(params(request), name=name) else: Query(fromUser: @[name]) @@ -27,7 +28,7 @@ template skipIf[T](cond: bool; default; body: Future[T]): Future[T] = else: body -proc fetchProfile*(after: string; query: Query; skipRail=false; +proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false; skipPinned=false): Future[Profile] {.async.} = let name = query.fromUser[0] @@ -50,6 +51,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; of posts: getTimeline(userId, after) of replies: getTimeline(userId, after, replies=true) of media: getMediaTimeline(userId, after) + of favorites: getFavorites(userId, cfg, after) else: getSearch[Tweet](query, after) rail = @@ -83,10 +85,10 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; if query.fromUser.len != 1: let timeline = await getSearch[Tweet](query, after) - html = renderTweetSearch(timeline, prefs, getPath()) + html = renderTweetSearch(timeline, cfg, prefs, getPath()) return renderMain(html, request, cfg, prefs, "Multi", rss=rss) - var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins) + var profile = await fetchProfile(after, query, cfg, skipPinned=prefs.hidePins) template u: untyped = profile.user if u.suspended: @@ -94,7 +96,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; if profile.user.id.len == 0: return - let pHtml = renderProfile(profile, prefs, getPath()) + let pHtml = renderProfile(profile, cfg, prefs, getPath()) result = renderMain(pHtml, request, cfg, prefs, pageTitle(u), pageDesc(u), rss=rss, images = @[u.getUserPic("_400x400")], banner=u.banner) @@ -124,7 +126,7 @@ proc createTimelineRouter*(cfg: Config) = get "/@name/?@tab?/?": cond '.' notin @"name" cond @"name" notin ["pic", "gif", "video"] - cond @"tab" in ["with_replies", "media", "search", ""] + cond @"tab" in ["with_replies", "media", "search", "favorites", ""] let prefs = cookiePrefs() after = getCursor() @@ -140,9 +142,9 @@ proc createTimelineRouter*(cfg: Config) = var timeline = await getSearch[Tweet](query, after) if timeline.content.len == 0: resp Http404 timeline.beginning = true - resp $renderTweetSearch(timeline, prefs, getPath()) + resp $renderTweetSearch(timeline, cfg, prefs, getPath()) else: - var profile = await fetchProfile(after, query, skipRail=true) + var profile = await fetchProfile(after, query, cfg, skipRail=true) if profile.tweets.content.len == 0: resp Http404 profile.tweets.beginning = true resp $renderTimelineTweets(profile.tweets, prefs, getPath()) diff --git a/src/types.nim b/src/types.nim index 6f742d1..f5fa6ae 100644 --- a/src/types.nim +++ b/src/types.nim @@ -20,6 +20,7 @@ type userRestId userScreenName status + favorites RateLimit* = object remaining*: int @@ -95,7 +96,7 @@ type variants*: seq[VideoVariant] QueryKind* = enum - posts, replies, media, users, tweets, userList + posts, replies, media, users, tweets, userList, favorites Query* = object kind*: QueryKind @@ -257,6 +258,9 @@ type redisMaxConns*: int redisPassword*: string + cookieHeader*: string + xCsrfToken*: string + Rss* = object feed*, cursor*: string diff --git a/src/views/profile.nim b/src/views/profile.nim index 2b2e410..75cc169 100644 --- a/src/views/profile.nim +++ b/src/views/profile.nim @@ -99,7 +99,7 @@ proc renderProtected(username: string): VNode = h2: text "This account's tweets are protected." p: text &"Only confirmed followers have access to @{username}'s tweets." -proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode = +proc renderProfile*(profile: var Profile; cfg: Config; prefs: Prefs; path: string): VNode = profile.tweets.query.fromUser = @[profile.user.username] buildHtml(tdiv(class="profile-tabs")): @@ -116,4 +116,4 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode = if profile.user.protected: renderProtected(profile.user.username) else: - renderTweetSearch(profile.tweets, prefs, path, profile.pinned) + renderTweetSearch(profile.tweets, cfg, prefs, path, profile.pinned) diff --git a/src/views/search.nim b/src/views/search.nim index 77ba14f..cb37fdc 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -29,7 +29,7 @@ proc renderSearch*(): VNode = placeholder="Enter username...", dir="auto") button(`type`="submit"): icon "search" -proc renderProfileTabs*(query: Query; username: string): VNode = +proc renderProfileTabs*(query: Query; username: string; cfg: Config): VNode = let link = "/" & username buildHtml(ul(class="tab")): li(class=query.getTabClass(posts)): @@ -38,6 +38,9 @@ proc renderProfileTabs*(query: Query; username: string): VNode = a(href=(link & "/with_replies")): text "Tweets & Replies" li(class=query.getTabClass(media)): a(href=(link & "/media")): text "Media" + if len(cfg.xCsrfToken) != 0 and len(cfg.cookieHeader) != 0: + li(class=query.getTabClass(favorites)): + a(href=(link & "/favorites")): text "Likes" li(class=query.getTabClass(tweets)): a(href=(link & "/search")): text "Search" @@ -90,7 +93,7 @@ proc renderSearchPanel*(query: Query): VNode = span(class="search-title"): text "Near" genInput("near", "", query.near, "Location...", autofocus=false) -proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string; +proc renderTweetSearch*(results: Result[Tweet]; cfg: Config; prefs: Prefs; path: string; pinned=none(Tweet)): VNode = let query = results.query buildHtml(tdiv(class="timeline-container")): @@ -99,7 +102,7 @@ proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string; text query.fromUser.join(" | ") if query.fromUser.len > 0: - renderProfileTabs(query, query.fromUser.join(",")) + renderProfileTabs(query, query.fromUser.join(","), cfg) if query.fromUser.len == 0 or query.kind == tweets: tdiv(class="timeline-header"): From a6dd229444271877a9a174c6bf6646ecd9e3a1d8 Mon Sep 17 00:00:00 2001 From: PrivacyDev Date: Wed, 5 Apr 2023 01:14:30 -0400 Subject: [PATCH 2/9] fixed token issue that broke all pages besides the favorites / likes timeline --- src/api.nim | 7 ++++--- src/apiutils.nim | 13 ++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/api.nim b/src/api.nim index 8993e18..45e0f65 100644 --- a/src/api.nim +++ b/src/api.nim @@ -70,9 +70,10 @@ proc getFavorites*(id: string; cfg: Config; after=""): Future[Timeline] {.async. let ps = genParams({"userId": id}, after) url = consts.favorites / (id & ".json") ? ps - headers = genHeaders() - headers.add("Cookie", cfg.cookieHeader) - headers.add("x-csrf-token", cfg.xCsrfToken) + headers = newHttpHeaders({ + "Cookie": cfg.cookieHeader, + "x-csrf-token": cfg.xCsrfToken + }) result = parseTimeline(await fetch(url, Api.favorites, headers), after) proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} = diff --git a/src/apiutils.nim b/src/apiutils.nim index 78c8c45..ff10735 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -50,7 +50,7 @@ template updateToken() = reset = parseInt(resp.headers[rlReset]) token.setRateLimit(api, remaining, reset) -template fetchImpl(result, headers, fetchBody) {.dirty.} = +template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = once: pool = HttpPool() @@ -60,6 +60,9 @@ template fetchImpl(result, headers, fetchBody) {.dirty.} = try: var resp: AsyncResponse + var headers = genHeaders(token) + for key, value in additional_headers.pairs(): + headers.add(key, value) pool.use(headers): template getContent = resp = await c.get($url) @@ -96,9 +99,9 @@ template fetchImpl(result, headers, fetchBody) {.dirty.} = release(token, invalid=true) raise rateLimitError() -proc fetch*(url: Uri; api: Api; headers: HttpHeaders = genHeaders()): Future[JsonNode] {.async.} = +proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} = var body: string - fetchImpl(body, headers): + fetchImpl(body, additional_headers): if body.startsWith('{') or body.startsWith('['): result = parseJson(body) else: @@ -113,8 +116,8 @@ proc fetch*(url: Uri; api: Api; headers: HttpHeaders = genHeaders()): Future[Jso release(token, invalid=true) raise rateLimitError() -proc fetchRaw*(url: Uri; api: Api; headers: HttpHeaders = genHeaders()): Future[string] {.async.} = - fetchImpl(result, headers): +proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} = + fetchImpl(result, additional_headers): if not (result.startsWith('{') or result.startsWith('[')): echo resp.status, ": ", result, " --- url: ", url result.setLen(0) From d5689f2253d761dd72723868742fdc3211c850bd Mon Sep 17 00:00:00 2001 From: PrivacyDev Date: Sat, 8 Apr 2023 10:33:49 -0400 Subject: [PATCH 3/9] added login-based workaround to view NSFW content --- nitter.example.conf | 4 ++-- src/api.nim | 6 +----- src/apiutils.nim | 5 +++++ src/config.nim | 5 +++++ src/nitter.nim | 4 ---- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/nitter.example.conf b/nitter.example.conf index 656e879..a950916 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -33,8 +33,8 @@ tokenCount = 10 # always at least $tokenCount usable tokens. again, only increase this if # you receive major bursts all the time -#cookieHeader = "ct0=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX; auth_token=XXXXXXXXXXXXXXXXXXXXXXX" # authentication cookie of a logged in account, required for the likes tab -#xCsrfToken = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" # required for the likes tab +#cookieHeader = "ct0=XXXXXXXXXXXXXXXXX; auth_token=XXXXXXXXXXXXXX" # authentication cookie of a logged in account, required for the likes tab and NSFW content +#xCsrfToken = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" # required for the likes tab and NSFW content # Change default preferences here, see src/prefs_impl.nim for a complete list [Preferences] diff --git a/src/api.nim b/src/api.nim index 45e0f65..a5eb0fd 100644 --- a/src/api.nim +++ b/src/api.nim @@ -70,11 +70,7 @@ proc getFavorites*(id: string; cfg: Config; after=""): Future[Timeline] {.async. let ps = genParams({"userId": id}, after) url = consts.favorites / (id & ".json") ? ps - headers = newHttpHeaders({ - "Cookie": cfg.cookieHeader, - "x-csrf-token": cfg.xCsrfToken - }) - result = parseTimeline(await fetch(url, Api.favorites, headers), after) + result = parseTimeline(await fetch(url, Api.favorites), after) proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return diff --git a/src/apiutils.nim b/src/apiutils.nim index ff10735..15c7200 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -3,6 +3,7 @@ import httpclient, asyncdispatch, options, strutils, uri import jsony, packedjson, zippy import types, tokens, consts, parserutils, http_pool import experimental/types/common +import config const rlRemaining = "x-rate-limit-remaining" @@ -42,6 +43,10 @@ proc genHeaders*(token: Token = nil): HttpHeaders = "accept": "*/*", "DNT": "1" }) + if len(cfg.cookieHeader) != 0: + result.add("Cookie", cfg.cookieHeader) + if len(cfg.xCsrfToken) != 0: + result.add("x-csrf-token", cfg.xCsrfToken) template updateToken() = if api != Api.search and resp.headers.hasKey(rlRemaining): diff --git a/src/config.nim b/src/config.nim index 47f0fc3..fe4aba5 100644 --- a/src/config.nim +++ b/src/config.nim @@ -1,6 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import parsecfg except Config import types, strutils +from os import getEnv proc get*[T](config: parseCfg.Config; section, key: string; default: T): T = let val = config.getSectionValue(section, key) @@ -46,3 +47,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = ) return (conf, cfg) + + +let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf") +let (cfg*, fullCfg*) = getConfig(configPath) diff --git a/src/nitter.nim b/src/nitter.nim index 2e868a4..5eee56f 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -2,7 +2,6 @@ import asyncdispatch, strformat, logging from net import Port from htmlgen import a -from os import getEnv import jester @@ -15,9 +14,6 @@ import routes/[ const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" -let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf") -let (cfg, fullCfg) = getConfig(configPath) - if not cfg.enableDebug: # Silence Jester's query warning addHandler(newConsoleLogger()) From 6875569bf2122623132cf58a42e75affda4a957c Mon Sep 17 00:00:00 2001 From: PrivacyDev Date: Sun, 9 Apr 2023 17:32:57 -0400 Subject: [PATCH 4/9] stopped using Twitter session info for userID requests --- src/apiutils.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index 15c7200..e45954c 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -43,10 +43,6 @@ proc genHeaders*(token: Token = nil): HttpHeaders = "accept": "*/*", "DNT": "1" }) - if len(cfg.cookieHeader) != 0: - result.add("Cookie", cfg.cookieHeader) - if len(cfg.xCsrfToken) != 0: - result.add("x-csrf-token", cfg.xCsrfToken) template updateToken() = if api != Api.search and resp.headers.hasKey(rlRemaining): @@ -105,6 +101,12 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = raise rateLimitError() proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} = + + if len(cfg.cookieHeader) != 0: + additional_headers.add("Cookie", cfg.cookieHeader) + if len(cfg.xCsrfToken) != 0: + additional_headers.add("x-csrf-token", cfg.xCsrfToken) + var body: string fetchImpl(body, additional_headers): if body.startsWith('{') or body.startsWith('['): From 11279e2b4ff612f523380c2ff4678a056eb5c03c Mon Sep 17 00:00:00 2001 From: PrivacyDev Date: Sun, 16 Apr 2023 02:05:45 -0400 Subject: [PATCH 5/9] added authentication headers to user search for nsfw users --- src/api.nim | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/api.nim b/src/api.nim index a5eb0fd..f28a5c7 100644 --- a/src/api.nim +++ b/src/api.nim @@ -3,6 +3,7 @@ import asyncdispatch, httpclient, uri, strutils, sequtils, sugar import packedjson import types, query, formatters, consts, apiutils, parser import experimental/parser as newParser +import config proc getGraphUser*(username: string): Future[User] {.async.} = if username.len == 0: return @@ -86,11 +87,18 @@ proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} = result = parsePhotoRail(await fetch(url, Api.timeline)) proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = + + let additional_headers = newHttpHeaders() + when T is User: const searchMode = ("result_filter", "user") parse = parseUsers fetchFunc = fetchRaw + if len(cfg.cookieHeader) != 0: + additional_headers.add("Cookie", cfg.cookieHeader) + if len(cfg.xCsrfToken) != 0: + additional_headers.add("x-csrf-token", cfg.xCsrfToken) else: const searchMode = ("tweet_search_mode", "live") @@ -103,7 +111,7 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = let url = search ? genParams(searchParams & @[("q", q), searchMode], after) try: - result = parse(await fetchFunc(url, Api.search), after) + result = parse(await fetchFunc(url, Api.search, additional_headers), after) result.query = query except InternalError: return Result[T](beginning: true, query: query) From f4baef9a9bd018a1bccb12c85cd93acef3df2d5b Mon Sep 17 00:00:00 2001 From: PrivacyDev Date: Fri, 2 Jun 2023 23:47:05 -0400 Subject: [PATCH 6/9] added favoriters and retweeters endpoints --- src/api.nim | 18 ++++++++++++++++++ src/consts.nim | 8 ++++++++ src/parser.nim | 27 +++++++++++++++++++++++++++ src/routes/status.nim | 25 ++++++++++++++++++++++++- src/tokens.nim | 2 +- src/types.nim | 3 +++ src/views/search.nim | 5 +++++ 7 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/api.nim b/src/api.nim index c63a5b5..5e6eee4 100644 --- a/src/api.nim +++ b/src/api.nim @@ -94,6 +94,24 @@ proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} = js = await fetch(graphTweet ? params, Api.tweetDetail) result = parseGraphConversation(js, id) +proc getGraphFavoriters*(id: string; after=""): Future[UsersTimeline] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = reactorsVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + js = await fetch(graphFavoriters ? params, Api.favoriters) + result = parseGraphFavoritersTimeline(js, id) + +proc getGraphRetweeters*(id: string; after=""): Future[UsersTimeline] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = reactorsVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + js = await fetch(graphRetweeters ? params, Api.retweeters) + result = parseGraphRetweetersTimeline(js, id) + proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} = result = (await getGraphTweet(id, after)).replies result.beginning = after.len == 0 diff --git a/src/consts.nim b/src/consts.nim index 4063876..ecd2e0b 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -26,6 +26,8 @@ const graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug" graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers" graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline" + graphFavoriters* = graphql / "mDc_nU8xGv0cLRWtTaIEug/Favoriters" + graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters" timelineParams* = { "include_profile_interstitial_type": "0", @@ -122,3 +124,9 @@ const "withReactionsPerspective": false, "withVoice": false }""" + + reactorsVariables* = """{ + "tweetId" : "$1", $2 + "count" : 20, + "includePromotedContent": false +}""" diff --git a/src/parser.nim b/src/parser.nim index 5ec21e4..21f8561 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -493,6 +493,33 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline = elif entryId.startsWith("cursor-bottom"): result.bottom = e{"content", "value"}.getStr +proc parseGraphUsersTimeline(js: JsonNode; root: string; key: string; after=""): UsersTimeline = + result = UsersTimeline(beginning: after.len == 0) + + let instructions = ? js{"data", key, "timeline", "instructions"} + + if instructions.len == 0: + return + + for i in instructions: + if i{"type"}.getStr == "TimelineAddEntries": + for e in i{"entries"}: + let entryId = e{"entryId"}.getStr + if entryId.startsWith("user"): + with graphUser, e{"content", "itemContent"}: + let user = parseGraphUser(graphUser) + result.content.add user + elif entryId.startsWith("cursor-bottom"): + result.bottom = e{"content", "value"}.getStr + elif entryId.startsWith("cursor-top"): + result.top = e{"content", "value"}.getStr + +proc parseGraphFavoritersTimeline*(js: JsonNode; root: string; after=""): UsersTimeline = + return parseGraphUsersTimeline(js, root, "favoriters_timeline", after) + +proc parseGraphRetweetersTimeline*(js: JsonNode; root: string; after=""): UsersTimeline = + return parseGraphUsersTimeline(js, root, "retweeters_timeline", after) + proc parseGraphSearch*(js: JsonNode; after=""): Timeline = result = Timeline(beginning: after.len == 0) diff --git a/src/routes/status.nim b/src/routes/status.nim index 7e89220..d41f0d3 100644 --- a/src/routes/status.nim +++ b/src/routes/status.nim @@ -5,7 +5,7 @@ import jester, karax/vdom import router_utils import ".."/[types, formatters, api] -import ../views/[general, status] +import ../views/[general, status, timeline, search] export uri, sequtils, options, sugar export router_utils @@ -14,6 +14,29 @@ export status proc createStatusRouter*(cfg: Config) = router status: + get "/@name/status/@id/@reactors": + cond '.' notin @"name" + let id = @"id" + + if id.len > 19 or id.any(c => not c.isDigit): + resp Http404, showError("Invalid tweet ID", cfg) + + let prefs = cookiePrefs() + + # used for the infinite scroll feature + if @"scroll".len > 0: + let replies = await getReplies(id, getCursor()) + if replies.content.len == 0: + resp Http404, "" + resp $renderReplies(replies, prefs, getPath()) + + if @"reactors" == "favoriters": + resp renderMain(renderUserList(await getGraphFavoriters(id, getCursor()), prefs), + request, cfg, prefs) + elif @"reactors" == "retweeters": + resp renderMain(renderUserList(await getGraphRetweeters(id, getCursor()), prefs), + request, cfg, prefs) + get "/@name/status/@id/?": cond '.' notin @"name" let id = @"id" diff --git a/src/tokens.nim b/src/tokens.nim index 6ef81f5..d3ea3a9 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -45,7 +45,7 @@ proc getPoolJson*(): JsonNode = of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets, Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, Api.userRestId, Api.userScreenName, - Api.tweetDetail, Api.tweetResult, Api.search: 500 + Api.tweetDetail, Api.tweetResult, Api.search, Api.retweeters, Api.favoriters: 500 of Api.userSearch: 900 reqs = maxReqs - token.apis[api].remaining diff --git a/src/types.nim b/src/types.nim index d080ee2..98acaca 100644 --- a/src/types.nim +++ b/src/types.nim @@ -30,6 +30,8 @@ type userTweets userTweetsAndReplies userMedia + favoriters + retweeters RateLimit* = object remaining*: int @@ -224,6 +226,7 @@ type replies*: Result[Chain] Timeline* = Result[Tweet] + UsersTimeline* = Result[User] Profile* = object user*: User diff --git a/src/views/search.nim b/src/views/search.nim index df210a5..86bebf4 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -121,3 +121,8 @@ proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode = renderSearchTabs(results.query) renderTimelineUsers(results, prefs) + +proc renderUserList*(results: Result[User]; prefs: Prefs): VNode = + buildHtml(tdiv(class="timeline-container")): + tdiv(class="timeline-header") + renderTimelineUsers(results, prefs) From 9981159a4ac1c1ddee166a153b690b3c2cee72fe Mon Sep 17 00:00:00 2001 From: PrivacyDev Date: Sun, 4 Jun 2023 23:31:07 -0400 Subject: [PATCH 7/9] added favoriters and retweeters links to tweet-stats --- src/sass/tweet/_base.scss | 1 + src/views/tweet.nim | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss index 69f51c0..3431a7b 100644 --- a/src/sass/tweet/_base.scss +++ b/src/sass/tweet/_base.scss @@ -207,6 +207,7 @@ padding-top: 5px; min-width: 1em; margin-right: 0.8em; + pointer-events: all; } .show-thread { diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 3338b71..df2e023 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -181,14 +181,19 @@ func formatStat(stat: int): string = if stat > 0: insertSep($stat, ',') else: "" -proc renderStats(stats: TweetStats; views: string): VNode = +proc renderStats(stats: TweetStats; views: string; tweet: Tweet): VNode = buildHtml(tdiv(class="tweet-stats")): - span(class="tweet-stat"): icon "comment", formatStat(stats.replies) - span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets) - span(class="tweet-stat"): icon "quote", formatStat(stats.quotes) - span(class="tweet-stat"): icon "heart", formatStat(stats.likes) - if views.len > 0: - span(class="tweet-stat"): icon "play", insertSep(views, ',') + a(href=getLink(tweet)): + span(class="tweet-stat"): icon "comment", formatStat(stats.replies) + a(href=getLink(tweet, false) & "/retweeters"): + span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets) + a(href=getLink(tweet)): + span(class="tweet-stat"): icon "quote", formatStat(stats.quotes) + a(href=getLink(tweet, false) & "/favoriters"): + span(class="tweet-stat"): icon "heart", formatStat(stats.likes) + a(href=getLink(tweet)): + if views.len > 0: + span(class="tweet-stat"): icon "play", insertSep(views, ',') proc renderReply(tweet: Tweet): VNode = buildHtml(tdiv(class="replying-to")): @@ -344,7 +349,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; renderMediaTags(tweet.mediaTags) if not prefs.hideTweetStats: - renderStats(tweet.stats, views) + renderStats(tweet.stats, views, tweet) if showThread: a(class="show-thread", href=("/i/status/" & $tweet.threadId)): From 61144d34784bc5581309c80375f1e1d45e3fe113 Mon Sep 17 00:00:00 2001 From: PrivacyDev Date: Mon, 5 Jun 2023 19:41:04 -0400 Subject: [PATCH 8/9] added missing Api.favorites to getPoolJson --- src/tokens.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tokens.nim b/src/tokens.nim index d3ea3a9..316a6cd 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -45,7 +45,7 @@ proc getPoolJson*(): JsonNode = of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets, Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, Api.userRestId, Api.userScreenName, - Api.tweetDetail, Api.tweetResult, Api.search, Api.retweeters, Api.favoriters: 500 + Api.tweetDetail, Api.tweetResult, Api.search, Api.favorites, Api.retweeters, Api.favoriters: 500 of Api.userSearch: 900 reqs = maxReqs - token.apis[api].remaining From 896d65fd218fcba08fc719c4c73fc72b8e61655b Mon Sep 17 00:00:00 2001 From: PrivacyDev Date: Mon, 5 Jun 2023 19:42:31 -0400 Subject: [PATCH 9/9] turned quote stat in tweet-stat into a clickable link to the quotes --- src/views/tweet.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/views/tweet.nim b/src/views/tweet.nim index df2e023..c24b1b7 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -187,7 +187,7 @@ proc renderStats(stats: TweetStats; views: string; tweet: Tweet): VNode = span(class="tweet-stat"): icon "comment", formatStat(stats.replies) a(href=getLink(tweet, false) & "/retweeters"): span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets) - a(href=getLink(tweet)): + a(href="/search?q=quoted_tweet_id:" & $tweet.id): span(class="tweet-stat"): icon "quote", formatStat(stats.quotes) a(href=getLink(tweet, false) & "/favoriters"): span(class="tweet-stat"): icon "heart", formatStat(stats.likes)