Add list support

This commit is contained in:
Zed 2019-09-21 01:08:30 +02:00
parent d1fbcef64d
commit 9e3138e51b
25 changed files with 224 additions and 39 deletions

View File

@ -1,2 +1,2 @@
import api/[profile, timeline, tweet, search, media]
export profile, timeline, tweet, search, media
import api/[profile, timeline, tweet, search, media, list]
export profile, timeline, tweet, search, media, list

View File

@ -11,6 +11,8 @@ const
timelineUrl* = "i/profiles/show/$1/timeline/tweets"
timelineMediaUrl* = "i/profiles/show/$1/media_timeline"
listUrl* = "$1/lists/$2/timeline"
listMembersUrl* = "$1/lists/$2/members"
profilePopupUrl* = "i/profiles/popup"
profileIntentUrl* = "intent/user"
searchUrl* = "i/search/timeline"

83
src/api/list.nim Normal file
View File

@ -0,0 +1,83 @@
import httpclient, asyncdispatch, htmlparser, strformat
import sequtils, strutils, json, uri
import ".."/[types, parser, parserutils, query]
import utils, consts, timeline, search
proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
let url = base / (listUrl % [username, list])
let headers = newHttpHeaders({
"Accept": jsonAccept,
"Referer": $url,
"User-Agent": agent,
"X-Twitter-Active-User": "yes",
"X-Requested-With": "XMLHttpRequest",
"Accept-Language": lang
})
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
"reset_error_state": "false"
})
if after.len > 0:
params.add {"max_position": after}
let json = await fetchJson(url ? params, headers)
result = await finishTimeline(json, Query(), after, agent)
if result.content.len > 0:
result.minId = result.content[^1].id
proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
let url = base / (listMembersUrl % [username, list])
let headers = newHttpHeaders({
"Accept": htmlAccept,
"Referer": $(base / &"{username}/lists/{list}/members"),
"User-Agent": agent,
"Accept-Language": lang
})
let html = await fetchHtml(url, headers)
result = Result[Profile](
minId: html.selectAttr(".stream-container", "data-min-position"),
hasMore: html.select(".has-more-items") != nil,
beginning: true,
query: Query(kind: users),
content: html.selectAll(".account").map(parseListProfile)
)
proc getListMembersSearch*(username, list, agent, after: string): Future[Result[Profile]] {.async.} =
let url = base / ((listMembersUrl & "/timeline") % [username, list])
let headers = newHttpHeaders({
"Accept": jsonAccept,
"Referer": $(base / &"{username}/lists/{list}/members"),
"User-Agent": agent,
"X-Twitter-Active-User": "yes",
"X-Requested-With": "XMLHttpRequest",
"X-Push-With": "XMLHttpRequest",
"Accept-Language": lang
})
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
"reset_error_state": "false"
})
if after.len > 0:
params.add {"max_position": after}
let json = await fetchJson(url ? params, headers)
result = getResult[Profile](json, Query(kind: users), after)
if json == nil or not json.hasKey("items_html"): return
let html = json["items_html"].to(string)
result.hasMore = html != "\n"
for p in parseHtml(html).selectAll(".account"):
result.content.add parseListProfile(p)

View File

@ -7,7 +7,7 @@ import utils, consts, timeline
proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
if json == nil: return Result[T](beginning: true, query: query)
Result[T](
hasMore: json["has_more_items"].to(bool),
hasMore: json.getOrDefault("has_more_items").getBool(false),
maxId: json.getOrDefault("max_position").getStr(""),
minId: json.getOrDefault("min_position").getStr("").cleanPos(),
query: query,
@ -16,7 +16,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
let
kind = if query.kind == users: "users" else: "tweets"
kind = if query.kind == userSearch: "users" else: "tweets"
pos = when T is Tweet: genPos(after) else: after
param = genQueryParam(query)
@ -46,10 +46,9 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
return Result[T](query: query, beginning: true)
let json = await fetchJson(base / searchUrl ? params, headers)
if json == nil: return Result[T](query: query, beginning: true)
result = getResult[T](json, query, after)
if not json.hasKey("items_html"): return
if json == nil or not json.hasKey("items_html"): return
when T is Tweet:
result = await finishTimeline(json, query, after, agent)

View File

@ -1,4 +1,4 @@
import httpclient, asyncdispatch, htmlparser
import httpclient, asyncdispatch, htmlparser, strformat
import sequtils, strutils, json, xmltree, uri
import ".."/[types, parser, parserutils, formatters, query]

View File

@ -5,13 +5,14 @@ import jester
import types, config, prefs
import views/[general, about]
import routes/[preferences, timeline, status, media, search, rss]
import routes/[preferences, timeline, status, media, search, rss, list]
const configPath {.strdefine.} = "./nitter.conf"
let cfg = getConfig(configPath)
createPrefRouter(cfg)
createTimelineRouter(cfg)
createListRouter(cfg)
createStatusRouter(cfg)
createSearchRouter(cfg)
createMediaRouter(cfg)
@ -24,15 +25,16 @@ settings:
routes:
get "/":
resp renderMain(renderSearch(), Prefs(), cfg.title)
resp renderMain(renderSearch(), request, cfg.title)
get "/about":
resp renderMain(renderAbout(), Prefs(), cfg.title)
resp renderMain(renderAbout(), request, cfg.title)
extend preferences, ""
extend rss, ""
extend search, ""
extend timeline, ""
extend list, ""
extend status, ""
extend media, ""

View File

@ -39,6 +39,16 @@ proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile =
result.getPopupStats(profile)
proc parseListProfile*(profile: XmlNode): Profile =
result = Profile(
fullname: profile.getName(".fullname"),
username: profile.getUsername(".username"),
bio: profile.getBio(".bio"),
userpic: profile.getAvatar(".avatar"),
verified: isVerified(profile),
protected: isProtected(profile),
)
proc parseIntentProfile*(profile: XmlNode): Profile =
result = Profile(
fullname: profile.getName("a.fn.url.alternate-context"),

View File

@ -58,7 +58,7 @@ proc genQueryParam*(query: Query): string =
var filters: seq[string]
var param: string
if query.kind == users:
if query.kind == userSearch:
return query.text
for i, user in query.fromUser:
@ -84,7 +84,7 @@ proc genQueryParam*(query: Query): string =
result &= " " & query.text
proc genQueryUrl*(query: Query): string =
if query.kind notin {custom, users}: return
if query.kind notin {custom, userSearch}: return
var params = @[&"kind={query.kind}"]
if query.text.len > 0:

34
src/routes/list.nim Normal file
View File

@ -0,0 +1,34 @@
import strutils
import jester
import router_utils
import ".."/[query, types, api, agents]
import ../views/[general, timeline, list]
template respList*(list, timeline: typed) =
if list.minId.len == 0:
resp Http404, showError("List \"" & @"list" & "\" not found", cfg.title)
let html = renderList(timeline, list.query, @"name", @"list")
let rss = "/$1/lists/$2/rss" % [@"name", @"list"]
resp renderMain(html, request, cfg.title, rss=rss)
proc createListRouter*(cfg: Config) =
router list:
get "/@name/lists/@list":
cond '.' notin @"name"
let
list = await getListTimeline(@"name", @"list", getAgent(), @"after")
tweets = renderTimelineTweets(list, cookiePrefs(), request.path)
respList list, tweets
get "/@name/lists/@list/members":
cond '.' notin @"name"
let list =
if @"after".len == 0:
await getListMembers(@"name", @"list", getAgent())
else:
await getListMembersSearch(@"name", @"list", getAgent(), @"after")
let users = renderTimelineUsers(list, cookiePrefs(), request.path)
respList list, users

View File

@ -3,7 +3,7 @@ import asyncfile, uri, strutils, httpclient, os
import jester, regex
import router_utils
import ".."/[types, formatters, prefs]
import ".."/[types, formatters]
import ../views/general
export asyncfile, httpclient, os, strutils

View File

@ -3,7 +3,7 @@ import strutils, uri
import jester
import router_utils
import ".."/[prefs, types]
import ".."/[types]
import ../views/[general, preferences]
export preferences

View File

@ -1,5 +1,5 @@
import ../utils
export utils
import ../utils, ../prefs
export utils, prefs
template cookiePrefs*(): untyped {.dirty.} =
getPrefs(request.cookies.getOrDefault("preferences"))

View File

@ -34,3 +34,8 @@ proc createRssRouter*(cfg: Config) =
get "/@name/search/rss":
cond '.' notin @"name"
respRss(await showRss(@"name", initQuery(params(request), name=(@"name"))))
get "/@name/lists/@list/rss":
cond '.' notin @"name"
let list = await getListTimeline(@"name", @"list", getAgent(), "")
respRss(renderListRss(list.content, @"name", @"list"))

View File

@ -3,7 +3,7 @@ import strutils, sequtils, uri
import jester
import router_utils
import ".."/[query, types, api, agents, prefs]
import ".."/[query, types, api, agents]
import ../views/[general, search]
export search
@ -18,7 +18,7 @@ proc createSearchRouter*(cfg: Config) =
let query = initQuery(params(request))
case query.kind
of users:
of userSearch:
if "," in @"text":
redirect("/" & @"text")
let users = await getSearch[Profile](query, @"after", getAgent())

View File

@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri
import jester
import router_utils
import ".."/[api, prefs, types, formatters, agents]
import ".."/[api, types, formatters, agents]
import ../views/[general, status]
export uri, sequtils

View File

@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri
import jester
import router_utils
import ".."/[api, prefs, types, cache, formatters, agents, query]
import ".."/[api, types, cache, formatters, agents, query]
import ../views/[general, profile, timeline, status, search]
export uri, sequtils

View File

@ -85,8 +85,11 @@
.replying-to {
color: $fg_dark;
margin: -2px 0 4px;
a {
pointer-events: all;
}
}
.retweet, .pinned, .tweet-stats {
align-content: center;

View File

@ -57,7 +57,7 @@ dbFromTypes("cache.db", "", "", "", [Profile, Video])
type
QueryKind* = enum
posts, replies, media, users, custom
posts, replies, media, users, userSearch, custom
Query* = object
kind*: QueryKind

View File

@ -41,7 +41,7 @@ proc cleanFilename*(filename: string): string =
filename.replace(reg, "_")
proc filterParams*(params: Table): seq[(string, string)] =
let filter = ["name", "id"]
let filter = ["name", "id", "list"]
toSeq(params.pairs()).filterIt(it[0] notin filter and it[1].len > 0)
proc isTwitterUrl*(url: string): bool =

View File

@ -71,5 +71,5 @@ proc renderError*(error: string): VNode =
tdiv(class="error-panel"):
span: text error
proc showError*(error, title: string): string =
renderMain(renderError(error), Request(), title, "Error")
template showError*(error, title: string): string =
renderMain(renderError(error), request, title, "Error")

20
src/views/list.nim Normal file
View File

@ -0,0 +1,20 @@
import strformat
import karax/[karaxdsl, vdom]
import renderutils
import ".."/[types]
proc renderListTabs*(query: Query; path: string): VNode =
buildHtml(ul(class="tab")):
li(class=query.getTabClass(posts)):
a(href=(path)): text "Tweets"
li(class=query.getTabClass(users)):
a(href=(path & "/members")): text "Members"
proc renderList*(body: VNode; query: Query; name, list: string): VNode =
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"):
text &"\"{list}\" by @{name}"
renderListTabs(query, &"/{name}/lists/{list}")
body

View File

@ -30,10 +30,6 @@ proc linkUser*(profile: Profile, class=""): VNode =
text " "
icon "lock-circled", title="Protected account"
proc genImg*(url: string; class=""): VNode =
buildHtml():
img(src=getPicUrl(url), class=class, alt="Image")
proc linkText*(text: string; class=""): VNode =
let url = if "http" notin text: "http://" & text else: text
buildHtml():
@ -91,3 +87,12 @@ proc genDate*(pref, state: string): VNode =
else:
verbatim &"<input name={pref} type=\"date\"/>"
icon "calendar"
proc genImg*(url: string; class=""): VNode =
buildHtml():
img(src=getPicUrl(url), class=class, alt="Image")
proc getTabClass*(query: Query; tab: QueryKind): string =
result = "tab-item"
if query.kind == tab:
result &= " active"

View File

@ -71,3 +71,29 @@
</channel>
</rss>
#end proc
#
#proc renderListRss*(tweets: seq[Tweet]; name, list: string): string =
#let prefs = Prefs(replaceTwitter: hostname)
#result = ""
<?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
<channel>
<atom:link href="https://${hostname}/${name}/lists/${list}/rss" rel="self" type="application/rss+xml" />
<title>${list} / @${name}</title>
<link>https://${hostname}/${name}/lists/${list}</link>
<description>Twitter feed for: ${list} by @${name}. Generated by ${hostname}</description>
<language>en-us</language>
<ttl>40</ttl>
#for tweet in tweets:
<item>
<title>${getTitle(tweet, prefs)}</title>
<dc:creator>@${tweet.profile.username}</dc:creator>
<description><![CDATA[${renderRssTweet(tweet, prefs).strip(chars={'\n'})}]]></description>
<pubDate>${getRfc822Time(tweet)}</pubDate>
<guid>https://${hostname}${getLink(tweet)}</guid>
<link>https://${hostname}${getLink(tweet)}</link>
</item>
#end for
</channel>
</rss>
#end proc

View File

@ -23,15 +23,10 @@ proc renderSearch*(): VNode =
buildHtml(tdiv(class="panel-container")):
tdiv(class="search-bar"):
form(`method`="get", action="/search"):
hiddenField("kind", "users")
hiddenField("kind", "userSearch")
input(`type`="text", name="text", autofocus="", placeholder="Enter username...")
button(`type`="submit"): icon "search"
proc getTabClass(query: Query; tab: QueryKind): string =
result = "tab-item"
if query.kind == tab:
result &= " active"
proc renderProfileTabs*(query: Query; username: string): VNode =
let link = "/" & username
buildHtml(ul(class="tab")):
@ -50,8 +45,8 @@ proc renderSearchTabs*(query: Query): VNode =
li(class=query.getTabClass(custom)):
q.kind = custom
a(href=("?" & genQueryUrl(q))): text "Tweets"
li(class=query.getTabClass(users)):
q.kind = users
li(class=query.getTabClass(userSearch)):
q.kind = userSearch
a(href=("?" & genQueryUrl(q))): text "Users"
proc isPanelOpen(q: Query): bool =
@ -114,7 +109,7 @@ proc renderUserSearch*(users: Result[Profile]; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"):
form(`method`="get", action="/search", class="search-field"):
hiddenField("kind", "users")
hiddenField("kind", "userSearch")
genInput("text", "", users.query.text, "Enter username...", class="pref-inline")
button(`type`="submit"): icon "search"

View File

@ -64,6 +64,7 @@ proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNod
if results.content.len > 0:
for user in results.content:
renderUser(user, prefs)
if results.minId != "0":
renderMore(results.query, results.minId)
elif results.beginning:
renderNoneFound()