mirror of https://github.com/zedeus/nitter
Revamp profile api to display more metadata
This commit is contained in:
Zed
parent
3f1d9777b6
commit
7171486f03
|
|
@ -445,14 +445,6 @@ video {
|
|||
display: flex;
|
||||
}
|
||||
|
||||
.profile-card-tabs {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
flex: 1 1 auto;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.profile-card-tabs-name {
|
||||
max-width: 100%;
|
||||
}
|
||||
|
|
@ -496,20 +488,26 @@ video {
|
|||
.profile-card-extra {
|
||||
display: contents;
|
||||
flex: 100%;
|
||||
margin-top: 4px;
|
||||
margin-top: 6px;
|
||||
}
|
||||
|
||||
.profile-bio {
|
||||
overflow: hidden;
|
||||
overflow-wrap: break-word;
|
||||
width: 100%;
|
||||
margin: 10px -6px 0px 0px;
|
||||
margin: 4px -6px 6px 0px;
|
||||
}
|
||||
|
||||
.profile-bio p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.profile-location, .profile-website, .profile-joindate {
|
||||
color: #f8f8f2cf;
|
||||
margin: 2px 0px;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.profile-description {
|
||||
font-size: 14px;
|
||||
font-weight: 400;
|
||||
|
|
@ -735,6 +733,7 @@ video {
|
|||
margin: 0;
|
||||
padding: 0;
|
||||
width: 100%;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.profile-statlist > li {
|
||||
|
|
@ -742,19 +741,6 @@ video {
|
|||
text-align: center;
|
||||
}
|
||||
|
||||
.profile-statlist .posts {
|
||||
flex: 0.4 1 0;
|
||||
}
|
||||
|
||||
.profile-statlist .followers {
|
||||
flex: 1 1 0;
|
||||
padding: 0 3px;
|
||||
}
|
||||
|
||||
.profile-statlist .following {
|
||||
flex: 0.5 1 0;
|
||||
}
|
||||
|
||||
.profile-stat-header {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
|
|
|||
44
src/api.nim
44
src/api.nim
|
|
@ -6,7 +6,7 @@ import types, parser, parserutils, formatters, search
|
|||
const
|
||||
lang = "en-US,en;q=0.9"
|
||||
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
||||
cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
|
||||
accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
|
||||
jsonAccept = "application/json, text/javascript, */*; q=0.01"
|
||||
|
||||
base = parseUri("https://twitter.com/")
|
||||
|
|
@ -38,7 +38,7 @@ macro genMediaGet(media: untyped; token=false) =
|
|||
single = ident("get" & mediaName)
|
||||
|
||||
quote do:
|
||||
proc `multi`(thread: Thread; agent: string; token="") {.async.} =
|
||||
proc `multi`(thread: Thread | Timeline; agent: string; token="") {.async.} =
|
||||
if thread == nil: return
|
||||
var `media` = thread.tweets.filterIt(it.`media`.isSome)
|
||||
when `token`:
|
||||
|
|
@ -165,7 +165,7 @@ proc getPoll*(tweet: Tweet; agent: string) {.async.} =
|
|||
if tweet.poll.isNone(): return
|
||||
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": cardAccept,
|
||||
"Accept": accept,
|
||||
"Referer": $(base / getLink(tweet)),
|
||||
"User-Agent": agent,
|
||||
"Authority": "twitter.com",
|
||||
|
|
@ -182,7 +182,7 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} =
|
|||
if tweet.card.isNone(): return
|
||||
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": cardAccept,
|
||||
"Accept": accept,
|
||||
"Referer": $(base / getLink(tweet)),
|
||||
"User-Agent": agent,
|
||||
"Authority": "twitter.com",
|
||||
|
|
@ -350,3 +350,39 @@ proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.
|
|||
|
||||
let json = await fetchJson(base / timelineSearchUrl ? params, headers)
|
||||
result = await finishTimeline(json, some(query), after, agent)
|
||||
|
||||
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"authority": "twitter.com",
|
||||
"accept": accept,
|
||||
"referer": "https://twitter.com/" & username,
|
||||
"accept-language": lang
|
||||
})
|
||||
|
||||
var url = base / username
|
||||
if after.len > 0:
|
||||
url = url ? {"max_position": after}
|
||||
|
||||
let
|
||||
html = await fetchHtml(url, headers)
|
||||
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
|
||||
profile = parseTimelineProfile(html)
|
||||
|
||||
vidsFut = getVideos(timeline, agent)
|
||||
pollFut = getPolls(timeline, agent)
|
||||
cardFut = getCards(timeline, agent)
|
||||
|
||||
await all(vidsFut, pollFut, cardFut)
|
||||
result = (profile, timeline)
|
||||
|
||||
proc getProfileFull*(username: string): Future[Profile] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"authority": "twitter.com",
|
||||
"accept": accept,
|
||||
"referer": "https://twitter.com/" & username,
|
||||
"accept-language": lang
|
||||
})
|
||||
|
||||
let html = await fetchHtml(base / username, headers)
|
||||
if html == nil: return
|
||||
result = parseTimelineProfile(html)
|
||||
|
|
|
|||
|
|
@ -9,23 +9,36 @@ withDb:
|
|||
|
||||
var profileCacheTime = initDuration(minutes=10)
|
||||
|
||||
proc outdated(profile: Profile): bool =
|
||||
proc isOutdated*(profile: Profile): bool =
|
||||
getTime() - profile.updated > profileCacheTime
|
||||
|
||||
proc cache*(profile: var Profile) =
|
||||
withDb:
|
||||
try:
|
||||
let p = Profile.getOne("lower(username) = ?", toLower(profile.username))
|
||||
profile.id = p.id
|
||||
profile.update()
|
||||
except KeyError:
|
||||
if profile.username.len > 0:
|
||||
profile.insert()
|
||||
|
||||
proc hasCachedProfile*(username: string): Option[Profile] =
|
||||
withDb:
|
||||
try:
|
||||
let p = Profile.getOne("lower(username) = ?", toLower(username))
|
||||
doAssert not p.isOutdated
|
||||
result = some(p)
|
||||
except AssertionError, KeyError:
|
||||
result = none(Profile)
|
||||
|
||||
proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} =
|
||||
withDb:
|
||||
try:
|
||||
result.getOne("username = ?", username)
|
||||
doAssert not result.outdated()
|
||||
except AssertionError:
|
||||
var profile = await getProfile(username, agent)
|
||||
profile.id = result.id
|
||||
result = profile
|
||||
result.update()
|
||||
except KeyError:
|
||||
result = await getProfile(username, agent)
|
||||
if result.username.len > 0:
|
||||
result.insert()
|
||||
result.getOne("lower(username) = ?", toLower(username))
|
||||
doAssert not result.isOutdated
|
||||
except AssertionError, KeyError:
|
||||
result = await getProfileFull(username)
|
||||
cache(result)
|
||||
|
||||
proc setProfileCacheTime*(minutes: int) =
|
||||
profileCacheTime = initDuration(minutes=minutes)
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ proc stripTwitterUrls*(text: string): string =
|
|||
result = result.replace(ellipsisRegex, "")
|
||||
|
||||
proc getUserpic*(userpic: string; style=""): string =
|
||||
let pic = userpic.replace(re"_(normal|bigger|mini|200x200)(\.[A-z]+)$", "$2")
|
||||
let pic = userpic.replace(re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$", "$2")
|
||||
pic.replace(re"(\.[A-z]+)$", style & "$1")
|
||||
|
||||
proc getUserpic*(profile: Profile; style=""): string =
|
||||
|
|
@ -77,6 +77,12 @@ proc pageTitle*(profile: Profile): string =
|
|||
proc pageDesc*(profile: Profile): string =
|
||||
"The latest tweets from " & profile.fullname
|
||||
|
||||
proc getJoinDate*(profile: Profile): string =
|
||||
profile.joinDate.format("'Joined' MMMM YYYY")
|
||||
|
||||
proc getJoinDateFull*(profile: Profile): string =
|
||||
profile.joinDate.format("h:mm tt - d MMM YYYY")
|
||||
|
||||
proc getTime*(tweet: Tweet): string =
|
||||
tweet.time.format("d/M/yyyy', ' HH:mm:ss")
|
||||
|
||||
|
|
|
|||
|
|
@ -10,20 +10,31 @@ const configPath {.strdefine.} = "./nitter.conf"
|
|||
let cfg = getConfig(configPath)
|
||||
|
||||
proc showSingleTimeline(name, after, agent: string; query: Option[Query]): Future[string] {.async.} =
|
||||
let profileFut = getCachedProfile(name, agent)
|
||||
let railFut = getPhotoRail(name, agent)
|
||||
|
||||
var timelineFut: Future[Timeline]
|
||||
if query.isNone:
|
||||
timelineFut = getTimeline(name, after, agent)
|
||||
else:
|
||||
timelineFut = getTimelineSearch(get(query), after, agent)
|
||||
var timeline: Timeline
|
||||
var profile: Profile
|
||||
var cachedProfile = hasCachedProfile(name)
|
||||
|
||||
if cachedProfile.isSome:
|
||||
profile = get(cachedProfile)
|
||||
|
||||
if query.isNone:
|
||||
if cachedProfile.isSome:
|
||||
timeline = await getTimeline(name, after, agent)
|
||||
else:
|
||||
(profile, timeline) = await getProfileAndTimeline(name, agent, after)
|
||||
cache(profile)
|
||||
else:
|
||||
var timelineFut = getTimelineSearch(get(query), after, agent)
|
||||
if cachedProfile.isNone:
|
||||
profile = await getCachedProfile(name, agent)
|
||||
timeline = await timelineFut
|
||||
|
||||
let profile = await profileFut
|
||||
if profile.username.len == 0:
|
||||
return ""
|
||||
|
||||
let profileHtml = renderProfile(profile, await timelineFut, await railFut)
|
||||
let profileHtml = renderProfile(profile, timeline, await railFut)
|
||||
return renderMain(profileHtml, title=cfg.title, titleText=pageTitle(profile), desc=pageDesc(profile))
|
||||
|
||||
proc showMultiTimeline(names: seq[string]; after, agent: string; query: Option[Query]): Future[string] {.async.} =
|
||||
|
|
|
|||
|
|
@ -2,6 +2,26 @@ import xmltree, sequtils, strutils, json
|
|||
|
||||
import types, parserutils, formatters
|
||||
|
||||
proc parseTimelineProfile*(node: XmlNode): Profile =
|
||||
let profile = node.select(".ProfileHeaderCard")
|
||||
if profile == nil: return
|
||||
|
||||
let pre = ".ProfileHeaderCard-"
|
||||
result = Profile(
|
||||
fullname: profile.getName(pre & "nameLink"),
|
||||
username: profile.getUsername(pre & "screenname"),
|
||||
joinDate: profile.getDate(pre & "joinDateText"),
|
||||
location: profile.selectText(pre & "locationText").stripText(),
|
||||
website: profile.selectText(pre & "url").stripText(),
|
||||
bio: profile.getBio(pre & "bio"),
|
||||
userpic: node.getAvatar(".profile-picture img"),
|
||||
verified: isVerified(profile),
|
||||
protected: isProtected(profile),
|
||||
banner: getTimelineBanner(node)
|
||||
)
|
||||
|
||||
result.getProfileStats(node.select(".ProfileNav-list"))
|
||||
|
||||
proc parsePopupProfile*(node: XmlNode): Profile =
|
||||
let profile = node.select(".profile-card")
|
||||
if profile == nil: return
|
||||
|
|
@ -125,6 +145,16 @@ proc parseConversation*(node: XmlNode): Conversation =
|
|||
else:
|
||||
result.replies.add parseThread(thread)
|
||||
|
||||
proc parseTimeline*(node: XmlNode; after: string): Timeline =
|
||||
if node == nil: return
|
||||
result = Timeline(
|
||||
tweets: parseThread(node.select(".stream > .stream-items")).tweets,
|
||||
minId: node.attr("data-min-position"),
|
||||
maxId: node.attr("data-max-position"),
|
||||
hasMore: node.select(".has-more-items") != nil,
|
||||
beginning: after.len == 0
|
||||
)
|
||||
|
||||
proc parseVideo*(node: JsonNode; tweetId: string): Video =
|
||||
let
|
||||
track = node{"track"}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ proc getHeader(profile: XmlNode): XmlNode =
|
|||
result = profile.select(".stream-item-header")
|
||||
if result == nil:
|
||||
result = profile.select(".ProfileCard-userFields")
|
||||
if result == nil:
|
||||
result = profile
|
||||
|
||||
proc isVerified*(profile: XmlNode): bool =
|
||||
getHeader(profile).select(".Icon.Icon--verified") != nil
|
||||
|
|
@ -39,12 +41,6 @@ proc isVerified*(profile: XmlNode): bool =
|
|||
proc isProtected*(profile: XmlNode): bool =
|
||||
getHeader(profile).select(".Icon.Icon--protected") != nil
|
||||
|
||||
proc getName*(profile: XmlNode; selector: string): string =
|
||||
profile.selectText(selector).stripText()
|
||||
|
||||
proc getUsername*(profile: XmlNode; selector: string): string =
|
||||
profile.selectText(selector).strip(chars={'@', ' '})
|
||||
|
||||
proc emojify*(node: XmlNode) =
|
||||
for i in node.selectAll(".Emoji"):
|
||||
i.add newText(i.attr("alt"))
|
||||
|
|
@ -79,23 +75,54 @@ proc getTimestamp*(tweet: XmlNode): Time =
|
|||
proc getShortTime*(tweet: XmlNode): string =
|
||||
getTime(tweet).innerText()
|
||||
|
||||
proc getDate*(node: XmlNode; selector: string): Time =
|
||||
let date = node.select(selector)
|
||||
if date == nil: return
|
||||
parseTime(date.attr("title"), "h:mm tt - d MMM YYYY", utc())
|
||||
|
||||
proc getName*(profile: XmlNode; selector: string): string =
|
||||
profile.selectText(selector).stripText()
|
||||
|
||||
proc getUsername*(profile: XmlNode; selector: string): string =
|
||||
profile.selectText(selector).strip(chars={'@', ' ', '\n'})
|
||||
|
||||
proc getBio*(profile: XmlNode; selector: string): string =
|
||||
profile.selectText(selector).stripText()
|
||||
|
||||
proc getAvatar*(profile: XmlNode; selector: string): string =
|
||||
profile.selectAttr(selector, "src").getUserpic()
|
||||
|
||||
proc getBanner*(tweet: XmlNode): string =
|
||||
let url = tweet.selectAttr("svg > image", "xlink:href")
|
||||
proc getBanner*(node: XmlNode): string =
|
||||
let url = node.selectAttr("svg > image", "xlink:href")
|
||||
if url.len > 0:
|
||||
result = url.replace("600x200", "1500x500")
|
||||
else:
|
||||
result = tweet.selectAttr(".ProfileCard-bg", "style")
|
||||
result = node.selectAttr(".ProfileCard-bg", "style")
|
||||
result = result.replace("background-color: ", "")
|
||||
|
||||
if result.len == 0:
|
||||
result = "#161616"
|
||||
|
||||
proc getTimelineBanner*(node: XmlNode): string =
|
||||
let banner = node.select(".ProfileCanopy-headerBg img")
|
||||
let img = banner.attr("src")
|
||||
if img.len > 0:
|
||||
return img
|
||||
|
||||
let style = node.select("style").innerText()
|
||||
var m: RegexMatch
|
||||
if style.find(re"a:active \{\n +color: (#[A-Z0-9]+)", m):
|
||||
return style[m.group(0)[0]]
|
||||
|
||||
proc getProfileStats*(profile: var Profile; node: XmlNode) =
|
||||
for s in node.selectAll( ".ProfileNav-stat"):
|
||||
let text = s.attr("title").split(" ")[0]
|
||||
case s.attr("data-nav")
|
||||
of "followers": profile.followers = text
|
||||
of "following": profile.following = text
|
||||
of "favorites": profile.likes = text
|
||||
of "tweets": profile.tweets = text
|
||||
|
||||
proc getPopupStats*(profile: var Profile; node: XmlNode) =
|
||||
for s in node.selectAll( ".ProfileCardStats-statLink"):
|
||||
let text = s.attr("title").split(" ")[0]
|
||||
|
|
|
|||
|
|
@ -12,12 +12,15 @@ db("cache.db", "", "", ""):
|
|||
Profile* = object
|
||||
username*: string
|
||||
fullname*: string
|
||||
location*: string
|
||||
website*: string
|
||||
bio*: string
|
||||
userpic*: string
|
||||
banner*: string
|
||||
following*: string
|
||||
followers*: string
|
||||
tweets*: string
|
||||
likes*: string
|
||||
verified* {.
|
||||
dbType: "STRING",
|
||||
parseIt: parseBool(it.s)
|
||||
|
|
@ -28,6 +31,11 @@ db("cache.db", "", "", ""):
|
|||
parseIt: parseBool(it.s)
|
||||
formatIt: $it
|
||||
.}: bool
|
||||
joinDate* {.
|
||||
dbType: "INTEGER",
|
||||
parseIt: it.i.fromUnix(),
|
||||
formatIt: it.toUnix()
|
||||
.}: Time
|
||||
updated* {.
|
||||
dbType: "INTEGER",
|
||||
parseIt: it.i.fromUnix(),
|
||||
|
|
|
|||
|
|
@ -15,21 +15,35 @@ proc renderProfileCard*(profile: Profile): VNode =
|
|||
a(class="profile-card-avatar", href=profile.getUserPic().getSigUrl("pic")):
|
||||
genImg(profile.getUserpic("_200x200"))
|
||||
|
||||
tdiv(class="profile-card-tabs"):
|
||||
tdiv(class="profile-card-tabs-name"):
|
||||
linkUser(profile, class="profile-card-fullname")
|
||||
linkUser(profile, class="profile-card-username")
|
||||
tdiv(class="profile-card-tabs-name"):
|
||||
linkUser(profile, class="profile-card-fullname")
|
||||
linkUser(profile, class="profile-card-username")
|
||||
|
||||
tdiv(class="profile-card-extra"):
|
||||
if profile.bio.len > 0:
|
||||
tdiv(class="profile-bio"):
|
||||
p: verbatim linkifyText(profile.bio)
|
||||
|
||||
if profile.location.len > 0:
|
||||
tdiv(class="profile-location"):
|
||||
span: text "📍 " & profile.location
|
||||
|
||||
if profile.website.len > 0:
|
||||
tdiv(class="profile-website"):
|
||||
span:
|
||||
text "🔗 "
|
||||
a(href=profile.website): text profile.website
|
||||
|
||||
tdiv(class="profile-joindate"):
|
||||
span(title=getJoinDateFull(profile)):
|
||||
text "📅 " & getJoinDate(profile)
|
||||
|
||||
tdiv(class="profile-card-extra-links"):
|
||||
ul(class="profile-statlist"):
|
||||
renderStat(profile.tweets, "posts", text="Tweets")
|
||||
renderStat(profile.followers, "followers")
|
||||
renderStat(profile.following, "following")
|
||||
renderStat(profile.likes, "likes")
|
||||
|
||||
proc renderPhotoRail(username: string; photoRail: seq[GalleryPhoto]): VNode =
|
||||
buildHtml(tdiv(class="photo-rail-card")):
|
||||
|
|
|
|||
Loading…
Reference in New Issue