diff --git a/src/api.nim b/src/api.nim index 1de4e32..2cffc66 100644 --- a/src/api.nim +++ b/src/api.nim @@ -1,388 +1,2 @@ -import httpclient, asyncdispatch, htmlparser, times -import sequtils, strutils, json, xmltree, uri - -import types, parser, parserutils, formatters, search - -const - lang = "en-US,en;q=0.9" - auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" - accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" - jsonAccept = "application/json, text/javascript, */*; q=0.01" - - base = parseUri("https://twitter.com/") - apiBase = parseUri("https://api.twitter.com/1.1/") - - timelineUrl = "i/profiles/show/$1/timeline/tweets" - timelineMediaUrl = "i/profiles/show/$1/media_timeline" - profilePopupUrl = "i/profiles/popup" - profileIntentUrl = "intent/user" - searchUrl = "i/search/timeline" - tweetUrl = "status" - videoUrl = "videos/tweet/config/$1.json" - tokenUrl = "guest/activate.json" - cardUrl = "i/cards/tfw/v1/$1" - pollUrl = cardUrl & "?cardname=poll2choice_text_only&lang=en" - -var - guestToken = "" - tokenUses = 0 - tokenMaxUses = 230 - tokenUpdated: Time - tokenLifetime = initDuration(minutes=20) - -macro genMediaGet(media: untyped; token=false) = - let - mediaName = capitalizeAscii($media) - multi = ident("get" & mediaName & "s") - convo = ident("getConversation" & mediaName & "s") - single = ident("get" & mediaName) - - quote do: - proc `multi`(thread: Thread | Timeline; agent: string; token="") {.async.} = - if thread == nil: return - var `media` = thread.content.filterIt(it.`media`.isSome) - when `token`: - var gToken = token - if gToken.len == 0: gToken = await getGuestToken(agent) - await all(`media`.mapIt(`single`(it, token, agent))) - else: - await all(`media`.mapIt(`single`(it, agent))) - - proc `convo`(convo: Conversation; agent: string) {.async.} = - var futs: seq[Future[void]] - when `token`: - var token = await getGuestToken(agent) - futs.add `single`(convo.tweet, agent, token) - futs.add `multi`(convo.before, agent, token=token) - futs.add `multi`(convo.after, agent, token=token) - futs.add convo.replies.mapIt(`multi`(it, agent, token=token)) - else: - futs.add `single`(convo.tweet, agent) - futs.add `multi`(convo.before, agent) - futs.add `multi`(convo.after, agent) - futs.add convo.replies.mapIt(`multi`(it, agent)) - await all(futs) - -template newClient() {.dirty.} = - var client = newAsyncHttpClient() - defer: client.close() - client.headers = headers - -proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} = - newClient() - - var resp = "" - try: - resp = await client.getContent($url) - except: - return nil - - if jsonKey.len > 0: - let json = parseJson(resp)[jsonKey].str - return parseHtml(json) - else: - return parseHtml(resp) - -proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} = - newClient() - - var resp = "" - try: - resp = await client.getContent($url) - result = parseJson(resp) - except: - return nil - -proc getGuestToken(agent: string; force=false): Future[string] {.async.} = - if getTime() - tokenUpdated < tokenLifetime and - not force and tokenUses < tokenMaxUses: - return guestToken - - tokenUpdated = getTime() - tokenUses = 0 - - let headers = newHttpHeaders({ - "Accept": jsonAccept, - "Referer": $base, - "User-Agent": agent, - "Authorization": auth - }) - - newClient() - - let - url = apiBase / tokenUrl - json = parseJson(await client.postContent($url)) - - result = json["guest_token"].to(string) - guestToken = result - -proc getVideoFetch*(tweet: Tweet; agent, token: string) {.async.} = - if tweet.video.isNone(): return - - let headers = newHttpHeaders({ - "Accept": jsonAccept, - "Referer": $(base / getLink(tweet)), - "User-Agent": agent, - "Authorization": auth, - "x-guest-token": token - }) - - let url = apiBase / (videoUrl % tweet.id) - let json = await fetchJson(url, headers) - - if json == nil: - if getTime() - tokenUpdated > initDuration(seconds=1): - tokenUpdated = getTime() - discard await getGuestToken(agent, force=true) - await getVideoFetch(tweet, agent, guestToken) - return - - if tweet.card.isNone: - tweet.video = some(parseVideo(json, tweet.id)) - else: - get(tweet.card).video = some(parseVideo(json, tweet.id)) - tweet.video = none(Video) - tokenUses.inc - -proc getVideoVar*(tweet: Tweet): var Option[Video] = - if tweet.card.isSome(): - return get(tweet.card).video - else: - return tweet.video - -proc getVideo*(tweet: Tweet; agent, token: string; force=false) {.async.} = - withDb: - try: - getVideoVar(tweet) = some(Video.getOne("videoId = ?", tweet.id)) - except KeyError: - await getVideoFetch(tweet, agent, token) - var video = getVideoVar(tweet) - if video.isSome(): - get(video).insert() - -proc getPoll*(tweet: Tweet; agent: string) {.async.} = - if tweet.poll.isNone(): return - - let headers = newHttpHeaders({ - "Accept": accept, - "Referer": $(base / getLink(tweet)), - "User-Agent": agent, - "Authority": "twitter.com", - "Accept-Language": lang, - }) - - let url = base / (pollUrl % tweet.id) - let html = await fetchHtml(url, headers) - if html == nil: return - - tweet.poll = some(parsePoll(html)) - -proc getCard*(tweet: Tweet; agent: string) {.async.} = - if tweet.card.isNone(): return - - let headers = newHttpHeaders({ - "Accept": accept, - "Referer": $(base / getLink(tweet)), - "User-Agent": agent, - "Authority": "twitter.com", - "Accept-Language": lang, - }) - - let query = get(tweet.card).query.replace("sensitive=true", "sensitive=false") - let html = await fetchHtml(base / query, headers) - if html == nil: return - - parseCard(get(tweet.card), html) - -genMediaGet(video, token=true) -genMediaGet(poll) -genMediaGet(card) - -proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} = - let headers = newHttpHeaders({ - "Accept": jsonAccept, - "Referer": $(base / username), - "User-Agent": agent, - "X-Requested-With": "XMLHttpRequest" - }) - - let params = { - "for_photo_rail": "true", - "oldest_unread_id": "0" - } - - let url = base / (timelineMediaUrl % username) ? params - let html = await fetchHtml(url, headers, jsonKey="items_html") - - result = parsePhotoRail(html) - -proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} = - let url = base / profileIntentUrl ? {"screen_name": username} - let html = await fetchHtml(url, headers) - if html == nil: return Profile() - - result = parseIntentProfile(html) - -proc getProfile*(username, agent: string): Future[Profile] {.async.} = - let headers = newHttpHeaders({ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9", - "Referer": $(base / username), - "User-Agent": agent, - "X-Twitter-Active-User": "yes", - "X-Requested-With": "XMLHttpRequest", - "Accept-Language": lang - }) - - let - params = { - "screen_name": username, - "wants_hovercard": "true", - "_": $(epochTime().int) - } - url = base / profilePopupUrl ? params - html = await fetchHtml(url, headers, jsonKey="html") - - if html == nil: return Profile() - - if html.select(".ProfileCard-sensitiveWarningContainer") != nil: - return await getProfileFallback(username, headers) - - result = parsePopupProfile(html) - -proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} = - let headers = newHttpHeaders({ - "Accept": jsonAccept, - "Referer": $base, - "User-Agent": agent, - "X-Twitter-Active-User": "yes", - "X-Requested-With": "XMLHttpRequest", - "Accept-Language": lang, - "pragma": "no-cache", - "x-previous-page-name": "profile" - }) - - let - url = base / username / tweetUrl / id - html = await fetchHtml(url, headers) - - if html == nil: return - - result = parseConversation(html) - - let - vidsFut = getConversationVideos(result, agent) - pollFut = getConversationPolls(result, agent) - cardFut = getConversationCards(result, agent) - - await all(vidsFut, pollFut, cardFut) - -proc finishTimeline(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} = - if json == nil: return Timeline() - - result = Timeline( - hasMore: json["has_more_items"].to(bool), - maxId: json.getOrDefault("max_position").getStr(""), - minId: json.getOrDefault("min_position").getStr("").cleanPos(), - query: query, - beginning: after.len == 0 - ) - - if json["new_latent_count"].to(int) == 0: return - if not json.hasKey("items_html"): return - - let - html = parseHtml(json["items_html"].to(string)) - thread = parseThread(html) - vidsFut = getVideos(thread, agent) - pollFut = getPolls(thread, agent) - cardFut = getCards(thread, agent) - - await all(vidsFut, pollFut, cardFut) - result.content = thread.content - -proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} = - let headers = newHttpHeaders({ - "Accept": jsonAccept, - "Referer": $(base / username), - "User-Agent": agent, - "X-Twitter-Active-User": "yes", - "X-Requested-With": "XMLHttpRequest", - "Accept-Language": lang - }) - - var params = toSeq({ - "include_available_features": "1", - "include_entities": "1", - "include_new_items_bar": "false", - "reset_error_state": "false" - }) - - if after.len > 0: - params.add {"max_position": after} - - let json = await fetchJson(base / (timelineUrl % username) ? params, headers) - result = await finishTimeline(json, none(Query), after, agent) - -proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.async.} = - let queryParam = genQueryParam(query) - let queryEncoded = encodeUrl(queryParam, usePlus=false) - - let headers = newHttpHeaders({ - "Accept": jsonAccept, - "Referer": $(base / ("search?f=tweets&vertical=default&q=$1&src=typd" % queryEncoded)), - "User-Agent": agent, - "X-Requested-With": "XMLHttpRequest", - "Authority": "twitter.com", - "Accept-Language": lang - }) - - let params = { - "f": "tweets", - "vertical": "default", - "q": queryParam, - "src": "typd", - "include_available_features": "1", - "include_entities": "1", - "max_position": if after.len > 0: genPos(after) else: "0", - "reset_error_state": "false" - } - - let json = await fetchJson(base / searchUrl ? params, headers) - result = await finishTimeline(json, some(query), after, agent) - -proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} = - let headers = newHttpHeaders({ - "authority": "twitter.com", - "accept": accept, - "referer": "https://twitter.com/" & username, - "accept-language": lang - }) - - var url = base / username - if after.len > 0: - url = url ? {"max_position": after} - - let - html = await fetchHtml(url, headers) - timeline = parseTimeline(html.select("#timeline > .stream-container"), after) - profile = parseTimelineProfile(html) - - vidsFut = getVideos(timeline, agent) - pollFut = getPolls(timeline, agent) - cardFut = getCards(timeline, agent) - - await all(vidsFut, pollFut, cardFut) - result = (profile, timeline) - -proc getProfileFull*(username: string): Future[Profile] {.async.} = - let headers = newHttpHeaders({ - "authority": "twitter.com", - "accept": accept, - "referer": "https://twitter.com/" & username, - "accept-language": lang - }) - - let html = await fetchHtml(base / username, headers) - if html == nil: return - result = parseTimelineProfile(html) +import api/[media, profile, timeline, tweet, search] +export profile, timeline, tweet, search, media diff --git a/src/api/consts.nim b/src/api/consts.nim new file mode 100644 index 0000000..0be86d3 --- /dev/null +++ b/src/api/consts.nim @@ -0,0 +1,21 @@ +import uri + +const + lang* = "en-US,en;q=0.9" + auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" + htmlAccept* = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" + jsonAccept* = "application/json, text/javascript, */*; q=0.01" + + base* = parseUri("https://twitter.com/") + apiBase* = parseUri("https://api.twitter.com/1.1/") + + timelineUrl* = "i/profiles/show/$1/timeline/tweets" + timelineMediaUrl* = "i/profiles/show/$1/media_timeline" + profilePopupUrl* = "i/profiles/popup" + profileIntentUrl* = "intent/user" + searchUrl* = "i/search/timeline" + tweetUrl* = "status" + videoUrl* = "videos/tweet/config/$1.json" + tokenUrl* = "guest/activate.json" + cardUrl* = "i/cards/tfw/v1/$1" + pollUrl* = cardUrl & "?cardname=poll2choice_text_only&lang=en" diff --git a/src/api/media.nim b/src/api/media.nim new file mode 100644 index 0000000..608b034 --- /dev/null +++ b/src/api/media.nim @@ -0,0 +1,168 @@ +import httpclient, asyncdispatch, times, sequtils, strutils, json, uri + +import ".."/[types, parser, formatters] +import utils, consts + +var + guestToken = "" + tokenUses = 0 + tokenMaxUses = 230 + tokenUpdated: Time + tokenLifetime = initDuration(minutes=20) + +macro genMediaGet(media: untyped; token=false) = + let + mediaName = capitalizeAscii($media) + multi = ident("get" & mediaName & "s") + convo = ident("getConversation" & mediaName & "s") + single = ident("get" & mediaName) + + quote do: + proc `multi`*(thread: Thread | Timeline; agent: string; token="") {.async.} = + if thread == nil: return + var `media` = thread.content.filterIt(it.`media`.isSome) + when `token`: + var gToken = token + if gToken.len == 0: gToken = await getGuestToken(agent) + await all(`media`.mapIt(`single`(it, token, agent))) + else: + await all(`media`.mapIt(`single`(it, agent))) + + proc `convo`*(convo: Conversation; agent: string) {.async.} = + var futs: seq[Future[void]] + when `token`: + var token = await getGuestToken(agent) + futs.add `single`(convo.tweet, agent, token) + futs.add `multi`(convo.before, agent, token=token) + futs.add `multi`(convo.after, agent, token=token) + futs.add convo.replies.mapIt(`multi`(it, agent, token=token)) + else: + futs.add `single`(convo.tweet, agent) + futs.add `multi`(convo.before, agent) + futs.add `multi`(convo.after, agent) + futs.add convo.replies.mapIt(`multi`(it, agent)) + await all(futs) + +proc getGuestToken(agent: string; force=false): Future[string] {.async.} = + if getTime() - tokenUpdated < tokenLifetime and + not force and tokenUses < tokenMaxUses: + return guestToken + + tokenUpdated = getTime() + tokenUses = 0 + + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $base, + "User-Agent": agent, + "Authorization": auth + }) + + newClient() + + let + url = apiBase / tokenUrl + json = parseJson(await client.postContent($url)) + + result = json["guest_token"].to(string) + guestToken = result + +proc getVideoFetch(tweet: Tweet; agent, token: string) {.async.} = + if tweet.video.isNone(): return + + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $(base / getLink(tweet)), + "User-Agent": agent, + "Authorization": auth, + "x-guest-token": token + }) + + let url = apiBase / (videoUrl % tweet.id) + let json = await fetchJson(url, headers) + + if json == nil: + if getTime() - tokenUpdated > initDuration(seconds=1): + tokenUpdated = getTime() + discard await getGuestToken(agent, force=true) + await getVideoFetch(tweet, agent, guestToken) + return + + if tweet.card.isNone: + tweet.video = some(parseVideo(json, tweet.id)) + else: + get(tweet.card).video = some(parseVideo(json, tweet.id)) + tweet.video = none(Video) + tokenUses.inc + +proc getVideoVar(tweet: Tweet): var Option[Video] = + if tweet.card.isSome(): + return get(tweet.card).video + else: + return tweet.video + +proc getVideo*(tweet: Tweet; agent, token: string; force=false) {.async.} = + withDb: + try: + getVideoVar(tweet) = some(Video.getOne("videoId = ?", tweet.id)) + except KeyError: + await getVideoFetch(tweet, agent, token) + var video = getVideoVar(tweet) + if video.isSome(): + get(video).insert() + +proc getPoll*(tweet: Tweet; agent: string) {.async.} = + if tweet.poll.isNone(): return + + let headers = newHttpHeaders({ + "Accept": htmlAccept, + "Referer": $(base / getLink(tweet)), + "User-Agent": agent, + "Authority": "twitter.com", + "Accept-Language": lang, + }) + + let url = base / (pollUrl % tweet.id) + let html = await fetchHtml(url, headers) + if html == nil: return + + tweet.poll = some(parsePoll(html)) + +proc getCard*(tweet: Tweet; agent: string) {.async.} = + if tweet.card.isNone(): return + + let headers = newHttpHeaders({ + "Accept": htmlAccept, + "Referer": $(base / getLink(tweet)), + "User-Agent": agent, + "Authority": "twitter.com", + "Accept-Language": lang, + }) + + let query = get(tweet.card).query.replace("sensitive=true", "sensitive=false") + let html = await fetchHtml(base / query, headers) + if html == nil: return + + parseCard(get(tweet.card), html) + +proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} = + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $(base / username), + "User-Agent": agent, + "X-Requested-With": "XMLHttpRequest" + }) + + let params = { + "for_photo_rail": "true", + "oldest_unread_id": "0" + } + + let url = base / (timelineMediaUrl % username) ? params + let html = await fetchHtml(url, headers, jsonKey="items_html") + + result = parsePhotoRail(html) + +genMediaGet(video, token=true) +genMediaGet(poll) +genMediaGet(card) diff --git a/src/api/profile.nim b/src/api/profile.nim new file mode 100644 index 0000000..363f727 --- /dev/null +++ b/src/api/profile.nim @@ -0,0 +1,50 @@ +import httpclient, asyncdispatch, times, strutils, uri + +import ".."/[types, parser, parserutils] + +import utils, consts + +proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} = + let url = base / profileIntentUrl ? {"screen_name": username} + let html = await fetchHtml(url, headers) + if html == nil: return Profile() + + result = parseIntentProfile(html) + +proc getProfile*(username, agent: string): Future[Profile] {.async.} = + let headers = newHttpHeaders({ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9", + "Referer": $(base / username), + "User-Agent": agent, + "X-Twitter-Active-User": "yes", + "X-Requested-With": "XMLHttpRequest", + "Accept-Language": lang + }) + + let + params = { + "screen_name": username, + "wants_hovercard": "true", + "_": $(epochTime().int) + } + url = base / profilePopupUrl ? params + html = await fetchHtml(url, headers, jsonKey="html") + + if html == nil: return Profile() + + if html.select(".ProfileCard-sensitiveWarningContainer") != nil: + return await getProfileFallback(username, headers) + + result = parsePopupProfile(html) + +proc getProfileFull*(username: string): Future[Profile] {.async.} = + let headers = newHttpHeaders({ + "authority": "twitter.com", + "accept": htmlAccept, + "referer": "https://twitter.com/" & username, + "accept-language": lang + }) + + let html = await fetchHtml(base / username, headers) + if html == nil: return + result = parseTimelineProfile(html) diff --git a/src/api/search.nim b/src/api/search.nim new file mode 100644 index 0000000..f07a864 --- /dev/null +++ b/src/api/search.nim @@ -0,0 +1,32 @@ +import httpclient, asyncdispatch, htmlparser +import sequtils, strutils, json, xmltree, uri + +import ".."/[types, parser, parserutils, formatters, search] +import utils, consts, media, timeline + +proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.async.} = + let queryParam = genQueryParam(query) + let queryEncoded = encodeUrl(queryParam, usePlus=false) + + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $(base / ("search?f=tweets&vertical=default&q=$1&src=typd" % queryEncoded)), + "User-Agent": agent, + "X-Requested-With": "XMLHttpRequest", + "Authority": "twitter.com", + "Accept-Language": lang + }) + + let params = { + "f": "tweets", + "vertical": "default", + "q": queryParam, + "src": "typd", + "include_available_features": "1", + "include_entities": "1", + "max_position": if after.len > 0: genPos(after) else: "0", + "reset_error_state": "false" + } + + let json = await fetchJson(base / searchUrl ? params, headers) + result = await finishTimeline(json, some(query), after, agent) diff --git a/src/api/timeline.nim b/src/api/timeline.nim new file mode 100644 index 0000000..e917152 --- /dev/null +++ b/src/api/timeline.nim @@ -0,0 +1,76 @@ +import httpclient, asyncdispatch, htmlparser +import sequtils, strutils, json, xmltree, uri + +import ".."/[types, parser, parserutils, formatters, search] +import utils, consts, media + +proc finishTimeline*(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} = + if json == nil: return Timeline() + + result = Timeline( + hasMore: json["has_more_items"].to(bool), + maxId: json.getOrDefault("max_position").getStr(""), + minId: json.getOrDefault("min_position").getStr("").cleanPos(), + query: query, + beginning: after.len == 0 + ) + + if json["new_latent_count"].to(int) == 0: return + if not json.hasKey("items_html"): return + + let + html = parseHtml(json["items_html"].to(string)) + thread = parseThread(html) + vidsFut = getVideos(thread, agent) + pollFut = getPolls(thread, agent) + cardFut = getCards(thread, agent) + + await all(vidsFut, pollFut, cardFut) + result.content = thread.content + +proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} = + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $(base / username), + "User-Agent": agent, + "X-Twitter-Active-User": "yes", + "X-Requested-With": "XMLHttpRequest", + "Accept-Language": lang + }) + + var params = toSeq({ + "include_available_features": "1", + "include_entities": "1", + "include_new_items_bar": "false", + "reset_error_state": "false" + }) + + if after.len > 0: + params.add {"max_position": after} + + let json = await fetchJson(base / (timelineUrl % username) ? params, headers) + result = await finishTimeline(json, none(Query), after, agent) + +proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} = + let headers = newHttpHeaders({ + "authority": "twitter.com", + "accept": htmlAccept, + "referer": "https://twitter.com/" & username, + "accept-language": lang + }) + + var url = base / username + if after.len > 0: + url = url ? {"max_position": after} + + let + html = await fetchHtml(url, headers) + timeline = parseTimeline(html.select("#timeline > .stream-container"), after) + profile = parseTimelineProfile(html) + + vidsFut = getVideos(timeline, agent) + pollFut = getPolls(timeline, agent) + cardFut = getCards(timeline, agent) + + await all(vidsFut, pollFut, cardFut) + result = (profile, timeline) diff --git a/src/api/tweet.nim b/src/api/tweet.nim new file mode 100644 index 0000000..c0b587a --- /dev/null +++ b/src/api/tweet.nim @@ -0,0 +1,31 @@ +import httpclient, asyncdispatch, strutils, uri + +import ".."/[types, parser] +import utils, consts, media + +proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} = + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $base, + "User-Agent": agent, + "X-Twitter-Active-User": "yes", + "X-Requested-With": "XMLHttpRequest", + "Accept-Language": lang, + "pragma": "no-cache", + "x-previous-page-name": "profile" + }) + + let + url = base / username / tweetUrl / id + html = await fetchHtml(url, headers) + + if html == nil: return + + result = parseConversation(html) + + let + vidsFut = getConversationVideos(result, agent) + pollFut = getConversationPolls(result, agent) + cardFut = getConversationCards(result, agent) + + await all(vidsFut, pollFut, cardFut) diff --git a/src/api/utils.nim b/src/api/utils.nim new file mode 100644 index 0000000..b76c18f --- /dev/null +++ b/src/api/utils.nim @@ -0,0 +1,32 @@ +import httpclient, asyncdispatch, htmlparser +import strutils, json, xmltree, uri + +template newClient*() {.dirty.} = + var client = newAsyncHttpClient() + defer: client.close() + client.headers = headers + +proc fetchHtml*(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} = + newClient() + + var resp = "" + try: + resp = await client.getContent($url) + except: + return nil + + if jsonKey.len > 0: + let json = parseJson(resp)[jsonKey].str + return parseHtml(json) + else: + return parseHtml(resp) + +proc fetchJson*(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} = + newClient() + + var resp = "" + try: + resp = await client.getContent($url) + result = parseJson(resp) + except: + return nil diff --git a/src/routes/router_utils.nim b/src/routes/router_utils.nim index 4acc82f..064499b 100644 --- a/src/routes/router_utils.nim +++ b/src/routes/router_utils.nim @@ -1,5 +1,3 @@ -import ../utils - template cookiePrefs*(): untyped {.dirty.} = getPrefs(request.cookies.getOrDefault("preferences")) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 1ea1bc1..c93509e 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -80,20 +80,17 @@ proc createTimelineRouter*(cfg: Config) = get "/@name/search": cond '.' notin @"name" - let prefs = cookiePrefs() let query = initQuery(@"filter", @"include", @"not", @"sep", @"name") respTimeline(await showTimeline(@"name", @"after", some(query), cookiePrefs(), getPath(), cfg.title)) get "/@name/replies": cond '.' notin @"name" - let prefs = cookiePrefs() respTimeline(await showTimeline(@"name", @"after", some(getReplyQuery(@"name")), cookiePrefs(), getPath(), cfg.title)) get "/@name/media": cond '.' notin @"name" - let prefs = cookiePrefs() respTimeline(await showTimeline(@"name", @"after", some(getMediaQuery(@"name")), cookiePrefs(), getPath(), cfg.title)) diff --git a/src/types.nim b/src/types.nim index 40c3cb5..4837a83 100644 --- a/src/types.nim +++ b/src/types.nim @@ -1,6 +1,5 @@ import times, sequtils, options import norm/sqlite -import prefs_impl export sqlite, options diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index 48978a0..2e2a0a5 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -1,5 +1,5 @@ import strutils -import karax/[karaxdsl, vdom, vstyles] +import karax/[karaxdsl, vdom] import ../types, ../utils diff --git a/src/views/status.nim b/src/views/status.nim index 24dac1e..0a4c918 100644 --- a/src/views/status.nim +++ b/src/views/status.nim @@ -1,8 +1,7 @@ -import strutils, strformat import karax/[karaxdsl, vdom] import ../types -import tweet, renderutils +import tweet proc renderMoreReplies(thread: Thread): VNode = let num = if thread.more != -1: $thread.more & " " else: ""