More API refactoring

This commit is contained in:
Zed 2019-10-02 22:28:53 +02:00
parent bafe5fb550
commit bb83b6b6d3
5 changed files with 21 additions and 29 deletions

View File

@ -1,4 +1,4 @@
import httpclient, asyncdispatch, htmlparser, strformat import httpclient, asyncdispatch, htmlparser
import sequtils, strutils, json, uri import sequtils, strutils, json, uri
import ".."/[types, parser, parserutils, query] import ".."/[types, parser, parserutils, query]
@ -29,8 +29,7 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.
proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} = proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
let let
url = base / (listMembersUrl % [username, list]) url = base / (listMembersUrl % [username, list])
referer = base / &"{username}/lists/{list}/members" html = await fetchHtml(url, genHeaders(agent, url))
html = await fetchHtml(url, genHeaders(agent, referer))
result = Result[Profile]( result = Result[Profile](
minId: html.selectAttr(".stream-container", "data-min-position"), minId: html.selectAttr(".stream-container", "data-min-position"),
@ -42,8 +41,8 @@ proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.a
proc getListMembersSearch*(username, list, agent, after: string): Future[Result[Profile]] {.async.} = proc getListMembersSearch*(username, list, agent, after: string): Future[Result[Profile]] {.async.} =
let let
url = base / ((listMembersUrl & "/timeline") % [username, list]) referer = base / (listMembersUrl % [username, list])
referer = base / &"{username}/lists/{list}/members" url = referer / "timeline"
headers = genHeaders({"x-push-with": "XMLHttpRequest"}, agent, referer, xml=true) headers = genHeaders({"x-push-with": "XMLHttpRequest"}, agent, referer, xml=true)
var params = toSeq({ var params = toSeq({

View File

@ -53,8 +53,7 @@ proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
let let
headers = genHeaders({"authorization": auth}, agent, base, lang=false) headers = genHeaders({"authorization": auth}, agent, base, lang=false)
url = apiBase / tokenUrl json = await fetchJson(apiBase / tokenUrl, headers)
json = await fetchJson(url, headers)
if json != nil: if json != nil:
result = json["guest_token"].to(string) result = json["guest_token"].to(string)

View File

@ -1,9 +1,14 @@
import httpclient, asyncdispatch, htmlparser, strformat import httpclient, asyncdispatch, htmlparser, strformat
import sequtils, strutils, json, xmltree, uri import sequtils, strutils, json, uri
import ".."/[types, parser, parserutils, formatters, query] import ".."/[types, parser, parserutils, formatters, query]
import utils, consts, media, search import utils, consts, media, search
proc getMedia(thread: Thread | Timeline; agent: string) {.async.} =
await all(getVideos(thread, agent),
getCards(thread, agent),
getPolls(thread, agent))
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future[Timeline] {.async.} = proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future[Timeline] {.async.} =
result = getResult[Tweet](json, query, after) result = getResult[Tweet](json, query, after)
if json == nil: return if json == nil: return
@ -11,19 +16,13 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future
if json["new_latent_count"].to(int) == 0: return if json["new_latent_count"].to(int) == 0: return
if not json.hasKey("items_html"): return if not json.hasKey("items_html"): return
let let html = parseHtml(json["items_html"].to(string))
html = parseHtml(json["items_html"].to(string)) let thread = parseThread(html)
thread = parseThread(html)
vidsFut = getVideos(thread, agent)
pollFut = getPolls(thread, agent)
cardFut = getCards(thread, agent)
await all(vidsFut, pollFut, cardFut) await getMedia(thread, agent)
result.content = thread.content result.content = thread.content
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} = proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
let headers = genHeaders(agent, base / username, xml=true)
var params = toSeq({ var params = toSeq({
"include_available_features": "1", "include_available_features": "1",
"include_entities": "1", "include_entities": "1",
@ -34,7 +33,9 @@ proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
if after.len > 0: if after.len > 0:
params.add {"max_position": after} params.add {"max_position": after}
let headers = genHeaders(agent, base / username, xml=true)
let json = await fetchJson(base / (timelineUrl % username) ? params, headers) let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
result = await finishTimeline(json, Query(), after, agent) result = await finishTimeline(json, Query(), after, agent)
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} = proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
@ -48,9 +49,5 @@ proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Ti
timeline = parseTimeline(html.select("#timeline > .stream-container"), after) timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
profile = parseTimelineProfile(html) profile = parseTimelineProfile(html)
vidsFut = getVideos(timeline, agent) await getMedia(timeline, agent)
pollFut = getPolls(timeline, agent)
cardFut = getCards(timeline, agent)
await all(vidsFut, pollFut, cardFut)
result = (profile, timeline) result = (profile, timeline)

View File

@ -17,9 +17,6 @@ proc getTweet*(username, id, after, agent: string): Future[Conversation] {.async
result = parseConversation(html, after) result = parseConversation(html, after)
let await all(getConversationVideos(result, agent),
vidsFut = getConversationVideos(result, agent) getConversationCards(result, agent),
pollFut = getConversationPolls(result, agent) getConversationPolls(result, agent))
cardFut = getConversationCards(result, agent)
await all(vidsFut, pollFut, cardFut)

View File

@ -18,7 +18,7 @@ thread = [
['d0m96/status/1141811379407425537', [], 'I\'m', ['d0m96/status/1141811379407425537', [], 'I\'m',
['The', 'The', 'Today', 'Some', 'If', 'There', 'Above'], ['The', 'The', 'Today', 'Some', 'If', 'There', 'Above'],
[['Thank', 'Also,', 'Awesome.', 'Thank']]], [['Awesome.']]],
['gmpreussner/status/999766552546299904', [], 'A', [], ['gmpreussner/status/999766552546299904', [], 'A', [],
[['I'], ['I']]] [['I'], ['I']]]