From 861ac7a593238c5c5135ae3be7feaf9c5e05bd85 Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 24 Jun 2019 05:14:14 +0200 Subject: [PATCH] Better video/gif support --- public/style.css | 5 +++ src/api.nim | 81 ++++++++++++++++++++++++++++++++++++++++++++- src/formatters.nim | 6 ---- src/parser.nim | 20 ++++++++--- src/parserutils.nim | 27 +++++++++------ src/types.nim | 31 +++++++++++++---- src/views/tweet.nim | 20 +++++------ 7 files changed, 151 insertions(+), 39 deletions(-) diff --git a/public/style.css b/public/style.css index 78eb2ca..a7ea7ee 100644 --- a/public/style.css +++ b/public/style.css @@ -593,3 +593,8 @@ nav { top: calc(50% - 20px); font-size: 20px; } + +video { + height: 100%; + width: 100%; +} diff --git a/src/api.nim b/src/api.nim index 2aa913f..7db6732 100644 --- a/src/api.nim +++ b/src/api.nim @@ -5,12 +5,18 @@ import nimquery, regex import ./types, ./parser const - base = parseUri("https://twitter.com/") agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" + + base = parseUri("https://twitter.com/") + apiBase = parseUri("https://api.twitter.com/1.1/") + timelineUrl = "i/profiles/show/$1/timeline/tweets?include_available_features=1&include_entities=1&include_new_items_bar=true" profilePopupUrl = "i/profiles/popup" profileIntentUrl = "intent/user" tweetUrl = "i/status/" + videoUrl = "videos/tweet/config/$1.json" + tokenUrl = "guest/activate.json" proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} = var client = newAsyncHttpClient() @@ -30,6 +36,20 @@ proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {. else: return parseHtml(resp) +proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} = + var client = newAsyncHttpClient() + defer: client.close() + + client.headers = headers + + var resp = "" + try: + resp = await client.getContent($url) + except: + return nil + + return parseJson(resp) + proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} = let url = base / profileIntentUrl ? {"screen_name": username} @@ -61,6 +81,63 @@ proc getProfile*(username: string): Future[Profile] {.async.} = result = parsePopupProfile(html) +proc getGuestToken(): Future[string] {.async.} = + let headers = newHttpHeaders({ + "Accept": "application/json, text/javascript, */*; q=0.01", + "Referer": $base, + "User-Agent": agent, + "Authorization": auth + }) + + let client = newAsyncHttpClient() + client.headers = headers + + let + url = apibase / tokenUrl + json = parseJson(await client.postContent($url)) + + result = json["guest_token"].to(string) + +proc getVideo*(tweet: Tweet; token: string) {.async.} = + let headers = newHttpHeaders({ + "Accept": "application/json, text/javascript, */*; q=0.01", + "Referer": tweet.link, + "User-Agent": agent, + "Authorization": auth, + "x-guest-token": token + }) + + let + url = apiBase / (videoUrl % tweet.id) + json = await fetchJson(url, headers) + + + tweet.video = some(parseVideo(json)) + +proc getVideos*(tweets: Tweets; token="") {.async.} = + if not tweets.anyIt(it.video.isSome): return + + var + token = if token.len > 0: token else: await getGuestToken() + videoFuts: seq[Future[void]] + + for tweet in tweets: + if tweet.video.isSome: + videoFuts.add getVideo(tweet, token) + + await all(videoFuts) + +proc getConversationVideos*(convo: Conversation) {.async.} = + var token = await getGuestToken() + var futs: seq[Future[void]] + + futs.add getVideo(convo.tweet, token) + futs.add getVideos(convo.before, token=token) + futs.add getVideos(convo.after, token=token) + futs.add convo.replies.mapIt(getVideos(it, token=token)) + + await all(futs) + proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} = let headers = newHttpHeaders({ "Accept": "application/json, text/javascript, */*; q=0.01", @@ -78,6 +155,7 @@ proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} = let html = await fetchHtml(base / url, headers, jsonKey="items_html") result = parseTweets(html) + await getVideos(result) proc getTweet*(id: string): Future[Conversation] {.async.} = let headers = newHttpHeaders({ @@ -96,3 +174,4 @@ proc getTweet*(id: string): Future[Conversation] {.async.} = html = await fetchHtml(url, headers) result = parseConversation(html) + await getConversationVideos(result) diff --git a/src/formatters.nim b/src/formatters.nim index a004e82..046d1df 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -65,12 +65,6 @@ proc getUserpic*(userpic: string; style=""): string = proc getUserpic*(profile: Profile; style=""): string = getUserPic(profile.userpic, style) -proc getGifSrc*(tweet: Tweet): string = - fmt"https://video.twimg.com/tweet_video/{tweet.gif.get()}.mp4" - -proc getGifThumb*(tweet: Tweet): string = - fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif.get()}.jpg" - proc formatName*(profile: Profile): string = result = xmltree.escape(profile.fullname) if profile.verified: diff --git a/src/parser.nim b/src/parser.nim index 32b7b8e..0e0cd38 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,4 +1,4 @@ -import xmltree, sequtils, strtabs, strutils, strformat +import xmltree, sequtils, strtabs, strutils, strformat, json import nimquery import ./types, ./parserutils @@ -40,7 +40,6 @@ proc parseTweetProfile*(profile: XmlNode): Profile = proc parseQuote*(tweet: XmlNode): Tweet = let tweet = tweet.querySelector(".QuoteTweet-innerContainer") - result = Tweet( id: tweet.getAttr("data-item-id"), link: tweet.getAttr("href"), @@ -77,8 +76,10 @@ proc parseTweets*(node: XmlNode): Tweets = node.querySelectorAll(".tweet").map(parseTweet) proc parseConversation*(node: XmlNode): Conversation = - result.tweet = parseTweet(node.querySelector(".permalink-tweet-container > .tweet")) - result.before = parseTweets(node.querySelector(".in-reply-to")) + result = Conversation( + tweet: parseTweet(node.querySelector(".permalink-tweet-container > .tweet")), + before: parseTweets(node.querySelector(".in-reply-to")) + ) let replies = node.querySelector(".replies-to") if replies.isNil: return @@ -89,3 +90,14 @@ proc parseConversation*(node: XmlNode): Conversation = let thread = parseTweets(reply) if not thread.anyIt(it in result.after): result.replies.add thread + +proc parseVideo*(node: JsonNode): Video = + let track = node{"track"} + result = Video( + thumb: node["posterImage"].to(string), + id: track["contentId"].to(string), + length: track["durationMs"].to(int), + views: track["viewCount"].to(string), + url: track["playbackUrl"].to(string), + available: track{"mediaAvailability"}["status"].to(string) == "available" + ) diff --git a/src/parserutils.nim b/src/parserutils.nim index ad9e4d8..c81d9ac 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -1,7 +1,7 @@ -import xmltree, strtabs, times +import xmltree, strtabs, strformat, times import nimquery, regex -import ./types, ./formatters +import ./types, ./formatters, ./api const thumbRegex = re".+:url\('([^']+)'\)" @@ -84,11 +84,10 @@ proc getIntentStats*(profile: var Profile; node: XmlNode) = of "followers": profile.followers = text of "following": profile.following = text -proc getTweetStats*(tweet: var Tweet; node: XmlNode) = +proc getTweetStats*(tweet: Tweet; node: XmlNode) = tweet.replies = "0" tweet.retweets = "0" tweet.likes = "0" - for action in node.querySelectorAll(".ProfileTweet-actionCountForAria"): let text = action.innerText.split() case text[1] @@ -96,16 +95,22 @@ proc getTweetStats*(tweet: var Tweet; node: XmlNode) = of "likes": tweet.likes = text[0] of "retweets": tweet.retweets = text[0] -proc getTweetMedia*(tweet: var Tweet; node: XmlNode) = +proc getGif(player: XmlNode): Gif = + let + thumb = player.getAttr("style").replace(thumbRegex, "$1") + id = thumb.replace(gifRegex, "$1") + url = fmt"https://video.twimg.com/tweet_video/{id}.mp4" + Gif(url: url, thumb: thumb) + +proc getTweetMedia*(tweet: Tweet; node: XmlNode) = for photo in node.querySelectorAll(".AdaptiveMedia-photoContainer"): tweet.photos.add photo.attrs["data-image-url"] - let player = node.selectAttr(".PlayableMedia-player", "style") - if player.len == 0: + let player = node.querySelector(".PlayableMedia") + if player.isNil: return - let thumb = player.replace(thumbRegex, "$1") - if "tweet_video" in thumb: - tweet.gif = some(thumb.replace(gifRegex, "$1")) + if "gif" in player.getAttr("class"): + tweet.gif = some(getGif(player.querySelector(".PlayableMedia-player"))) else: - tweet.videoThumb = some(thumb) + tweet.video = some(Video()) diff --git a/src/types.nim b/src/types.nim index 9cb4c52..cca8e8d 100644 --- a/src/types.nim +++ b/src/types.nim @@ -31,7 +31,26 @@ db("cache.db", "", "", ""): .}: Time type - Tweet* = object + Video* = object + id*: string + url*: string + thumb*: string + length*: int + views*: string + available*: bool + + Gif* = object + url*: string + thumb*: string + + Quote* = ref object + id*: string + profile*: Profile + link*: string + text*: string + video*: Option[Video] + + Tweet* = ref object id*: string profile*: Profile link*: string @@ -42,16 +61,16 @@ type retweets*: string likes*: string pinned*: bool - photos*: seq[string] + quote*: Option[Quote] retweetBy*: Option[string] - gif*: Option[string] - video*: Option[string] - videoThumb*: Option[string] retweetId*: Option[string] + gif*: Option[Gif] + video*: Option[Video] + photos*: seq[string] Tweets* = seq[Tweet] - Conversation* = object + Conversation* = ref object tweet*: Tweet before*: Tweets after*: Tweets diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 0e8c188..8f22e37 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -54,11 +54,11 @@ #end proc # -#proc renderVideo(tweet: Tweet): string = +#proc renderVideo(video: Video): string =
- +

Video playback not supported

@@ -67,14 +67,12 @@
#end proc # -#proc renderGif(tweet: Tweet): string = -#let thumbUrl = getGifThumb(tweet).getSigUrl("pic") -#let videoUrl = getGifSrc(tweet).getSigUrl("video") -
+#proc renderGif(gif: Gif): string = +
-
@@ -103,10 +101,10 @@
#if tweet.photos.len > 0: ${renderMediaGroup(tweet)} - #elif tweet.videoThumb.isSome: - ${renderVideo(tweet)} + #elif tweet.video.isSome: + ${renderVideo(tweet.video.get())} #elif tweet.gif.isSome: - ${renderGif(tweet)} + ${renderGif(tweet.gif.get())} #end if ${renderStats(tweet)}