From 47c87f706885ab062f5da4f007d1bc4a98556c5d Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 7 Jun 2020 07:55:57 +0200 Subject: [PATCH] Strip https://pbs.twimg.com from image urls --- src/parser.nim | 14 +++++++------- src/parserutils.nim | 13 +++++++++---- src/routes/media.nim | 13 ++++++++----- src/utils.nim | 5 +++++ tests/test_card.py | 4 ++-- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/parser.nim b/src/parser.nim index 014e683..27067eb 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -10,7 +10,7 @@ proc parseProfile(js: JsonNode; id=""): Profile = fullname: js{"name"}.getStr, location: js{"location"}.getStr, bio: js{"description"}.getStr, - userpic: js{"profile_image_url_https"}.getStr.replace("_normal", ""), + userpic: js{"profile_image_url_https"}.getImageStr.replace("_normal", ""), banner: js.getBanner, following: $js{"friends_count"}.getInt, followers: $js{"followers_count"}.getInt, @@ -62,7 +62,7 @@ proc parseGraphList*(js: JsonNode): List = userId: list{"user", "legacy", "id_str"}.getStr, description: list{"description"}.getStr, members: list{"member_count"}.getInt, - banner: list{"custom_banner_media", "media_info", "url"}.getStr + banner: list{"custom_banner_media", "media_info", "url"}.getImageStr ) proc parseListMembers*(js: JsonNode; cursor: string): Result[Profile] = @@ -100,15 +100,15 @@ proc parsePoll(js: JsonNode): Poll = result.votes = result.values.sum proc parseGif(js: JsonNode): Gif = - Gif( - url: js{"video_info", "variants"}[0]{"url"}.getStr, - thumb: js{"media_url_https"}.getStr + result = Gif( + url: js{"video_info", "variants"}[0]{"url"}.getImageStr, + thumb: js{"media_url_https"}.getImageStr ) proc parseVideo(js: JsonNode): Video = result = Video( videoId: js{"id_str"}.getStr, - thumb: js{"media_url_https"}.getStr, + thumb: js{"media_url_https"}.getImageStr, views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr, available: js{"ext_media_availability", "status"}.getStr == "available", title: js{"ext_alt_text"}.getStr, @@ -249,7 +249,7 @@ proc parseTweet(js: JsonNode): Tweet = for m in jsMedia: case m{"type"}.getStr of "photo": - result.photos.add m{"media_url_https"}.getStr + result.photos.add m{"media_url_https"}.getImageStr of "video": result.video = some(parseVideo(m)) of "animated_gif": diff --git a/src/parserutils.nim b/src/parserutils.nim index 62813d2..1bb17c9 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -1,4 +1,4 @@ -import strutils, times, macros, htmlgen, uri, unicode, options +import strutils, times, macros, htmlgen, unicode, options import regex, packedjson import types, utils, formatters @@ -58,8 +58,13 @@ proc getId*(js: JsonNode): int64 {.inline.} = template getStrVal*(js: JsonNode; default=""): string = js{"string_value"}.getStr(default) -template getImageVal*(js: JsonNode; default=""): string = - js{"image_value", "url"}.getStr(default) +proc getImageStr*(js: JsonNode): string = + result = js.getStr + result.removePrefix(https) + result.removePrefix(twimg) + +template getImageVal*(js: JsonNode): string = + js{"image_value", "url"}.getImageStr proc getCardUrl*(js: JsonNode; kind: CardKind): string = result = js{"website_url"}.getStrVal @@ -81,7 +86,7 @@ proc getCardTitle*(js: JsonNode; kind: CardKind): string = result = js{"event_category"}.getStrVal proc getBanner*(js: JsonNode): string = - let url = js{"profile_banner_url"}.getStr + let url = js{"profile_banner_url"}.getImageStr if url.len > 0: return url & "/1500x500" diff --git a/src/routes/media.nim b/src/routes/media.nim index c86a786..18f863c 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -4,7 +4,7 @@ import asynchttpserver, asyncstreams, asyncfile, asyncnet import jester, regex import router_utils -import ".."/[types, formatters, agents] +import ".."/[types, formatters, agents, utils] import ../views/general export asynchttpserver, asyncstreams, asyncfile, asyncnet @@ -66,11 +66,14 @@ proc createMediaRouter*(cfg: Config) = resp Http404 get "/pic/@url": - cond "http" in @"url" - cond "twimg" in @"url" + var url = decodeUrl(@"url") + if "twimg.com" notin url: + url.insert(twimg) + if not url.startsWith(https): + url.insert(https) - let uri = parseUri(decodeUrl(@"url")) - cond isTwitterUrl($uri) == true + let uri = parseUri(url) + cond isTwitterUrl(uri) == true enableRawMode() let code = await proxyMedia(request, $uri) diff --git a/src/utils.nim b/src/utils.nim index 00b9a4c..6dfd6a2 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -4,6 +4,8 @@ import nimcrypto, regex var hmacKey = "secretkey" const + https* = "https://" + twimg* = "pbs.twimg.com/" badJpgExts = @["1500x500", "jpgn", "jpg:", "jpg_", "_jpg"] badPngExts = @["pngn", "png:", "png_", "_png"] twitterDomains = @[ @@ -43,5 +45,8 @@ proc filterParams*(params: Table): seq[(string, string)] = let filter = ["name", "id", "list", "referer", "scroll"] toSeq(params.pairs()).filterIt(it[0] notin filter and it[1].len > 0) +proc isTwitterUrl*(uri: Uri): bool = + uri.hostname in twitterDomains + proc isTwitterUrl*(url: string): bool = parseUri(url).hostname in twitterDomains diff --git a/tests/test_card.py b/tests/test_card.py index e194fc3..39c0c08 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -107,7 +107,7 @@ class CardTest(BaseTestCase): card = Card(Conversation.main + " ") self.assert_text(title, card.title) self.assert_text(destination, card.destination) - self.assertIn('twimg', self.get_image_url(card.image + ' img')) + self.assertIn('_img', self.get_image_url(card.image + ' img')) if len(description) > 0: self.assert_text(description, card.description) if large: @@ -130,7 +130,7 @@ class CardTest(BaseTestCase): card = Card(Conversation.main + " ") self.assert_text(title, card.title) self.assert_text(destination, card.destination) - self.assertIn('twimg', self.get_image_url(card.image + ' img')) + self.assertIn('_img', self.get_image_url(card.image + ' img')) self.assert_element_visible('.card-overlay') if len(description) > 0: self.assert_text(description, card.description)