From 81d6d1ac7fc51a7737de6ca941eed1f691980d77 Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 1 Jun 2020 02:25:39 +0200 Subject: [PATCH] Misc. changes --- src/formatters.nim | 31 ++++++-- src/nitter.nim | 8 +-- src/prefs_impl.nim | 7 +- src/query.nim | 2 +- src/types.nim | 172 +++++++++++++++++++++++---------------------- src/utils.nim | 1 + 6 files changed, 125 insertions(+), 96 deletions(-) diff --git a/src/formatters.nim b/src/formatters.nim index 8701fee..395cf31 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -28,6 +28,7 @@ proc stripHtml*(text: string): string = for el in html.findAll("a"): let link = el.attr("href") if "http" in link: + if el.len == 0: continue el[0].text = link html.innerText() @@ -94,9 +95,32 @@ proc getRfc822Time*(tweet: Tweet): string = proc getTweetTime*(tweet: Tweet): string = tweet.time.format("h:mm tt' ยท 'MMM d', 'YYYY") -proc getLink*(tweet: Tweet | Quote; focus=true): string = +proc getShortTime*(tweet: Tweet): string = + let + now = now().utc + then = tweet.time.utc + since = now - then + + if now.year != then.year: + result = tweet.time.format("d MMM yyyy") + elif since.inDays >= 1: + result = tweet.time.format("MMM d") + elif since.inHours >= 1: + result = $since.inHours & "h" + elif since.inMinutes >= 1: + result = $since.inMinutes & "m" + elif since.inSeconds > 1: + result = $since.inSeconds & "s" + else: + # this shouldn't happen, but just in case + result = "now" + +proc getLink*(tweet: Tweet; focus=true): string = if tweet.id == 0: return - result = &"/{tweet.profile.username}/status/{tweet.id}" + var username = tweet.profile.username + if username.len == 0: + username = "i" + result = &"/{username}/status/{tweet.id}" if focus: result &= "#m" proc getTombstone*(text: string): string = @@ -114,8 +138,7 @@ proc getTwitterLink*(path: string; params: Table[string, string]): string = let p = { "f": $query.kind, "q": genQueryParam(query), - "src": "typd", - "max_position": params.getOrDefault("max_position", "0") + "src": "typed_query" } result = $(parseUri("https://twitter.com") / path ? p) diff --git a/src/nitter.nim b/src/nitter.nim index 457156a..2e4a9b2 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -3,7 +3,7 @@ from net import Port import jester -import types, config, prefs, formatters, cache +import types, config, prefs, formatters, redis_cache, tokens import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, @@ -13,8 +13,10 @@ const configPath {.strdefine.} = "./nitter.conf" let (cfg, fullCfg) = getConfig(configPath) updateDefaultPrefs(fullCfg) - +setCacheTimes(cfg) setHmacKey(cfg.hmacKey) +initRedisPool(cfg) +asyncCheck initTokenPool(cfg) createUnsupportedRouter(cfg) createResolverRouter(cfg) @@ -27,8 +29,6 @@ createMediaRouter(cfg) createEmbedRouter(cfg) createRssRouter(cfg) -asyncCheck cacheCleaner() - settings: port = Port(cfg.port) staticDir = cfg.staticDir diff --git a/src/prefs_impl.nim b/src/prefs_impl.nim index d27a829..641b6cd 100644 --- a/src/prefs_impl.nim +++ b/src/prefs_impl.nim @@ -8,11 +8,14 @@ type name*: string label*: string kind*: PrefKind - options*: seq[string] - placeholder*: string + # checkbox defaultState*: bool + # select defaultOption*: string + options*: seq[string] + # input defaultInput*: string + placeholder*: string PrefList* = OrderedTable[string, seq[Pref]] diff --git a/src/query.nim b/src/query.nim index 5ab0391..d4f9578 100644 --- a/src/query.nim +++ b/src/query.nim @@ -3,7 +3,7 @@ import strutils, strformat, sequtils, tables, uri import types const - separators = @["AND", "OR"] + # separators = @["AND", "OR"] validFilters* = @[ "media", "images", "twimg", "videos", "native_video", "consumer_video", "pro_video", diff --git a/src/types.nim b/src/types.nim index 4ea8bf1..4739ce1 100644 --- a/src/types.nim +++ b/src/types.nim @@ -1,68 +1,72 @@ -import times, sequtils, options -import norm/sqlite - +import times, sequtils, options, tables import prefs_impl genPrefsType() type + Token* = ref object + tok*: string + limit*: int + remaining*: int + reset*: Time + init*: Time + # agent*: string + + Error* = enum + protectedUser = 22 + couldntAuth = 32 + doesntExist = 34 + notFound = 50 + suspended = 63 + invalidToken = 89 + listIdOrSlug = 112 + forbidden = 200 + noCsrf = 353 + + Profile* = object + id*: string + username*: string + fullname*: string + lowername*: string + location*: string + website*: string + bio*: string + userpic*: string + banner*: string + following*: string + followers*: string + tweets*: string + likes*: string + media*: string + verified*: bool + protected*: bool + suspended*: bool + joinDate*: Time + VideoType* = enum - vmap, m3u8, mp4 + m3u8 = "application/x-mpegURL" + mp4 = "video/mp4" + vmap = "video/vmap" -dbTypes: - type - Profile* = object - username*: string - fullname*: string - lowername*: string - location*: string - website*: string - bio*: string - userpic*: string - banner*: string - following*: string - followers*: string - tweets*: string - likes*: string - media*: string - verified*: bool - protected*: bool - suspended*: bool - joinDate* {. - dbType: "INTEGER" - parseIt: it.i.fromUnix() - formatIt: dbValue(it.toUnix()) - .}: Time - updated* {. - dbType: "INTEGER" - parseIt: it.i.fromUnix() - formatIt: dbValue(getTime().toUnix()) - .}: Time + VideoVariant* = object + videoType*: VideoType + url*: string + bitrate*: int - Video* = object - videoId*: string - contentId*: string - durationMs*: int - url*: string - thumb*: string - views*: string - available*: bool - reason*: string - title*: string - description*: string - playbackType* {. - dbType: "STRING" - parseIt: parseEnum[VideoType](it.s) - formatIt: dbValue($it) - .}: VideoType - updated* {. - dbType: "INTEGER" - parseIt: it.i.fromUnix() - formatIt: dbValue(getTime().toUnix()) - .}: Time + Video* = object + videoId*: string + contentId*: string + durationMs*: int + url*: string + thumb*: string + views*: string + available*: bool + reason*: string + title*: string + description*: string + playbackType*: VideoType + variants*: seq[VideoVariant] - -type QueryKind* = enum posts, replies, media, users, tweets, userList @@ -92,18 +96,20 @@ type Poll* = object options*: seq[string] values*: seq[int] - votes*: string - status*: string + votes*: int leader*: int + status*: string CardKind* = enum + player = "player" summary = "summary" summaryLarge = "summary_large_image" promoWebsite = "promo_website" promoVideo = "promo_video_website" promoVideoConvo = "promo_video_convo" - player = "player" liveEvent = "live_event" + broadcast = "broadcast" + periscope = "periscope_broadcast" Card* = object kind*: CardKind @@ -113,25 +119,9 @@ type title*: string dest*: string text*: string - image*: Option[string] + image*: string video*: Option[Video] - Quote* = object - id*: int64 - profile*: Profile - text*: string - reply*: seq[string] - hasThread*: bool - sensitive*: bool - available*: bool - tombstone*: string - thumb*: string - badge*: string - - Retweet* = object - by*: string - id*: int64 - TweetStats* = object replies*: int retweets*: int @@ -140,10 +130,10 @@ type Tweet* = ref object id*: int64 threadId*: int64 + replyId*: int64 profile*: Profile text*: string time*: Time - shortTime*: string reply*: seq[string] pinned*: bool hasThread*: bool @@ -151,27 +141,26 @@ type tombstone*: string location*: string stats*: TweetStats - retweet*: Option[Retweet] + retweet*: Option[Tweet] attribution*: Option[Profile] mediaTags*: seq[Profile] - quote*: Option[Quote] + quote*: Option[Tweet] card*: Option[Card] + poll*: Option[Poll] gif*: Option[Gif] video*: Option[Video] photos*: seq[string] - poll*: Option[Poll] - Result*[T] = ref object + Result*[T] = object content*: seq[T] - minId*: string - maxId*: string - hasMore*: bool + top*, bottom*: string beginning*: bool query*: Query - Chain* = ref object + Chain* = object content*: seq[Tweet] more*: int64 + cursor*: string Conversation* = ref object tweet*: Tweet @@ -181,6 +170,19 @@ type Timeline* = Result[Tweet] + List* = object + id*: string + name*: string + userId*: string + username*: string + description*: string + members*: int + banner*: string + + GlobalObjects* = ref object + tweets*: Table[string, Tweet] + users*: Table[string, Profile] + Config* = ref object address*: string port*: int diff --git a/src/utils.nim b/src/utils.nim index cfa779e..ff7d901 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -8,6 +8,7 @@ const badPngExts = @["pngn", "png:", "png_", "_png"] twitterDomains = @[ "twitter.com", + "pic.twitter.com", "twimg.com", "abs.twimg.com", "pbs.twimg.com",