Misc. changes

src/formatters.nim

@@ -28,6 +28,7 @@ proc stripHtml*(text: string): string =
   for el in html.findAll("a"):
     let link = el.attr("href")
     if "http" in link:
+      if el.len == 0: continue
       el[0].text = link
   html.innerText()
 
@@ -94,9 +95,32 @@ proc getRfc822Time*(tweet: Tweet): string =
 proc getTweetTime*(tweet: Tweet): string =
   tweet.time.format("h:mm tt' · 'MMM d', 'YYYY")
 
-proc getLink*(tweet: Tweet | Quote; focus=true): string =
+proc getShortTime*(tweet: Tweet): string =
+  let
+    now = now().utc
+    then = tweet.time.utc
+    since = now - then
+
+  if now.year != then.year:
+    result = tweet.time.format("d MMM yyyy")
+  elif since.inDays >= 1:
+    result = tweet.time.format("MMM d")
+  elif since.inHours >= 1:
+    result = $since.inHours & "h"
+  elif since.inMinutes >= 1:
+    result = $since.inMinutes & "m"
+  elif since.inSeconds > 1:
+    result = $since.inSeconds & "s"
+  else:
+    # this shouldn't happen, but just in case
+    result = "now"
+
+proc getLink*(tweet: Tweet; focus=true): string =
   if tweet.id == 0: return
-  result = &"/{tweet.profile.username}/status/{tweet.id}"
+  var username = tweet.profile.username
+  if username.len == 0:
+    username = "i"
+  result = &"/{username}/status/{tweet.id}"
   if focus: result &= "#m"
 
 proc getTombstone*(text: string): string =
@@ -114,8 +138,7 @@ proc getTwitterLink*(path: string; params: Table[string, string]): string =
   let p = {
     "f": $query.kind,
     "q": genQueryParam(query),
-    "src": "typd",
+    "src": "typed_query"
-    "max_position": params.getOrDefault("max_position", "0")
   }
 
   result = $(parseUri("https://twitter.com") / path ? p)

src/nitter.nim

@@ -3,7 +3,7 @@ from net import Port
 
 import jester
 
-import types, config, prefs, formatters, cache
+import types, config, prefs, formatters, redis_cache, tokens
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list,
@@ -13,8 +13,10 @@ const configPath {.strdefine.} = "./nitter.conf"
 let (cfg, fullCfg) = getConfig(configPath)
 
 updateDefaultPrefs(fullCfg)
-
+setCacheTimes(cfg)
 setHmacKey(cfg.hmacKey)
+initRedisPool(cfg)
+asyncCheck initTokenPool(cfg)
 
 createUnsupportedRouter(cfg)
 createResolverRouter(cfg)
@@ -27,8 +29,6 @@ createMediaRouter(cfg)
 createEmbedRouter(cfg)
 createRssRouter(cfg)
 
-asyncCheck cacheCleaner()
-
 settings:
   port = Port(cfg.port)
   staticDir = cfg.staticDir

src/prefs_impl.nim

@@ -8,11 +8,14 @@ type
     name*: string
     label*: string
     kind*: PrefKind
-    options*: seq[string]
+    # checkbox
-    placeholder*: string
     defaultState*: bool
+    # select
     defaultOption*: string
+    options*: seq[string]
+    # input
     defaultInput*: string
+    placeholder*: string
 
   PrefList* = OrderedTable[string, seq[Pref]]
 

src/query.nim

@@ -3,7 +3,7 @@ import strutils, strformat, sequtils, tables, uri
 import types
 
 const
-  separators = @["AND", "OR"]
+  # separators = @["AND", "OR"]
   validFilters* = @[
     "media", "images", "twimg", "videos",
     "native_video", "consumer_video", "pro_video",

src/types.nim

@@ -1,68 +1,72 @@
-import times, sequtils, options
+import times, sequtils, options, tables
-import norm/sqlite
-
 import prefs_impl
 
 genPrefsType()
 
 type
+  Token* = ref object
+    tok*: string
+    limit*: int
+    remaining*: int
+    reset*: Time
+    init*: Time
+    # agent*: string
+
+  Error* = enum
+    protectedUser = 22
+    couldntAuth = 32
+    doesntExist = 34
+    notFound = 50
+    suspended = 63
+    invalidToken = 89
+    listIdOrSlug = 112
+    forbidden = 200
+    noCsrf = 353
+
+  Profile* = object
+    id*: string
+    username*: string
+    fullname*: string
+    lowername*: string
+    location*: string
+    website*: string
+    bio*: string
+    userpic*: string
+    banner*: string
+    following*: string
+    followers*: string
+    tweets*: string
+    likes*: string
+    media*: string
+    verified*: bool
+    protected*: bool
+    suspended*: bool
+    joinDate*: Time
+
   VideoType* = enum
-    vmap, m3u8, mp4
+    m3u8 = "application/x-mpegURL"
+    mp4 = "video/mp4"
+    vmap = "video/vmap"
 
-dbTypes:
+  VideoVariant* = object
-  type
+    videoType*: VideoType
-    Profile* = object
+    url*: string
-      username*: string
+    bitrate*: int
-      fullname*: string
-      lowername*: string
-      location*: string
-      website*: string
-      bio*: string
-      userpic*: string
-      banner*: string
-      following*: string
-      followers*: string
-      tweets*: string
-      likes*: string
-      media*: string
-      verified*: bool
-      protected*: bool
-      suspended*: bool
-      joinDate* {.
-          dbType: "INTEGER"
-          parseIt: it.i.fromUnix()
-          formatIt: dbValue(it.toUnix())
-        .}: Time
-      updated* {.
-          dbType: "INTEGER"
-          parseIt: it.i.fromUnix()
-          formatIt: dbValue(getTime().toUnix())
-        .}: Time
 
-    Video* = object
+  Video* = object
-      videoId*: string
+    videoId*: string
-      contentId*: string
+    contentId*: string
-      durationMs*: int
+    durationMs*: int
-      url*: string
+    url*: string
-      thumb*: string
+    thumb*: string
-      views*: string
+    views*: string
-      available*: bool
+    available*: bool
-      reason*: string
+    reason*: string
-      title*: string
+    title*: string
-      description*: string
+    description*: string
-      playbackType* {.
+    playbackType*: VideoType
-          dbType: "STRING"
+    variants*: seq[VideoVariant]
-          parseIt: parseEnum[VideoType](it.s)
-          formatIt: dbValue($it)
-        .}: VideoType
-      updated* {.
-          dbType: "INTEGER"
-          parseIt: it.i.fromUnix()
-          formatIt: dbValue(getTime().toUnix())
-        .}: Time
 
-
-type
   QueryKind* = enum
     posts, replies, media, users, tweets, userList
 
@@ -92,18 +96,20 @@ type
   Poll* = object
     options*: seq[string]
     values*: seq[int]
-    votes*: string
+    votes*: int
-    status*: string
     leader*: int
+    status*: string
 
   CardKind* = enum
+    player = "player"
     summary = "summary"
     summaryLarge = "summary_large_image"
     promoWebsite = "promo_website"
     promoVideo = "promo_video_website"
     promoVideoConvo = "promo_video_convo"
-    player = "player"
     liveEvent = "live_event"
+    broadcast = "broadcast"
+    periscope = "periscope_broadcast"
 
   Card* = object
     kind*: CardKind
@@ -113,25 +119,9 @@ type
     title*: string
     dest*: string
     text*: string
-    image*: Option[string]
+    image*: string
     video*: Option[Video]
 
-  Quote* = object
-    id*: int64
-    profile*: Profile
-    text*: string
-    reply*: seq[string]
-    hasThread*: bool
-    sensitive*: bool
-    available*: bool
-    tombstone*: string
-    thumb*: string
-    badge*: string
-
-  Retweet* = object
-    by*: string
-    id*: int64
-
   TweetStats* = object
     replies*: int
     retweets*: int
@@ -140,10 +130,10 @@ type
   Tweet* = ref object
     id*: int64
     threadId*: int64
+    replyId*: int64
     profile*: Profile
     text*: string
     time*: Time
-    shortTime*: string
     reply*: seq[string]
     pinned*: bool
     hasThread*: bool
@@ -151,27 +141,26 @@ type
     tombstone*: string
     location*: string
     stats*: TweetStats
-    retweet*: Option[Retweet]
+    retweet*: Option[Tweet]
     attribution*: Option[Profile]
     mediaTags*: seq[Profile]
-    quote*: Option[Quote]
+    quote*: Option[Tweet]
     card*: Option[Card]
+    poll*: Option[Poll]
     gif*: Option[Gif]
     video*: Option[Video]
     photos*: seq[string]
-    poll*: Option[Poll]
 
-  Result*[T] = ref object
+  Result*[T] = object
     content*: seq[T]
-    minId*: string
+    top*, bottom*: string
-    maxId*: string
-    hasMore*: bool
     beginning*: bool
     query*: Query
 
-  Chain* = ref object
+  Chain* = object
     content*: seq[Tweet]
     more*: int64
+    cursor*: string
 
   Conversation* = ref object
     tweet*: Tweet
@@ -181,6 +170,19 @@ type
 
   Timeline* = Result[Tweet]
 
+  List* = object
+    id*: string
+    name*: string
+    userId*: string
+    username*: string
+    description*: string
+    members*: int
+    banner*: string
+
+  GlobalObjects* = ref object
+    tweets*: Table[string, Tweet]
+    users*: Table[string, Profile]
+
   Config* = ref object
     address*: string
     port*: int

src/utils.nim

@@ -8,6 +8,7 @@ const
   badPngExts = @["pngn", "png:", "png_", "_png"]
   twitterDomains = @[
     "twitter.com",
+    "pic.twitter.com",
     "twimg.com",
     "abs.twimg.com",
     "pbs.twimg.com",