Merge branch 'master' into thread-rss

.github/workflows/build-docker.yml

@@ -33,6 +33,6 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           push: true
           tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}

.gitignore

@@ -5,5 +5,7 @@ nitter
 /tests/geckodriver.log
 /tests/downloaded_files/*
 /tools/gencss
+/tools/rendermd
 /public/css/style.css
+/public/md/*.html
 nitter.conf

Dockerfile

@@ -2,14 +2,14 @@ FROM nimlang/nim:1.6.2-alpine-regular as nim
 LABEL maintainer="setenforce@protonmail.com"
 EXPOSE 8080
 
-RUN apk --no-cache add libsass-dev
+RUN apk --no-cache add libsass-dev pcre
 
 COPY . /src/nitter
 WORKDIR /src/nitter
 
-RUN nimble build -y -d:release -d:danger --passC:"-flto" --passL:"-flto" \
-    && strip -s nitter \
-    && nimble scss
+RUN nimble build -y -d:danger -d:lto -d:strip \
+    && nimble scss \
+    && nimble md
 
 FROM alpine:latest
 WORKDIR /src/

README.md

@@ -84,7 +84,7 @@ Running it with the default config is fine, Nitter's default config is set to
 use the default Redis port and localhost.
 
 Here's how to create a `nitter` user, clone the repo, and build the project
-along with the scss.
+along with the scss and md files.
 
 ```bash
 # useradd -m nitter
@@ -93,6 +93,7 @@ $ git clone https://github.com/zedeus/nitter
 $ cd nitter
 $ nimble build -d:release
 $ nimble scss
+$ nimble md
 $ cp nitter.example.conf nitter.conf
 ```
 
@@ -125,7 +126,7 @@ docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitte
 ```
 
 Using docker-compose to run both Nitter and Redis as different containers:
-Change `redisHost` from `localhost` to `redis` in `nitter.conf`, then run:
+Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run:
 ```bash
 docker-compose up -d
 ```

config.nims

@@ -7,6 +7,7 @@
 # disable annoying warnings
 warning("GcUnsafe2", off)
 hint("XDeclaredButNotUsed", off)
+hint("XCannotRaiseY", off)
 hint("User", off)
 
 const

docker-compose.yml

@@ -1,18 +1,25 @@
-version: "3.8"
+version: "3"
+
 services:
-  redis:
-    image: redis:6-alpine
-    restart: unless-stopped
-    volumes:
-      - redis-data:/var/lib/redis
+
   nitter:
     image: zedeus/nitter:latest
-    restart: unless-stopped
-    depends_on:
-      - redis
+    container_name: nitter
     ports:
-      - "8080:8080"
+      - "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
     volumes:
-      - ./nitter.conf:/src/nitter.conf
+      - ./nitter.conf:/src/nitter.conf:ro
+    depends_on:
+      - nitter-redis
+    restart: unless-stopped
+
+  nitter-redis:
+    image: redis:6-alpine
+    container_name: nitter-redis
+    command: redis-server --save 60 1 --loglevel warning
+    volumes:
+      - nitter-redis:/data
+    restart: unless-stopped
+
 volumes:
-  redis-data:
+  nitter-redis:

nitter.example.conf

@@ -10,7 +10,7 @@ hostname = "nitter.net"
 [Cache]
 listMinutes = 240  # how long to cache list info (not the tweets, so keep it high)
 rssMinutes = 10  # how long to cache rss queries
-redisHost = "localhost"  # Change to "redis" if using docker-compose
+redisHost = "localhost"  # Change to "nitter-redis" if using docker-compose
 redisPort = 6379
 redisPassword = ""
 redisConnections = 20  # connection pool size

nitter.nimble

@@ -14,18 +14,21 @@ requires "nim >= 1.4.8"
 requires "jester >= 0.5.0"
 requires "karax#c71bc92"
 requires "sass#e683aa1"
-requires "regex#eeefb4f"
 requires "nimcrypto#a5742a9"
 requires "markdown#abdbe5e"
 requires "packedjson#d11d167"
 requires "supersnappy#2.1.1"
-requires "redpool#f880f49"
+requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "zippy#0.7.3"
 requires "flatty#0.2.3"
+requires "jsony#d0e69bd"
 
 
 # Tasks
 
 task scss, "Generate css":
-  exec "nim c --hint[Processing]:off -d:danger -r tools/gencss"
+  exec "nimble c --hint[Processing]:off -d:danger -r tools/gencss"
+
+task md, "Render md":
+  exec "nimble c --hint[Processing]:off -d:danger -r tools/rendermd"

public/css/themes/auto_(twitter).css

@@ -0,0 +1,2 @@
+@import "twitter_dark.css" (prefers-color-scheme: dark);
+@import "twitter.css" (prefers-color-scheme: light);

public/md/feature.md

@@ -1,7 +0,0 @@
-# Unsupported feature
-
-Nitter doesn't support this feature yet, but it might in the future.
-You can check for an issue and open one if needed here:
-<https://github.com/zedeus/nitter/issues>
-
-To find out more about the Nitter project, see the [About page](/about).

src/agents.nim

@@ -1,70 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-import random, strformat, strutils, sequtils
-
-randomize()
-
-const rvs = [
-  "11.0", "40.0", "42.0", "43.0", "47.0", "50.0", "52.0", "53.0", "54.0",
-  "61.0", "66.0", "67.0", "69.0", "70.0"
-]
-
-proc rv(): string =
-  if rand(10) < 1: ""
-  else: "; rv:" & sample(rvs)
-
-# OS
-
-const enc = ["; U", "; N", "; I", ""]
-
-proc linux(): string =
-  const
-    window = ["X11", "Wayland", "Unknown"]
-    arch = ["i686", "x86_64", "arm"]
-    distro = ["", "; Ubuntu/14.10", "; Ubuntu/16.10", "; Ubuntu/19.10",
-              "; Ubuntu", "; Fedora"]
-  sample(window) & sample(enc) & "; Linux " & sample(arch) & sample(distro)
-
-proc windows(): string =
-  const
-    nt = ["5.1", "5.2", "6.0", "6.1", "6.2", "6.3", "6.4", "9.0", "10.0"]
-    arch = ["; WOW64", "; Win64; x64", "; ARM", ""]
-    trident = ["", "; Trident/5.0", "; Trident/6.0", "; Trident/7.0"]
-  "Windows " & sample(nt) & sample(enc) & sample(arch) & sample(trident)
-
-const macs = toSeq(6..15).mapIt($it) & @["14_4", "10_1", "9_3"]
-
-proc mac(): string =
-  "Macintosh; Intel Mac OS X 10_" & sample(macs) & sample(enc)
-
-# Browser
-
-proc presto(): string =
-  const p = ["2.12.388", "2.12.407", "22.9.168", "2.9.201", "2.8.131", "2.7.62",
-             "2.6.30", "2.5.24"]
-  const v = ["10.0", "11.0", "11.1", "11.5", "11.6", "12.00", "12.14", "12.16"]
-  &"Presto/{sample(p)} Version/{sample(v)}"
-
-# Samples
-
-proc product(): string =
-  const opera = ["Opera/9.80", "Opera/12.0"]
-  if rand(20) < 1: "Mozilla/5.0"
-  else: sample(opera)
-
-proc os(): string =
-  let r = rand(10)
-  let os =
-    if r < 6: windows()
-    elif r < 9: linux()
-    else: mac()
-  &"({os}{rv()})"
-
-proc browser(prod: string): string =
-  if "Opera" in prod: presto()
-  else: "like Gecko"
-
-# Agent
-
-proc getAgent*(): string =
-  let prod = product()
-  &"{prod} {os()} {browser(prod)}"

src/api.nim

@@ -1,7 +1,15 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, uri, strutils
+import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
 import packedjson
 import types, query, formatters, consts, apiutils, parser
+import experimental/parser/user
+
+proc getGraphUser*(id: string): Future[User] {.async.} =
+  if id.len == 0 or id.any(c => not c.isDigit): return
+  let
+    variables = %*{"userId": id, "withSuperFollowsUserFields": true}
+    js = await fetch(graphUser ? {"variables": $variables}, Api.userRestId)
+  result = parseGraphUser(js, id)
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
@@ -15,6 +23,22 @@ proc getGraphList*(id: string): Future[List] {.async.} =
     url = graphList ? {"variables": $variables}
   result = parseGraphList(await fetch(url, Api.list))
 
+proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
+  if list.id.len == 0: return
+  let
+    variables = %*{
+      "listId": list.id,
+      "cursor": after,
+      "withSuperFollowsUserFields": false,
+      "withBirdwatchPivots": false,
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false,
+      "withSuperFollowsTweetFields": false
+    }
+    url = graphListMembers ? {"variables": $variables}
+  result = parseGraphListMembers(await fetch(url, Api.listMembers), after)
+
 proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
@@ -22,44 +46,42 @@ proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
     url = listTimeline ? ps
   result = parseTimeline(await fetch(url, Api.timeline), after)
 
-proc getListMembers*(list: List; after=""): Future[Result[Profile]] {.async.} =
-  if list.id.len == 0: return
-  let
-    ps = genParams({"list_id": list.id}, after)
-    url = listMembers ? ps
-  result = parseListMembers(await fetch(url, Api.listMembers), after)
-
-proc getProfile*(username: string): Future[Profile] {.async.} =
+proc getUser*(username: string): Future[User] {.async.} =
+  if username.len == 0: return
   let
     ps = genParams({"screen_name": username})
-    js = await fetch(userShow ? ps, Api.userShow)
-  result = parseUserShow(js, username=username)
+    json = await fetchRaw(userShow ? ps, Api.userShow)
+  result = parseUser(json, username)
 
-proc getProfileById*(userId: string): Future[Profile] {.async.} =
+proc getUserById*(userId: string): Future[User] {.async.} =
+  if userId.len == 0: return
   let
     ps = genParams({"user_id": userId})
-    js = await fetch(userShow ? ps, Api.userShow)
-  result = parseUserShow(js, id=userId)
+    json = await fetchRaw(userShow ? ps, Api.userShow)
+  result = parseUser(json)
 
 proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
+  if id.len == 0: return
   let
     ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
     url = timeline / (id & ".json") ? ps
   result = parseTimeline(await fetch(url, Api.timeline), after)
 
 proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
   let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
   result = parseTimeline(await fetch(url, Api.timeline), after)
 
 proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
+  if name.len == 0: return
   let
     ps = genParams({"screen_name": name, "trim_user": "true"},
                    count="18", ext=false)
     url = photoRail ? ps
-  result = parsePhotoRail(await fetch(url, Api.photoRail))
+  result = parsePhotoRail(await fetch(url, Api.timeline))
 
 proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
-  when T is Profile:
+  when T is User:
     const
       searchMode = ("result_filter", "user")
       parse = parseUsers
@@ -92,6 +114,10 @@ proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
   if after.len > 0:
     result.replies = await getReplies(id, after)
 
+proc getStatus*(id: string): Future[Tweet] {.async.} =
+  let url = status / (id & ".json") ? genParams()
+  result = parseStatus(await fetch(url, Api.status))
+
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
   let client = newAsyncHttpClient(maxRedirects=0)
   try:

src/apiutils.nim

@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, times, strutils, uri
-import packedjson, zippy
+import httpclient, asyncdispatch, options, strutils, uri
+import jsony, packedjson, zippy
 import types, tokens, consts, parserutils, http_pool
+import experimental/types/common
 
 const
   rlRemaining = "x-rate-limit-remaining"
@@ -16,6 +17,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
     result &= p
   if ext:
     result &= ("ext", "mediaStats")
+    result &= ("include_ext_alt_text", "true")
+    result &= ("include_ext_media_availability", "true")
   if count.len > 0:
     result &= ("count", count)
   if cursor.len > 0:
@@ -40,7 +43,14 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
     "DNT": "1"
   })
 
-proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+template updateToken() =
+  if api != Api.search and resp.headers.hasKey(rlRemaining):
+    let
+      remaining = parseInt(resp.headers[rlRemaining])
+      reset = parseInt(resp.headers[rlReset])
+    token.setRateLimit(api, remaining, reset)
+
+template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
 
@@ -48,37 +58,25 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
   if token.tok.len == 0:
     raise rateLimitError()
 
-  let headers = genHeaders(token)
   try:
     var resp: AsyncResponse
-    var body = pool.use(headers):
+    pool.use(genHeaders(token)):
       resp = await c.get($url)
-      await resp.body
+      result = await resp.body
 
-    if body.len > 0:
+      if resp.status == $Http503:
+        badClient = true
+        raise newException(InternalError, result)
+
+    if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
-        body = uncompress(body, dfGzip)
+        result = uncompress(result, dfGzip)
       else:
-        echo "non-gzip body, url: ", url, ", body: ", body
+        echo "non-gzip body, url: ", url, ", body: ", result
 
-    if body.startsWith('{') or body.startsWith('['):
-      result = parseJson(body)
-    else:
-      echo resp.status, ": ", body
-      result = newJNull()
+    fetchBody
 
-    if api != Api.search and resp.headers.hasKey(rlRemaining):
-      let
-        remaining = parseInt(resp.headers[rlRemaining])
-        reset = parseInt(resp.headers[rlReset])
-      token.setRateLimit(api, remaining, reset)
-
-    if result.getError notin {invalidToken, forbidden, badToken}:
-      release(token, used=true)
-    else:
-      echo "fetch error: ", result.getError
-      release(token, invalid=true)
-      raise rateLimitError()
+    release(token, used=true)
 
     if resp.status == $Http400:
       raise newException(InternalError, $url)
@@ -89,3 +87,35 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
     if "length" notin e.msg and "descriptor" notin e.msg:
       release(token, invalid=true)
     raise rateLimitError()
+
+proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+  var body: string
+  fetchImpl body:
+    if body.startsWith('{') or body.startsWith('['):
+      result = parseJson(body)
+    else:
+      echo resp.status, ": ", body, " --- url: ", url
+      result = newJNull()
+
+    updateToken()
+
+    let error = result.getError
+    if error in {invalidToken, forbidden, badToken}:
+      echo "fetch error: ", result.getError
+      release(token, invalid=true)
+      raise rateLimitError()
+
+proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
+  fetchImpl result:
+    if not (result.startsWith('{') or result.startsWith('[')):
+      echo resp.status, ": ", result, " --- url: ", url
+      result.setLen(0)
+
+    updateToken()
+
+    if result.startsWith("{\"errors"):
+      let errors = result.fromJson(Errors)
+      if errors in {invalidToken, forbidden, badToken}:
+        echo "fetch error: ", errors
+        release(token, invalid=true)
+        raise rateLimitError()

src/consts.nim

@@ -2,14 +2,14 @@
 import uri, sequtils
 
 const
-  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
 
   api = parseUri("https://api.twitter.com")
   activate* = $(api / "1.1/guest/activate.json")
 
-  listMembers* = api / "1.1/lists/members.json"
   userShow* = api / "1.1/users/show.json"
   photoRail* = api / "1.1/statuses/media_timeline.json"
+  status* = api / "1.1/statuses/show"
   search* = api / "2/search/adaptive.json"
 
   timelineApi = api / "2/timeline"
@@ -19,8 +19,10 @@ const
   tweet* = timelineApi / "conversation"
 
   graphql = api / "graphql"
-  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
+  graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
   graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
+  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
+  graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
 
   timelineParams* = {
     "include_profile_interstitial_type": "0",
@@ -35,16 +37,13 @@ const
     "cards_platform": "Web-12",
     "include_cards": "1",
     "include_composer_source": "false",
-    "include_ext_alt_text": "true",
     "include_reply_count": "1",
     "tweet_mode": "extended",
     "include_entities": "true",
     "include_user_entities": "true",
     "include_ext_media_color": "false",
-    "include_ext_media_availability": "true",
     "send_error_codes": "true",
     "simple_quoted_tweet": "true",
-    "ext": "mediaStats",
     "include_quote_count": "true"
   }.toSeq
 

src/experimental/parser/slices.nim

@@ -0,0 +1,67 @@
+import std/[macros, htmlgen, unicode]
+import ../types/common
+import ".."/../[formatters, utils]
+
+type
+  ReplaceSliceKind = enum
+    rkRemove, rkUrl, rkHashtag, rkMention
+
+  ReplaceSlice* = object
+    slice: Slice[int]
+    kind: ReplaceSliceKind
+    url, display: string
+
+proc cmp*(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
+
+proc dedupSlices*(s: var seq[ReplaceSlice]) =
+  var
+    len = s.len
+    i = 0
+  while i < len:
+    var j = i + 1
+    while j < len:
+      if s[i].slice.a == s[j].slice.a:
+        s.del j
+        dec len
+      else:
+        inc j
+    inc i
+
+proc extractUrls*(result: var seq[ReplaceSlice]; url: Url;
+                  textLen: int; hideTwitter = false) =
+  let
+    link = url.expandedUrl
+    slice = url.indices[0] ..< url.indices[1]
+
+  if hideTwitter and slice.b.succ >= textLen and link.isTwitterUrl:
+    if slice.a < textLen:
+      result.add ReplaceSlice(kind: rkRemove, slice: slice)
+  else:
+    result.add ReplaceSlice(kind: rkUrl, url: link,
+                            display: link.shortLink, slice: slice)
+
+proc replacedWith*(runes: seq[Rune]; repls: openArray[ReplaceSlice];
+                   textSlice: Slice[int]): string =
+  template extractLowerBound(i: int; idx): int =
+    if i > 0: repls[idx].slice.b.succ else: textSlice.a
+
+  result = newStringOfCap(runes.len)
+
+  for i, rep in repls:
+    result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
+    case rep.kind
+    of rkHashtag:
+      let
+        name = $runes[rep.slice.a.succ .. rep.slice.b]
+        symbol = $runes[rep.slice.a]
+      result.add a(symbol & name, href = "/search?q=%23" & name)
+    of rkMention:
+      result.add a($runes[rep.slice], href = rep.url, title = rep.display)
+    of rkUrl:
+      result.add a(rep.display, href = rep.url)
+    of rkRemove:
+      discard
+
+  let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
+  if rest.a <= rest.b:
+    result.add $runes[rest]

src/experimental/parser/unifiedcard.nim

@@ -0,0 +1,91 @@
+import std/[options, tables, strutils, strformat, sugar]
+import jsony
+import ../types/unifiedcard
+from ../../types import Card, CardKind, Video
+from ../../utils import twimg, https
+
+proc getImageUrl(entity: MediaEntity): string =
+  entity.mediaUrlHttps.dup(removePrefix(twimg), removePrefix(https))
+
+proc parseDestination(id: string; card: UnifiedCard; result: var Card) =
+  let destination = card.destinationObjects[id].data
+  result.dest = destination.urlData.vanity
+  result.url = destination.urlData.url
+
+proc parseDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
+  data.destination.parseDestination(card, result)
+
+  result.text = data.title
+  if result.text.len == 0:
+    result.text = data.name
+
+proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
+  data.destination.parseDestination(card, result)
+
+  result.kind = summary
+  result.image = card.mediaEntities[data.mediaId].getImageUrl
+  result.text = data.topicDetail.title
+  result.dest = "Topic"
+
+proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
+  let app = card.appStoreData[data.appId][0]
+
+  case app.kind
+  of androidApp:
+    result.url = "http://play.google.com/store/apps/details?id=" & app.id
+  of iPhoneApp, iPadApp:
+    result.url = "https://itunes.apple.com/app/id" & app.id
+
+  result.text = app.title
+  result.dest = app.category
+
+proc parseListDetails(data: ComponentData; result: var Card) =
+  result.dest = &"List · {data.memberCount} Members"
+
+proc parseCommunityDetails(data: ComponentData; result: var Card) =
+  result.dest = &"Community · {data.memberCount} Members"
+
+proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
+  let mediaId =
+    if component.kind == swipeableMedia:
+      component.data.mediaList[0].id
+    else:
+      component.data.id
+
+  let rMedia = card.mediaEntities[mediaId]
+  case rMedia.kind:
+  of photo:
+    result.kind = summaryLarge
+    result.image = rMedia.getImageUrl
+  of video:
+    let videoInfo = rMedia.videoInfo.get
+    result.kind = promoVideo
+    result.video = some Video(
+      available: true,
+      thumb: rMedia.getImageUrl,
+      durationMs: videoInfo.durationMillis,
+      variants: videoInfo.variants
+    )
+
+proc parseUnifiedCard*(json: string): Card =
+  let card = json.fromJson(UnifiedCard)
+
+  for component in card.componentObjects.values:
+    case component.kind
+    of details, communityDetails, twitterListDetails:
+      component.data.parseDetails(card, result)
+    of appStoreDetails:
+      component.data.parseAppDetails(card, result)
+    of mediaWithDetailsHorizontal:
+      component.data.parseMediaDetails(card, result)
+    of media, swipeableMedia:
+      component.parseMedia(card, result)
+    of buttonGroup:
+      discard
+
+    case component.kind
+    of twitterListDetails:
+      component.data.parseListDetails(result)
+    of communityDetails:
+      component.data.parseCommunityDetails(result)
+    else: discard

src/experimental/parser/user.nim

@@ -0,0 +1,67 @@
+import std/[algorithm, unicode, re, strutils]
+import jsony
+import utils, slices
+import ../types/user as userType
+from ../../types import User, Error
+
+let
+  unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
+  unReplace = "$1<a href=\"/$2\">@$2</a>"
+
+  htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
+  htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
+
+proc expandUserEntities(user: var User; raw: RawUser) =
+  let
+    orig = user.bio.toRunes
+    ent = raw.entities
+
+  if ent.url.urls.len > 0:
+    user.website = ent.url.urls[0].expandedUrl
+
+  var replacements = newSeq[ReplaceSlice]()
+
+  for u in ent.description.urls:
+    replacements.extractUrls(u, orig.high)
+
+  replacements.dedupSlices
+  replacements.sort(cmp)
+
+  user.bio = orig.replacedWith(replacements, 0 .. orig.len)
+                 .replacef(unRegex, unReplace)
+                 .replacef(htRegex, htReplace)
+
+proc getBanner(user: RawUser): string =
+  if user.profileBannerUrl.len > 0:
+    return user.profileBannerUrl & "/1500x500"
+  if user.profileLinkColor.len > 0:
+    return '#' & user.profileLinkColor
+
+proc parseUser*(json: string; username=""): User =
+  handleErrors:
+    case error.code
+    of suspended: return User(username: username, suspended: true)
+    of userNotFound: return
+    else: echo "[error - parseUser]: ", error
+
+  let user = json.fromJson(RawUser)
+
+  result = User(
+    id: user.idStr,
+    username: user.screenName,
+    fullname: user.name,
+    location: user.location,
+    bio: user.description,
+    following: user.friendsCount,
+    followers: user.followersCount,
+    tweets: user.statusesCount,
+    likes: user.favouritesCount,
+    media: user.mediaCount,
+    verified: user.verified,
+    protected: user.protected,
+    joinDate: parseTwitterDate(user.createdAt),
+    banner: getBanner(user),
+    userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "")
+  )
+
+  result.expandUserEntities(user)

src/experimental/parser/utils.nim

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import std/[sugar, strutils, times]
+import ../types/common
+import ../../utils as uutils
+
+template parseTime(time: string; f: static string; flen: int): DateTime =
+  if time.len != flen: return
+  parse(time, f, utc())
+
+proc parseIsoDate*(date: string): DateTime =
+  date.parseTime("yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", 20)
+
+proc parseTwitterDate*(date: string): DateTime =
+  date.parseTime("ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
+
+proc getImageUrl*(url: string): string =
+  url.dup(removePrefix(twimg), removePrefix(https))
+
+template handleErrors*(body) =
+  if json.startsWith("{\"errors"):
+    for error {.inject.} in json.fromJson(Errors).errors:
+      body

src/experimental/types/common.nim

@@ -0,0 +1,20 @@
+from ../../types import Error
+
+type
+  Url* = object
+    url*: string
+    expandedUrl*: string
+    displayUrl*: string
+    indices*: array[2, int]
+
+  ErrorObj* = object
+    code*: Error
+    message*: string
+
+  Errors* = object
+    errors*: seq[ErrorObj]
+
+proc contains*(codes: set[Error]; errors: Errors): bool =
+  for e in errors.errors:
+    if e.code in codes:
+      return true

src/experimental/types/unifiedcard.nim

@@ -0,0 +1,79 @@
+import options, tables
+from ../../types import VideoType, VideoVariant
+
+type
+  UnifiedCard* = object
+    componentObjects*: Table[string, Component]
+    destinationObjects*: Table[string, Destination]
+    mediaEntities*: Table[string, MediaEntity]
+    appStoreData*: Table[string, seq[AppStoreData]]
+
+  ComponentType* = enum
+    details
+    media
+    swipeableMedia
+    buttonGroup
+    appStoreDetails
+    twitterListDetails
+    communityDetails
+    mediaWithDetailsHorizontal
+
+  Component* = object
+    kind*: ComponentType
+    data*: ComponentData
+
+  ComponentData* = object
+    id*: string
+    appId*: string
+    mediaId*: string
+    destination*: string
+    title*: Text
+    subtitle*: Text
+    name*: Text
+    memberCount*: int
+    mediaList*: seq[MediaItem]
+    topicDetail*: tuple[title: Text]
+
+  MediaItem* = object
+    id*: string
+    destination*: string
+
+  Destination* = object
+    kind*: string
+    data*: tuple[urlData: UrlData]
+
+  UrlData* = object
+    url*: string
+    vanity*: string
+
+  MediaType* = enum
+    photo, video
+
+  MediaEntity* = object
+    kind*: MediaType
+    mediaUrlHttps*: string
+    videoInfo*: Option[VideoInfo]
+
+  VideoInfo* = object
+    durationMillis*: int
+    variants*: seq[VideoVariant]
+
+  AppType* = enum
+    androidApp, iPhoneApp, iPadApp
+
+  AppStoreData* = object
+    kind*: AppType
+    id*: string
+    title*: Text
+    category*: Text
+
+  Text = object
+    content: string
+
+  HasTypeField = Component | Destination | MediaEntity | AppStoreData
+
+converter fromText*(text: Text): string = text.content
+
+proc renameHook*(v: var HasTypeField; fieldName: var string) =
+  if fieldName == "type":
+    fieldName = "kind"

src/experimental/types/user.nim

@@ -0,0 +1,28 @@
+import common
+
+type
+  RawUser* = object
+    idStr*: string
+    name*: string
+    screenName*: string
+    location*: string
+    description*: string
+    entities*: Entities
+    createdAt*: string
+    followersCount*: int
+    friendsCount*: int
+    favouritesCount*: int
+    statusesCount*: int
+    mediaCount*: int
+    verified*: bool
+    protected*: bool
+    profileBannerUrl*: string
+    profileImageUrlHttps*: string
+    profileLinkColor*: string
+
+  Entities* = object
+    url*: Urls
+    description*: Urls
+
+  Urls* = object
+    urls*: seq[Url]

src/formatters.nim

@@ -1,9 +1,17 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import strutils, strformat, times, uri, tables, xmltree, htmlparser, htmlgen
-import regex
+import std/[enumerate, re]
 import types, utils, query
 
 const
+  cards = "cards.twitter.com/cards"
+  tco = "https://t.co"
+  twitter = parseUri("https://twitter.com")
+
+let
+  twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
+  twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
+
   ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
   igRegex = re"(www\.)?instagram\.com"
 
@@ -14,108 +22,104 @@ const
   # Images aren't supported due to errors from Teddit when the image
   # wasn't first displayed via a post on the Teddit instance.
 
-  twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
-  twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
-
-  cards = "cards.twitter.com/cards"
-  tco = "https://t.co"
-
   wwwRegex = re"https?://(www[0-9]?\.)?"
   m3u8Regex = re"""url="(.+.m3u8)""""
-  manifestRegex = re"\/(.+(.ts|.m4s|.m3u8|.vmap|.mp4))"
   userPicRegex = re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$"
   extRegex = re"(\.[A-z]+)$"
-  illegalXmlRegex = re"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]"
-
-  twitter = parseUri("https://twitter.com")
+  illegalXmlRegex = re"(*UTF8)[^\x09\x0A\x0D\x20-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}]"
 
 proc getUrlPrefix*(cfg: Config): string =
   if cfg.useHttps: https & cfg.hostname
   else: "http://" & cfg.hostname
 
-proc stripHtml*(text: string): string =
+proc shortLink*(text: string; length=28): string =
+  result = text.replace(wwwRegex, "")
+  if result.len > length:
+    result = result[0 ..< length] & "…"
+    
+proc stripHtml*(text: string; shorten=false): string =
   var html = parseHtml(text)
   for el in html.findAll("a"):
     let link = el.attr("href")
     if "http" in link:
       if el.len == 0: continue
-      el[0].text = link
+      el[0].text =
+        if shorten: link.shortLink
+        else: link
   html.innerText()
 
 proc sanitizeXml*(text: string): string =
   text.replace(illegalXmlRegex, "")
 
-proc shortLink*(text: string; length=28): string =
-  result = text.replace(wwwRegex, "")
-  if result.len > length:
-    result = result[0 ..< length] & "…"
-
 proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
   result = body
 
-  if prefs.replaceYouTube.len > 0 and ytRegex in result:
+  if prefs.replaceYouTube.len > 0 and "youtu" in result:
     result = result.replace(ytRegex, prefs.replaceYouTube)
     if prefs.replaceYouTube in result:
       result = result.replace("/c/", "/")
 
-  if prefs.replaceTwitter.len > 0 and
-     (twRegex in result or twLinkRegex in result or tco in result):
+  if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
     result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
     result = result.replace(cards, prefs.replaceTwitter & "/cards")
     result = result.replace(twRegex, prefs.replaceTwitter)
-    result = result.replace(twLinkRegex, a(
+    result = result.replacef(twLinkRegex, a(
       prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
 
-  if prefs.replaceReddit.len > 0 and (rdRegex in result or "redd.it" in result):
+  if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
     result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
     result = result.replace(rdRegex, prefs.replaceReddit)
     if prefs.replaceReddit in result and "/gallery/" in result:
       result = result.replace("/gallery/", "/comments/")
 
-  if prefs.replaceInstagram.len > 0 and igRegex in result:
+  if prefs.replaceInstagram.len > 0 and "instagram.com" in result:
     result = result.replace(igRegex, prefs.replaceInstagram)
 
   if absolute.len > 0 and "href" in result:
     result = result.replace("href=\"/", "href=\"" & absolute & "/")
 
 proc getM3u8Url*(content: string): string =
-  var m: RegexMatch
-  if content.find(m3u8Regex, m):
-    result = content[m.group(0)[0]]
+  var matches: array[1, string]
+  if re.find(content, m3u8Regex, matches) != -1:
+    result = matches[0]
 
 proc proxifyVideo*(manifest: string; proxy: bool): string =
-  proc cb(m: RegexMatch; s: string): string =
-    result = "https://video.twimg.com/" & s[m.group(0)[0]]
-    if proxy: result = getVidUrl(result)
-  result = manifest.replace(manifestRegex, cb)
+  var replacements: seq[(string, string)]
+  for line in manifest.splitLines:
+    let url =
+      if line.startsWith("#EXT-X-MAP:URI"): line[16 .. ^2]
+      else: line
+    if url.startsWith('/'):
+      let path = "https://video.twimg.com" & url
+      replacements.add (url, if proxy: path.getVidUrl else: path)
+  return manifest.multiReplace(replacements)
 
 proc getUserPic*(userPic: string; style=""): string =
-  let pic = userPic.replace(userPicRegex, "$2")
-  pic.replace(extRegex, style & "$1")
+  userPic.replacef(userPicRegex, "$2").replacef(extRegex, style & "$1")
 
-proc getUserPic*(profile: Profile; style=""): string =
-  getUserPic(profile.userPic, style)
+proc getUserPic*(user: User; style=""): string =
+  getUserPic(user.userPic, style)
 
 proc getVideoEmbed*(cfg: Config; id: int64): string =
   &"{getUrlPrefix(cfg)}/i/videos/{id}"
 
-proc pageTitle*(profile: Profile): string =
-  &"{profile.fullname} (@{profile.username})"
+proc pageTitle*(user: User): string =
+  &"{user.fullname} (@{user.username})"
 
 proc pageTitle*(tweet: Tweet): string =
-  &"{pageTitle(tweet.profile)}: \"{stripHtml(tweet.text)}\""
+  &"{pageTitle(tweet.user)}: \"{stripHtml(tweet.text)}\""
 
-proc pageDesc*(profile: Profile): string =
-  if profile.bio.len > 0:
-    stripHtml(profile.bio)
+proc pageDesc*(user: User): string =
+  if user.bio.len > 0:
+    stripHtml(user.bio)
   else:
-    "The latest tweets from " & profile.fullname
+    "The latest tweets from " & user.fullname
 
-proc getJoinDate*(profile: Profile): string =
-  profile.joinDate.format("'Joined' MMMM YYYY")
+proc getJoinDate*(user: User): string =
+  user.joinDate.format("'Joined' MMMM YYYY")
 
-proc getJoinDateFull*(profile: Profile): string =
-  profile.joinDate.format("h:mm tt - d MMM YYYY")
+proc getJoinDateFull*(user: User): string =
+  user.joinDate.format("h:mm tt - d MMM YYYY")
 
 proc getTime*(tweet: Tweet): string =
   tweet.time.format("MMM d', 'YYYY' · 'h:mm tt' UTC'")
@@ -142,7 +146,7 @@ proc getShortTime*(tweet: Tweet): string =
 
 proc getLink*(tweet: Tweet; focus=true): string =
   if tweet.id == 0: return
-  var username = tweet.profile.username
+  var username = tweet.user.username
   if username.len == 0:
     username = "i"
   result = &"/{username}/status/{tweet.id}"
@@ -171,7 +175,7 @@ proc getTwitterLink*(path: string; params: Table[string, string]): string =
   if username.len > 0:
     result = result.replace("/" & username, "")
 
-proc getLocation*(u: Profile | Tweet): (string, string) =
+proc getLocation*(u: User | Tweet): (string, string) =
   if "://" in u.location: return (u.location, "")
   let loc = u.location.split(":")
   let url = if loc.len > 1: "/search?q=place:" & loc[1] else: ""
@@ -179,3 +183,13 @@ proc getLocation*(u: Profile | Tweet): (string, string) =
 
 proc getSuspended*(username: string): string =
   &"User \"{username}\" has been suspended"
+
+proc titleize*(str: string): string =
+  const
+    lowercase = {'a'..'z'}
+    delims = {' ', '('}
+
+  result = str
+  for i, c in enumerate(str):
+    if c in lowercase and (i == 0 or str[i - 1] in delims):
+      result[i] = c.toUpperAscii

src/http_pool.nim

@@ -1,12 +1,13 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient
+import httpclient
 
 type
   HttpPool* = ref object
     conns*: seq[AsyncHttpClient]
 
-var maxConns: int
-var proxy: Proxy
+var
+  maxConns: int
+  proxy: Proxy
 
 proc setMaxHttpConns*(n: int) =
   maxConns = n
@@ -17,20 +18,24 @@ proc setHttpProxy*(url: string; auth: string) =
   else:
     proxy = nil
 
-proc release*(pool: HttpPool; client: AsyncHttpClient) =
-  if pool.conns.len >= maxConns:
-    client.close()
+proc release*(pool: HttpPool; client: AsyncHttpClient; badClient=false) =
+  if pool.conns.len >= maxConns or badClient:
+    try: client.close()
+    except: discard
   elif client != nil:
     pool.conns.insert(client)
 
-template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
-  var c {.inject.}: AsyncHttpClient
-
+proc acquire*(pool: HttpPool; heads: HttpHeaders): AsyncHttpClient =
   if pool.conns.len == 0:
-    c = newAsyncHttpClient(headers=heads, proxy=proxy)
+    result = newAsyncHttpClient(headers=heads, proxy=proxy)
   else:
-    c = pool.conns.pop()
-    c.headers = heads
+    result = pool.conns.pop()
+    result.headers = heads
+
+template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
+  var
+    c {.inject.} = pool.acquire(heads)
+    badClient {.inject.} = false
 
   try:
     body
@@ -38,4 +43,4 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
     # Twitter closed the connection, retry
     body
   finally:
-    pool.release(c)
+    pool.release(c, badClient)

src/nitter.nim

@@ -32,6 +32,7 @@ setHmacKey(cfg.hmacKey)
 setProxyEncoding(cfg.base64Media)
 setMaxHttpConns(cfg.httpMaxConns)
 setHttpProxy(cfg.proxy, cfg.proxyAuth)
+initAboutPage(cfg.staticDir)
 
 waitFor initRedisPool(cfg)
 stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"

src/parser.nim

@@ -1,12 +1,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import strutils, options, tables, times, math
-import packedjson
-import packedjson / deserialiser
+import packedjson, packedjson/deserialiser
 import types, parserutils, utils
+import experimental/parser/unifiedcard
 
-proc parseProfile(js: JsonNode; id=""): Profile =
+proc parseUser(js: JsonNode; id=""): User =
   if js.isNull: return
-  result = Profile(
+  result = User(
     id: if id.len > 0: id else: js{"id_str"}.getStr,
     username: js{"screen_name"}.getStr,
     fullname: js{"name"}.getStr,
@@ -14,32 +14,27 @@ proc parseProfile(js: JsonNode; id=""): Profile =
     bio: js{"description"}.getStr,
     userPic: js{"profile_image_url_https"}.getImageStr.replace("_normal", ""),
     banner: js.getBanner,
-    following: $js{"friends_count"}.getInt,
-    followers: $js{"followers_count"}.getInt,
-    tweets: $js{"statuses_count"}.getInt,
-    likes: $js{"favourites_count"}.getInt,
-    media: $js{"media_count"}.getInt,
+    following: js{"friends_count"}.getInt,
+    followers: js{"followers_count"}.getInt,
+    tweets: js{"statuses_count"}.getInt,
+    likes: js{"favourites_count"}.getInt,
+    media: js{"media_count"}.getInt,
     verified: js{"verified"}.getBool,
     protected: js{"protected"}.getBool,
     joinDate: js{"created_at"}.getTime
   )
 
-  result.expandProfileEntities(js)
-
-proc parseUserShow*(js: JsonNode; username=""; id=""): Profile =
-  if id.len > 0:
-    result = Profile(id: id)
-  else:
-    result = Profile(username: username)
+  result.expandUserEntities(js)
 
+proc parseGraphUser*(js: JsonNode; id: string): User =
   if js.isNull: return
 
-  with error, js{"errors"}:
-    if error.getError == suspended:
-      result.suspended = true
-    return
+  with user, js{"data", "user", "result", "legacy"}:
+    result = parseUser(user, id)
 
-  result = parseProfile(js)
+    with pinned, user{"pinned_tweet_ids_str"}:
+      if pinned.kind == JArray and pinned.len > 0:
+        result.pinnedTweet = parseBiggestInt(pinned[0].getStr)
 
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
@@ -60,21 +55,25 @@ proc parseGraphList*(js: JsonNode): List =
     banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
   )
 
-proc parseListMembers*(js: JsonNode; cursor: string): Result[Profile] =
-  result = Result[Profile](
+proc parseGraphListMembers*(js: JsonNode; cursor: string): Result[User] =
+  result = Result[User](
     beginning: cursor.len == 0,
     query: Query(kind: userList)
   )
 
   if js.isNull: return
 
-  result.top = js{"previous_cursor_str"}.getStr
-  result.bottom = js{"next_cursor_str"}.getStr
-  if result.bottom.len == 1:
-    result.bottom.setLen 0
+  let root = js{"data", "list", "members_timeline", "timeline", "instructions"}
+  for instruction in root:
+    if instruction{"type"}.getStr == "TimelineAddEntries":
+      for entry in instruction{"entries"}:
+        let content = entry{"content"}
+        if content{"entryType"}.getStr == "TimelineTimelineItem":
+          with legacy, content{"itemContent", "user_results", "result", "legacy"}:
+            result.content.add parseUser(legacy)
+        elif content{"cursorType"}.getStr == "Bottom":
+          result.bottom = content{"value"}.getStr
 
-  for u in js{"users"}:
-    result.content.add parseProfile(u)
 
 proc parsePoll(js: JsonNode): Poll =
   let vals = js{"binding_values"}
@@ -102,7 +101,6 @@ proc parseGif(js: JsonNode): Gif =
 
 proc parseVideo(js: JsonNode): Video =
   result = Video(
-    videoId: js{"id_str"}.getStr,
     thumb: js{"media_url_https"}.getImageStr,
     views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
     available: js{"ext_media_availability", "status"}.getStr == "available",
@@ -119,7 +117,7 @@ proc parseVideo(js: JsonNode): Video =
 
   for v in js{"video_info", "variants"}:
     result.variants.add VideoVariant(
-      videoType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
+      contentType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
       bitrate: v{"bitrate"}.getInt,
       url: v{"url"}.getStr
     )
@@ -129,19 +127,17 @@ proc parsePromoVideo(js: JsonNode): Video =
     thumb: js{"player_image_large"}.getImageVal,
     available: true,
     durationMs: js{"content_duration_seconds"}.getStrVal("0").parseInt * 1000,
-    playbackType: vmap,
-    videoId: js{"player_content_id"}.getStrVal(js{"card_id"}.getStrVal(
-        js{"amplify_content_id"}.getStrVal())),
+    playbackType: vmap
   )
 
   var variant = VideoVariant(
-    videoType: vmap,
+    contentType: vmap,
     url: js{"player_hls_url"}.getStrVal(js{"player_stream_url"}.getStrVal(
         js{"amplify_url_vmap"}.getStrVal()))
   )
 
   if "m3u8" in variant.url:
-    variant.videoType = m3u8
+    variant.contentType = m3u8
     result.playbackType = m3u8
 
   result.variants.add variant
@@ -154,7 +150,7 @@ proc parseBroadcast(js: JsonNode): Card =
     title: js{"broadcaster_display_name"}.getStrVal,
     text: js{"broadcast_title"}.getStrVal,
     image: image,
-    video: some Video(videoId: js{"broadcast_media_id"}.getStrVal, thumb: image)
+    video: some Video(thumb: image)
   )
 
 proc parseCard(js: JsonNode; urls: JsonNode): Card =
@@ -166,6 +162,9 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
     name = js{"name"}.getStr
     kind = parseEnum[CardKind](name[(name.find(":") + 1) ..< name.len], unknown)
 
+  if kind == unified:
+    return parseUnifiedCard(vals{"unified_card", "string_value"}.getStr)
+
   result = Card(
     kind: kind,
     url: vals.getCardUrl(kind),
@@ -190,7 +189,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
     result.url = vals{"player_url"}.getStrVal
     if "youtube.com" in result.url:
       result.url = result.url.replace("/embed/", "/watch?v=")
-  of audiospace, unified, unknown:
+  of audiospace, unknown:
     result.title = "This card type is not supported."
   else: discard
 
@@ -221,7 +220,7 @@ proc parseTweet(js: JsonNode): Tweet =
     time: js{"created_at"}.getTime,
     hasThread: js{"self_thread"}.notNull,
     available: true,
-    profile: Profile(id: js{"user_id_str"}.getStr),
+    user: User(id: js{"user_id_str"}.getStr),
     stats: TweetStats(
       replies: js{"reply_count"}.getInt,
       retweets: js{"retweet_count"}.getInt,
@@ -259,7 +258,7 @@ proc parseTweet(js: JsonNode): Tweet =
       of "video":
         result.video = some(parseVideo(m))
         with user, m{"additional_media_info", "source_user"}:
-          result.attribution = some(parseProfile(user))
+          result.attribution = some(parseUser(user))
       of "animated_gif":
         result.gif = some(parseGif(m))
       else: discard
@@ -313,36 +312,32 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
     users = ? js{"globalObjects", "users"}
 
   for k, v in users:
-    result.users[k] = parseProfile(v, k)
+    result.users[k] = parseUser(v, k)
 
   for k, v in tweets:
     var tweet = parseTweet(v)
-    if tweet.profile.id in result.users:
-      tweet.profile = result.users[tweet.profile.id]
+    if tweet.user.id in result.users:
+      tweet.user = result.users[tweet.user.id]
     result.tweets[k] = tweet
 
 proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
   result.thread = Chain()
-  for t in js{"content", "timelineModule", "items"}:
-    let content = t{"item", "content"}
-    if "Self" in content{"tweet", "displayType"}.getStr:
+
+  let thread = js{"content", "item", "content", "conversationThread"}
+  with cursor, thread{"showMoreCursor"}:
+    result.thread.cursor = cursor{"value"}.getStr
+    result.thread.hasMore = true
+
+  for t in thread{"conversationComponents"}:
+    let content = t{"conversationTweetComponent", "tweet"}
+
+    if content{"displayType"}.getStr == "SelfThread":
       result.self = true
 
-    let entry = t{"entryId"}.getStr
-    if "show_more" in entry:
-      let
-        cursor = content{"timelineCursor"}
-        more = cursor{"displayTreatment", "actionText"}.getStr
-      result.thread.cursor = cursor{"value"}.getStr
-      if more.len > 0 and more[0].isDigit():
-        result.thread.more = parseInt(more[0 ..< more.find(" ")])
-      else:
-        result.thread.more = -1
-    else:
-      var tweet = finalizeTweet(global, t.getEntryId)
-      if not tweet.available:
-        tweet.tombstone = getTombstone(content{"tombstone"})
-      result.thread.content.add tweet
+    var tweet = finalizeTweet(global, content{"id"}.getStr)
+    if not tweet.available:
+      tweet.tombstone = getTombstone(content{"tombstone"})
+    result.thread.content.add tweet
 
 proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
   result = Conversation(replies: Result[Chain](beginning: true))
@@ -372,6 +367,18 @@ proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
     elif "cursor-bottom" in entry:
       result.replies.bottom = e.getCursor
 
+proc parseStatus*(js: JsonNode): Tweet =
+  with e, js{"errors"}:
+    if e.getError == tweetNotFound:
+      return
+
+  result = parseTweet(js)
+  if not result.isNil:
+    result.user = parseUser(js{"user"})
+
+    with quote, js{"quoted_status"}:
+      result.quote = some parseStatus(js{"quoted_status"})
+
 proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
   if js.kind != JArray or js.len == 0:
     return
@@ -388,8 +395,8 @@ proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNod
       elif "bottom" in r{"entryId"}.getStr:
         res.bottom = r.getCursor
 
-proc parseUsers*(js: JsonNode; after=""): Result[Profile] =
-  result = Result[Profile](beginning: after.len == 0)
+proc parseUsers*(js: JsonNode; after=""): Result[User] =
+  result = Result[User](beginning: after.len == 0)
   let global = parseGlobalObjects(? js)
 
   let instructions = ? js{"timeline", "instructions"}
@@ -419,7 +426,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
 
   for e in instructions[0]{"addEntries", "entries"}:
     let entry = e{"entryId"}.getStr
-    if "tweet" in entry or "sq-I-t" in entry or "tombstone" in entry:
+    if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
       let tweet = finalizeTweet(global, e.getEntryId)
       if not tweet.available: continue
       result.content.add tweet
@@ -427,6 +434,12 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
       result.top = e.getCursor
     elif "cursor-bottom" in entry:
       result.bottom = e.getCursor
+    elif entry.startsWith("sq-C"):
+      with cursor, e{"content", "operation", "cursor"}:
+        if cursor{"cursorType"}.getStr == "Bottom":
+          result.bottom = cursor{"value"}.getStr
+        else:
+          result.top = cursor{"value"}.getStr
 
 proc parsePhotoRail*(js: JsonNode): PhotoRail =
   for tweet in js:

src/parserutils.nim

@@ -1,13 +1,14 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, times, macros, htmlgen, unicode, options, algorithm
-import regex, packedjson
+import std/[strutils, times, macros, htmlgen, options, algorithm, re]
+import std/unicode except strip
+import packedjson
 import types, utils, formatters
 
-const
+let
   unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
   unReplace = "$1<a href=\"/$2\">@$2</a>"
 
-  htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
+  htRegex = re"(^|[^\w-_./?])([#$]|＃)([\w_]+)"
   htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
 
 type
@@ -128,8 +129,6 @@ proc getBanner*(js: JsonNode): string =
       result.add toHex(pal{"blue"}.getInt, 2)
       return
 
-  return "#161616"
-
 proc getTombstone*(js: JsonNode): string =
   result = js{"tombstoneInfo", "richText", "text"}.getStr
   result.removeSuffix(" Learn more")
@@ -195,13 +194,13 @@ proc deduplicate(s: var seq[ReplaceSlice]) =
 
 proc cmp(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
 
-proc expandProfileEntities*(profile: var Profile; js: JsonNode) =
+proc expandUserEntities*(user: var User; js: JsonNode) =
   let
-    orig = profile.bio.toRunes
+    orig = user.bio.toRunes
     ent = ? js{"entities"}
 
   with urls, ent{"url", "urls"}:
-    profile.website = urls[0]{"expanded_url"}.getStr
+    user.website = urls[0]{"expanded_url"}.getStr
 
   var replacements = newSeq[ReplaceSlice]()
 
@@ -212,9 +211,9 @@ proc expandProfileEntities*(profile: var Profile; js: JsonNode) =
   replacements.deduplicate
   replacements.sort(cmp)
 
-  profile.bio = orig.replacedWith(replacements, 0 .. orig.len)
-  profile.bio = profile.bio.replace(unRegex, unReplace)
-                           .replace(htRegex, htReplace)
+  user.bio = orig.replacedWith(replacements, 0 .. orig.len)
+  user.bio = user.bio.replacef(unRegex, unReplace)
+                     .replacef(htRegex, htReplace)
 
 proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
   let
@@ -274,3 +273,4 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
   replacements.sort(cmp)
 
   tweet.text = orig.replacedWith(replacements, textSlice)
+                   .strip(leading=false)

src/prefs_impl.nim

@@ -75,6 +75,9 @@ genPrefs:
     hideReplies(checkbox, false):
       "Hide tweet replies"
 
+    squareAvatars(checkbox, false):
+      "Square profile pictures"
+
   Media:
     mp4Playback(checkbox, true):
       "Enable mp4 video playback (only for gifs)"

src/redis_cache.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, times, strutils, tables, hashes
+import asyncdispatch, times, strformat, strutils, tables, hashes
 import redis, redpool, flatty, supersnappy
 
 import types, api
@@ -13,6 +13,9 @@ var
   rssCacheTime: int
   listCacheTime*: int
 
+template dawait(future) =
+  discard await future
+
 # flatty can't serialize DateTime, so we need to define this
 proc toFlatty*(s: var string, x: DateTime) =
   s.toFlatty(x.toTime().toUnix())
@@ -33,9 +36,9 @@ proc migrate*(key, match: string) {.async.} =
       let list = await r.scan(newCursor(0), match, 100000)
       r.startPipelining()
       for item in list:
-        discard await r.del(item)
+        dawait r.del(item)
       await r.setk(key, "true")
-      discard await r.flushPipeline()
+      dawait r.flushPipeline()
 
 proc initRedisPool*(cfg: Config) {.async.} =
   try:
@@ -47,9 +50,11 @@ proc initRedisPool*(cfg: Config) {.async.} =
     await migrate("snappyRss", "rss:*")
     await migrate("userBuckets", "p:*")
     await migrate("profileDates", "p:*")
+    await migrate("profileStats", "p:*")
+    await migrate("userType", "p:*")
 
     pool.withAcquire(r):
-      # optimize memory usage for profile ID buckets
+      # optimize memory usage for user ID buckets
       await r.configSet("hash-max-ziplist-entries", "1000")
 
   except OSError:
@@ -57,9 +62,10 @@ proc initRedisPool*(cfg: Config) {.async.} =
     stdout.flushFile
     quit(1)
 
-template pidKey(name: string): string = "pid:" & $(hash(name) div 1_000_000)
-template profileKey(name: string): string = "p:" & name
+template uidKey(name: string): string = "pid:" & $(hash(name) div 1_000_000)
+template userKey(name: string): string = "p:" & name
 template listKey(l: List): string = "l:" & l.id
+template tweetKey(id: int64): string = "t:" & $id
 
 proc get(query: string): Future[string] {.async.} =
   pool.withAcquire(r):
@@ -67,7 +73,13 @@ proc get(query: string): Future[string] {.async.} =
 
 proc setEx(key: string; time: int; data: string) {.async.} =
   pool.withAcquire(r):
-    discard await r.setEx(key, time, data)
+    dawait r.setEx(key, time, data)
+
+proc cacheUserId(username, id: string) {.async.} =
+  if username.len == 0 or id.len == 0: return
+  let name = toLower(username)
+  pool.withAcquire(r):
+    dawait r.hSet(name.uidKey, name, id)
 
 proc cache*(data: List) {.async.} =
   await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
@@ -75,59 +87,79 @@ proc cache*(data: List) {.async.} =
 proc cache*(data: PhotoRail; name: string) {.async.} =
   await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
 
-proc cache*(data: Profile) {.async.} =
-  if data.username.len == 0 or data.id.len == 0: return
+proc cache*(data: User) {.async.} =
+  if data.username.len == 0: return
   let name = toLower(data.username)
+  await cacheUserId(name, data.id)
   pool.withAcquire(r):
-    r.startPipelining()
-    discard await r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
-    discard await r.setEx("i:" & data.id , baseCacheTime, data.username)
-    discard await r.hSet(name.pidKey, name, data.id)
-    discard await r.flushPipeline()
+    dawait r.setEx(name.userKey, baseCacheTime, compress(toFlatty(data)))
 
-proc cacheProfileId*(username, id: string) {.async.} =
-  if username.len == 0 or id.len == 0: return
-  let name = toLower(username)
+proc cache*(data: Tweet) {.async.} =
+  if data.isNil or data.id == 0: return
   pool.withAcquire(r):
-    discard await r.hSet(name.pidKey, name, id)
+    dawait r.setEx(data.id.tweetKey, baseCacheTime, compress(toFlatty(data)))
 
 proc cacheRss*(query: string; rss: Rss) {.async.} =
   let key = "rss:" & query
   pool.withAcquire(r):
-    r.startPipelining()
-    discard await r.hSet(key, "rss", rss.feed)
-    discard await r.hSet(key, "min", rss.cursor)
-    discard await r.expire(key, rssCacheTime)
-    discard await r.flushPipeline()
+    dawait r.hSet(key, "rss", rss.feed)
+    dawait r.hSet(key, "min", rss.cursor)
+    dawait r.expire(key, rssCacheTime)
 
-proc getProfileId*(username: string): Future[string] {.async.} =
+template deserialize(data, T) =
+  try:
+    result = fromFlatty(uncompress(data), T)
+  except:
+    echo "Decompression failed($#): '$#'" % [astToStr(T), data]
+
+proc getUserId*(username: string): Future[string] {.async.} =
   let name = toLower(username)
   pool.withAcquire(r):
-    result = await r.hGet(name.pidKey, name)
+    result = await r.hGet(name.uidKey, name)
     if result == redisNil:
-      result.setLen(0)
+      let user = await getUser(username)
+      if user.suspended:
+        return "suspended"
+      else:
+        await cacheUserId(name, user.id)
+        return user.id
 
-proc getCachedProfile*(username: string; fetch=true): Future[Profile] {.async.} =
+proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
   let prof = await get("p:" & toLower(username))
   if prof != redisNil:
-    result = fromFlatty(uncompress(prof), Profile)
+    prof.deserialize(User)
   elif fetch:
-    result = await getProfile(username)
+    let userId = await getUserId(username)
+    result = await getGraphUser(userId)
+    await cache(result)
+
+proc getCachedUsername*(userId: string): Future[string] {.async.} =
+  let
+    key = "i:" & userId
+    username = await get(key)
 
-proc getCachedProfileUsername*(userId: string): Future[string] {.async.} =
-  let username = await get("i:" & userId)
   if username != redisNil:
     result = username
   else:
-    let profile = await getProfileById(userId)
-    result = profile.username
-    await cache(profile)
+    let user = await getUserById(userId)
+    result = user.username
+    await setEx(key, baseCacheTime, result)
+
+proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
+  if id == 0: return
+  let tweet = await get(id.tweetKey)
+  if tweet != redisNil:
+    tweet.deserialize(Tweet)
+  else:
+    result = await getStatus($id)
+    if result.isNil:
+      await cache(result)
 
 proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
   if name.len == 0: return
   let rail = await get("pr:" & toLower(name))
   if rail != redisNil:
-    result = fromFlatty(uncompress(rail), PhotoRail)
+    rail.deserialize(PhotoRail)
   else:
     result = await getPhotoRail(name)
     await cache(result, name)
@@ -137,7 +169,7 @@ proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
              else: await get("l:" & id)
 
   if list != redisNil:
-    result = fromFlatty(uncompress(list), List)
+    list.deserialize(List)
   else:
     if id.len > 0:
       result = await getGraphList(id)

src/routes/embed.nim

@@ -1,9 +1,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, strutils, options
-import jester
-import ".."/[types, api], ../views/embed
+import jester, karax/vdom
+import ".."/[types, api]
+import ../views/[embed, tweet, general]
+import router_utils
 
-export api, embed
+export api, embed, vdom, tweet, general, router_utils
 
 proc createEmbedRouter*(cfg: Config) =
   router embed:
@@ -12,4 +14,23 @@ proc createEmbedRouter*(cfg: Config) =
       if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
         resp Http404
 
-      resp renderVideoEmbed(cfg, convo.tweet)
+      resp renderVideoEmbed(convo.tweet, cfg, request)
+
+    get "/@user/status/@id/embed":
+      let
+        convo = await getTweet(@"id")
+        prefs = cookiePrefs()
+        path = getPath()
+
+      if convo == nil or convo.tweet == nil:
+        resp Http404
+
+      resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
+
+    get "/embed/Tweet.html":
+      let id = @"id"
+
+      if id.len > 0:
+        redirect("/i/status/" & id & "/embed")
+      else:
+        resp Http404

src/routes/list.nim

@@ -47,5 +47,5 @@ proc createListRouter*(cfg: Config) =
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
         title = "@" & list.username & "/" & list.name
-        members = await getListMembers(list, getCursor())
+        members = await getGraphListMembers(list, getCursor())
       respList(list, members, title, renderTimelineUsers(members, prefs, request.path))

src/routes/media.nim

@@ -5,7 +5,7 @@ import asynchttpserver, asyncstreams, asyncfile, asyncnet
 import jester
 
 import router_utils
-import ".."/[types, formatters, agents, utils]
+import ".."/[types, formatters, utils]
 
 export asynchttpserver, asyncstreams, asyncfile, asyncnet
 export httpclient, os, strutils, asyncstreams, base64, re
@@ -14,10 +14,8 @@ const
   m3u8Mime* = "application/vnd.apple.mpegurl"
   maxAge* = "max-age=604800"
 
-let mediaAgent* = getAgent()
-
-proc safeFetch*(url, agent: string): Future[string] {.async.} =
-  let client = newAsyncHttpClient(userAgent=agent)
+proc safeFetch*(url: string): Future[string] {.async.} =
+  let client = newAsyncHttpClient()
   try: result = await client.getContent(url)
   except: discard
   finally: client.close()
@@ -34,7 +32,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
   result = Http200
   let
     request = req.getNativeReq()
-    client = newAsyncHttpClient(userAgent=mediaAgent)
+    client = newAsyncHttpClient()
 
   try:
     let res = await client.get(url)
@@ -116,14 +114,14 @@ proc createMediaRouter*(cfg: Config) =
 
       var content: string
       if ".vmap" in url:
-        let m3u8 = getM3u8Url(await safeFetch(url, mediaAgent))
+        let m3u8 = getM3u8Url(await safeFetch(url))
         if m3u8.len > 0:
-          content = await safeFetch(url, mediaAgent)
+          content = await safeFetch(url)
         else:
           resp Http404
 
       if ".m3u8" in url:
-        let vid = await safeFetch(url, mediaAgent)
+        let vid = await safeFetch(url)
         content = proxifyVideo(vid, cookiePref(proxyVideos))
 
       resp content, m3u8Mime

src/routes/preferences.nim

@@ -4,7 +4,7 @@ import strutils, uri, os, algorithm
 import jester
 
 import router_utils
-import ".."/[types]
+import ".."/[types, formatters]
 import ../views/[general, preferences]
 
 export preferences
@@ -12,7 +12,7 @@ export preferences
 proc findThemes*(dir: string): seq[string] =
   for kind, path in walkDir(dir / "css" / "themes"):
     let theme = path.splitFile.name
-    result.add theme.capitalizeAscii.replace("_", " ")
+    result.add theme.replace("_", " ").titleize
   sort(result)
 
 proc createPrefRouter*(cfg: Config) =

src/routes/router_utils.nim

@@ -4,12 +4,12 @@ from jester import Request, cookies
 
 import ../views/general
 import ".."/[utils, prefs, types]
-export utils, prefs, types
+export utils, prefs, types, uri
 
 template savePref*(pref, value: string; req: Request; expire=false) =
   if not expire or pref in cookies(req):
     setCookie(pref, value, daysForward(when expire: -10 else: 360),
-              httpOnly=true, secure=cfg.useHttps)
+              httpOnly=true, secure=cfg.useHttps, sameSite=None)
 
 template cookiePrefs*(): untyped {.dirty.} =
   getPrefs(cookies(request))

src/routes/rss.nim

@@ -12,33 +12,32 @@ export times, hashes, supersnappy
 
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
   var profile: Profile
-  var timeline: Timeline
   let
     name = req.params.getOrDefault("name")
     after = getCursor(req)
     names = getNames(name)
 
   if names.len == 1:
-    (profile, timeline) =
-      await fetchSingleTimeline(after, query, skipRail=true)
+    profile = await fetchProfile(after, query, skipRail=true, skipPinned=true)
   else:
     var q = query
     q.fromUser = names
-    timeline = await getSearch[Tweet](q, after)
-    # this is kinda dumb
     profile = Profile(
-      username: name,
-      fullname: names.join(" | "),
-      userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
+      tweets: await getSearch[Tweet](q, after),
+      # this is kinda dumb
+      user: User(
+        username: name,
+        fullname: names.join(" | "),
+        userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
+      )
     )
 
-  if profile.suspended:
-    return Rss(feed: profile.username, cursor: "suspended")
+  if profile.user.suspended:
+    return Rss(feed: profile.user.username, cursor: "suspended")
 
-  if profile.fullname.len > 0:
-    let rss = compress renderTimelineRss(timeline, profile, cfg,
-                                         multi=(names.len > 1))
-    return Rss(feed: rss, cursor: timeline.bottom)
+  if profile.user.fullname.len > 0:
+    let rss = compress renderTimelineRss(profile, cfg, multi=(names.len > 1))
+    return Rss(feed: rss, cursor: profile.tweets.bottom)
 
 template respRss*(rss, page) =
   if rss.cursor.len == 0:

src/routes/search.nim

@@ -25,7 +25,7 @@ proc createSearchRouter*(cfg: Config) =
       of users:
         if "," in @"q":
           redirect("/" & @"q")
-        let users = await getSearch[Profile](query, getCursor())
+        let users = await getSearch[User](query, getCursor())
         resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
       of tweets:
         let

src/routes/status.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, sequtils, uri, options
+import asyncdispatch, strutils, sequtils, uri, options, sugar
 
 import jester, karax/vdom
 
@@ -7,7 +7,7 @@ import router_utils
 import ".."/[types, formatters, api]
 import ../views/[general, status]
 
-export uri, sequtils, options
+export uri, sequtils, options, sugar
 export router_utils
 export api, formatters
 export status
@@ -16,6 +16,7 @@ proc createStatusRouter*(cfg: Config) =
   router status:
     get "/@name/status/@id/?":
       cond '.' notin @"name"
+      cond not @"id".any(c => not c.isDigit)
       let prefs = cookiePrefs()
 
       # used for the infinite scroll feature
@@ -37,7 +38,7 @@ proc createStatusRouter*(cfg: Config) =
 
       let
         title = pageTitle(conv.tweet)
-        ogTitle = pageTitle(conv.tweet.profile)
+        ogTitle = pageTitle(conv.tweet.user)
         desc = conv.tweet.text
 
       var

src/routes/timeline.nim

@@ -19,64 +19,64 @@ proc getQuery*(request: Request; tab, name: string): Query =
   of "search": initQuery(params(request), name=name)
   else: Query(fromUser: @[name])
 
-proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
-                        Future[(Profile, Timeline, PhotoRail)] {.async.} =
-  let name = query.fromUser[0]
-
-  var
-    profile: Profile
-    profileId = await getProfileId(name)
-    fetched = false
-
-  if profileId.len == 0:
-    profile = await getCachedProfile(name)
-    profileId = if profile.suspended: "s"
-                else: profile.id
-
-    if profileId.len > 0:
-       await cacheProfileId(profile.username, profileId)
-
-    fetched = true
-
-  if profileId.len == 0 or profile.protected:
-    result[0] = profile
-    return
-  elif profileId == "s":
-    result[0] = Profile(username: name, suspended: true)
-    return
-
-  var rail: Future[PhotoRail]
-  if skipRail or profile.protected or query.kind == media:
-    rail = newFuture[PhotoRail]()
-    rail.complete(@[])
+template skipIf[T](cond: bool; default; body: Future[T]): Future[T] =
+  if cond:
+    let fut = newFuture[T]()
+    fut.complete(default)
+    fut
   else:
-    rail = getCachedPhotoRail(name)
+    body
 
-  var timeline =
-    case query.kind
-    of posts: await getTimeline(profileId, after)
-    of replies: await getTimeline(profileId, after, replies=true)
-    of media: await getMediaTimeline(profileId, after)
-    else: await getSearch[Tweet](query, after)
+proc fetchProfile*(after: string; query: Query; skipRail=false;
+                   skipPinned=false): Future[Profile] {.async.} =
+  let
+    name = query.fromUser[0]
+    userId = await getUserId(name)
 
-  timeline.query = query
+  if userId.len == 0:
+    return Profile(user: User(username: name))
+  elif userId == "suspended":
+    return Profile(user: User(username: name, suspended: true))
 
-  var found = false
-  for tweet in timeline.content.mitems:
-    if tweet.profile.id == profileId or
-       tweet.profile.username.cmpIgnoreCase(name) == 0:
-      profile = tweet.profile
-      found = true
-      break
+  # temporary fix to prevent errors from people browsing
+  # timelines during/immediately after deployment
+  var after = after
+  if query.kind in {posts, replies} and after.startsWith("scroll"):
+    after.setLen 0
 
-  if profile.username.len == 0:
-    profile = await getCachedProfile(name)
-    fetched = true
+  let
+    timeline =
+      case query.kind
+      of posts: getTimeline(userId, after)
+      of replies: getTimeline(userId, after, replies=true)
+      of media: getMediaTimeline(userId, after)
+      else: getSearch[Tweet](query, after)
 
-  if fetched and not found:
-    await cache(profile)
+    rail =
+      skipIf(skipRail or query.kind == media, @[]):
+        getCachedPhotoRail(name)
 
-  return (profile, timeline, await rail)
+    user = await getCachedUser(name)
+
+  var pinned: Option[Tweet]
+  if not skipPinned and user.pinnedTweet > 0 and
+     after.len == 0 and query.kind in {posts, replies}:
+    let tweet = await getCachedTweet(user.pinnedTweet)
+    if not tweet.isNil:
+      tweet.pinned = true
+      pinned = some tweet
+
+  result = Profile(
+    user: user,
+    pinned: pinned,
+    tweets: await timeline,
+    photoRail: await rail
+  )
+
+  if result.user.protected or result.user.suspended:
+    return
+
+  result.tweets.query = query
 
 proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
                    rss, after: string): Future[string] {.async.} =
@@ -86,15 +86,18 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
 
-  var (p, t, r) = await fetchSingleTimeline(after, query)
+  var profile = await fetchProfile(after, query)
+  template u: untyped = profile.user
 
-  if p.suspended: return showError(getSuspended(p.username), cfg)
-  if p.id.len == 0: return
+  if u.suspended:
+    return showError(getSuspended(u.username), cfg)
 
-  let pHtml = renderProfile(p, t, r, prefs, getPath())
-  result = renderMain(pHtml, request, cfg, prefs, pageTitle(p), pageDesc(p),
-                      rss=rss, images = @[p.getUserPic("_400x400")],
-                      banner=p.banner)
+  if profile.user.id.len == 0: return
+
+  let pHtml = renderProfile(profile, prefs, getPath())
+  result = renderMain(pHtml, request, cfg, prefs, pageTitle(u), pageDesc(u),
+                      rss=rss, images = @[u.getUserPic("_400x400")],
+                      banner=u.banner)
 
 template respTimeline*(timeline: typed) =
   let t = timeline
@@ -104,7 +107,7 @@ template respTimeline*(timeline: typed) =
 
 template respUserId*() =
   cond @"user_id".len > 0
-  let username = await getCachedProfileUsername(@"user_id")
+  let username = await getCachedUsername(@"user_id")
   if username.len > 0:
     redirect("/" & username)
   else:
@@ -139,10 +142,10 @@ proc createTimelineRouter*(cfg: Config) =
           timeline.beginning = true
           resp $renderTweetSearch(timeline, prefs, getPath())
         else:
-          var (_, timeline, _) = await fetchSingleTimeline(after, query, skipRail=true)
-          if timeline.content.len == 0: resp Http404
-          timeline.beginning = true
-          resp $renderTimelineTweets(timeline, prefs, getPath())
+          var profile = await fetchProfile(after, query, skipRail=true)
+          if profile.tweets.content.len == 0: resp Http404
+          profile.tweets.beginning = true
+          resp $renderTimelineTweets(profile.tweets, prefs, getPath())
 
       let rss =
         if @"tab".len == 0:

src/routes/unsupported.nim

@@ -3,7 +3,9 @@ import jester
 
 import router_utils
 import ../types
-import ../views/[general, about]
+import ../views/[general, feature]
+
+export feature
 
 proc createUnsupportedRouter*(cfg: Config) =
   router unsupported:

src/sass/profile/_base.scss

@@ -15,23 +15,22 @@
 }
 
 .profile-banner {
-    padding-bottom: 4px;
+    margin-bottom: 4px;
+    background-color: var(--bg_panel);
 
     a {
-        display: inherit;
-        line-height: 0;
+        display: block;
+        position: relative;
+        padding: 33.34% 0 0 0;
     }
 
     img {
-        width: 100%;
+        max-width: 100%;
+        position: absolute;
+        top: 0;
     }
 }
 
-.profile-banner-color {
-    width: 100%;
-    padding-bottom: 25%;
-}
-
 .profile-tab {
     padding: 0 4px 0 0;
     box-sizing: border-box;

src/sass/profile/card.scss

@@ -35,19 +35,25 @@
 }
 
 .profile-card-avatar {
-    display: block;
+    display: inline-block;
+    position: relative;
     width: 100%;
-    padding-bottom: 6px;
     margin-right: 4px;
+    margin-bottom: 6px;
+
+    &:after {
+        content: '';
+        display: block;
+        margin-top: 100%;
+    }
 
     img {
-        display: block;
         box-sizing: border-box;
+        position: absolute;
         width: 100%;
         height: 100%;
-        margin: 0;
         border: 4px solid var(--darker_grey);
-        background: var(--bg_color);
+        background: var(--bg_panel);
     }
 }
 
@@ -113,8 +119,8 @@
     }
 
     .profile-card-avatar {
-        height: 60px;
-        width: unset;
+        width: 80px;
+        height: 80px;
 
         img {
             border-width: 2px;

src/sass/profile/photo-rail.scss

@@ -32,6 +32,7 @@
         a {
             position: relative;
             border-radius: 5px;
+            background-color: var(--darker_grey);
 
             &:before {
                 content: "";

src/sass/tweet/_base.scss

@@ -98,14 +98,37 @@
 }
 
 .avatar {
-    border-radius: 50%;
+    position: absolute;
+
+    &.round {
+        border-radius: 50%;
+    }
+    
+    &.mini {
+        position: unset;
+        margin-right: 5px;
+        margin-top: -1px;
+        width: 20px;
+        height: 20px;
+    }
 }
 
-.avatar.mini {
-    margin-right: 5px;
-    margin-top: -1px;
-    width: 20px;
-    height: 20px;
+.tweet-embed {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    height: 100%;
+    background-color: var(--bg_panel);
+
+    .tweet-content { 
+        font-size: 18px 
+    }
+    
+    .tweet-body { 
+        display: flex;
+        flex-direction: column;
+        max-height: calc(100vh - 0.75em * 2);
+    }
 }
 
 .attribution {

src/tokens.nim

@@ -2,12 +2,12 @@
 import asyncdispatch, httpclient, times, sequtils, json, random
 import strutils, tables
 import zippy
-import types, agents, consts, http_pool
+import types, consts, http_pool
 
 const
   maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
-  maxAge = 3.hours       # tokens expire after 3 hours
   maxLastUse = 1.hours   # if a token is unused for 60 minutes, it expires
+  maxAge = 2.hours + 55.minutes  # tokens expire after 3 hours
   failDelay = initDuration(minutes=30)
 
 var
@@ -37,7 +37,7 @@ proc getPoolJson*(): JsonNode =
       let
         maxReqs =
           case api
-          of Api.listBySlug, Api.list: 500
+          of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500
           of Api.timeline: 187
           else: 180
         reqs = maxReqs - token.apis[api].remaining
@@ -65,7 +65,6 @@ proc fetchToken(): Future[Token] {.async.} =
     "accept-encoding": "gzip",
     "accept-language": "en-US,en;q=0.5",
     "connection": "keep-alive",
-    "user-agent": getAgent(),
     "authorization": auth
   })
 

src/types.nim

@@ -10,13 +10,14 @@ type
 
   Api* {.pure.} = enum
     userShow
-    photoRail
     timeline
     search
     tweet
     list
     listBySlug
     listMembers
+    userRestId
+    status
 
   RateLimit* = object
     remaining*: int
@@ -40,25 +41,26 @@ type
     rateLimited = 88
     invalidToken = 89
     listIdOrSlug = 112
+    tweetNotFound = 144
     forbidden = 200
     badToken = 239
     noCsrf = 353
 
-  Profile* = object
+  User* = object
     id*: string
     username*: string
     fullname*: string
-    lowername*: string
     location*: string
     website*: string
     bio*: string
     userPic*: string
     banner*: string
-    following*: string
-    followers*: string
-    tweets*: string
-    likes*: string
-    media*: string
+    pinnedTweet*: int64
+    following*: int
+    followers*: int
+    tweets*: int
+    likes*: int
+    media*: int
     verified*: bool
     protected*: bool
     suspended*: bool
@@ -70,12 +72,11 @@ type
     vmap = "video/vmap"
 
   VideoVariant* = object
-    videoType*: VideoType
+    contentType*: VideoType
     url*: string
     bitrate*: int
 
   Video* = object
-    videoId*: string
     durationMs*: int
     url*: string
     thumb*: string
@@ -147,8 +148,6 @@ type
     
   Card* = object
     kind*: CardKind
-    id*: string
-    query*: string
     url*: string
     title*: string
     dest*: string
@@ -166,7 +165,7 @@ type
     id*: int64
     threadId*: int64
     replyId*: int64
-    profile*: Profile
+    user*: User
     text*: string
     time*: DateTime
     reply*: seq[string]
@@ -177,8 +176,8 @@ type
     location*: string
     stats*: TweetStats
     retweet*: Option[Tweet]
-    attribution*: Option[Profile]
-    mediaTags*: seq[Profile]
+    attribution*: Option[User]
+    mediaTags*: seq[User]
     quote*: Option[Tweet]
     card*: Option[Card]
     poll*: Option[Poll]
@@ -194,7 +193,7 @@ type
 
   Chain* = object
     content*: seq[Tweet]
-    more*: int64
+    hasMore*: bool
     cursor*: string
 
   Conversation* = ref object
@@ -205,6 +204,12 @@ type
 
   Timeline* = Result[Tweet]
 
+  Profile* = object
+    user*: User
+    photoRail*: PhotoRail
+    pinned*: Option[Tweet]
+    tweets*: Timeline
+
   List* = object
     id*: string
     name*: string
@@ -216,7 +221,7 @@ type
 
   GlobalObjects* = ref object
     tweets*: Table[string, Tweet]
-    users*: Table[string, Profile]
+    users*: Table[string, User]
 
   Config* = ref object
     address*: string

src/utils.nim

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import strutils, strformat, uri, tables, base64
-import nimcrypto, regex
+import nimcrypto
 
 var
   hmacKey: string

src/views/about.nim

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strformat
-import karax/[karaxdsl, vdom], markdown
+import os, strformat
+import karax/[karaxdsl, vdom]
 
 const
   date = staticExec("git show -s --format=\"%cd\" --date=format:\"%Y.%m.%d\"")
@@ -8,18 +8,19 @@ const
   link = "https://github.com/zedeus/nitter/commit/" & hash
   version = &"{date}-{hash}"
 
-let
-  about = markdown(readFile("public/md/about.md"))
-  feature = markdown(readFile("public/md/feature.md"))
+var aboutHtml: string
+
+proc initAboutPage*(dir: string) =
+  try:
+    aboutHtml = readFile(dir/"md/about.html")
+  except IOError:
+    stderr.write (dir/"md/about.html") & " not found, please run `nimble md`\n"
+    aboutHtml = "<h1>About page is missing</h1><br><br>"
 
 proc renderAbout*(): VNode =
   buildHtml(tdiv(class="overlay-panel")):
-    verbatim about
+    verbatim aboutHtml
     h2: text "Instance info"
     p:
       text "Version "
       a(href=link): text version
-
-proc renderFeature*(): VNode =
-  buildHtml(tdiv(class="overlay-panel")):
-    verbatim feature

src/views/embed.nim

@@ -1,18 +1,19 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import options
 import karax/[karaxdsl, vdom]
+from jester import Request
 
 import ".."/[types, formatters]
 import general, tweet
 
 const doctype = "<!DOCTYPE html>\n"
 
-proc renderVideoEmbed*(cfg: Config; tweet: Tweet): string =
+proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
   let thumb = get(tweet.video).thumb
   let vidUrl = getVideoEmbed(cfg, tweet.id)
   let prefs = Prefs(hlsPlayback: true)
   let node = buildHtml(html(lang="en")):
-    renderHead(prefs, cfg, video=vidUrl, images=(@[thumb]))
+    renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
 
     tdiv(class="embed-video"):
       renderVideo(get(tweet.video), prefs, "")

src/views/feature.nim

@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import karax/[karaxdsl, vdom]
+
+proc renderFeature*(): VNode =
+  buildHtml(tdiv(class="overlay-panel")):
+    h1: text "Unsupported feature"
+    p:
+      text "Nitter doesn't support this feature yet, but it might in the future. "
+      text "You can check for an issue and open one if needed here: "
+      a(href="https://github.com/zedeus/nitter/issues"):
+        text "https://github.com/zedeus/nitter/issues"
+    p:
+      text "To find out more about the Nitter project, see the "
+      a(href="/about"): text "About page"

src/views/general.nim

@@ -11,6 +11,9 @@ const
   doctype = "<!DOCTYPE html>\n"
   lp = readFile("public/lp.svg")
 
+proc toTheme(theme: string): string =
+  theme.toLowerAscii.replace(" ", "_")
+
 proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
   var path = req.params.getOrDefault("referer")
   if path.len == 0:
@@ -33,9 +36,13 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
         icon "info", title="About", href="/about"
         icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path))
 
-proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
-                 images: seq[string] = @[]; banner=""; ogTitle=""; theme="";
+proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
+                 video=""; images: seq[string] = @[]; banner=""; ogTitle="";
                  rss=""; canonical=""): VNode =
+  var theme = prefs.theme.toTheme
+  if "theme" in req.params:
+    theme = req.params["theme"].toTheme
+    
   let ogType =
     if video.len > 0: "video"
     elif images.len > 0: "photo"
@@ -44,7 +51,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=9")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=15")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:
@@ -117,15 +124,12 @@ proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
 proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
                  titleText=""; desc=""; ogTitle=""; rss=""; video="";
                  images: seq[string] = @[]; banner=""): string =
-  var theme = toLowerAscii(prefs.theme).replace(" ", "_")
-  if "theme" in req.params:
-    theme = toLowerAscii(req.params["theme"]).replace(" ", "_")
 
   let canonical = getTwitterLink(req.path, req.params)
 
   let node = buildHtml(html(lang="en")):
-    renderHead(prefs, cfg, titleText, desc, video, images, banner, ogTitle,
-               theme, rss, canonical)
+    renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,
+               rss, canonical)
 
     body:
       renderNavbar(cfg, req, rss, canonical)

src/views/profile.nim

@@ -5,39 +5,39 @@ import karax/[karaxdsl, vdom, vstyles]
 import renderutils, search
 import ".."/[types, utils, formatters]
 
-proc renderStat(num, class: string; text=""): VNode =
+proc renderStat(num: int; class: string; text=""): VNode =
   let t = if text.len > 0: text else: class
   buildHtml(li(class=class)):
     span(class="profile-stat-header"): text capitalizeAscii(t)
     span(class="profile-stat-num"):
-      text if num.len == 0: "?" else: insertSep(num, ',')
+      text insertSep($num, ',')
 
-proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode =
+proc renderUserCard*(user: User; prefs: Prefs): VNode =
   buildHtml(tdiv(class="profile-card")):
     tdiv(class="profile-card-info"):
       let
-        url = getPicUrl(profile.getUserPic())
+        url = getPicUrl(user.getUserPic())
         size =
-          if prefs.autoplayGifs and profile.userPic.endsWith("gif"): ""
+          if prefs.autoplayGifs and user.userPic.endsWith("gif"): ""
           else: "_400x400"
 
       a(class="profile-card-avatar", href=url, target="_blank"):
-        genImg(profile.getUserPic(size))
+        genImg(user.getUserPic(size))
 
       tdiv(class="profile-card-tabs-name"):
-        linkUser(profile, class="profile-card-fullname")
-        linkUser(profile, class="profile-card-username")
+        linkUser(user, class="profile-card-fullname")
+        linkUser(user, class="profile-card-username")
 
     tdiv(class="profile-card-extra"):
-      if profile.bio.len > 0:
+      if user.bio.len > 0:
         tdiv(class="profile-bio"):
           p(dir="auto"):
-            verbatim replaceUrls(profile.bio, prefs)
+            verbatim replaceUrls(user.bio, prefs)
 
-      if profile.location.len > 0:
+      if user.location.len > 0:
         tdiv(class="profile-location"):
           span: icon "location"
-          let (place, url) = getLocation(profile)
+          let (place, url) = getLocation(user)
           if url.len > 1:
             a(href=url): text place
           elif "://" in place:
@@ -45,29 +45,29 @@ proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode =
           else:
             span: text place
 
-      if profile.website.len > 0:
+      if user.website.len > 0:
         tdiv(class="profile-website"):
           span:
-            let url = replaceUrls(profile.website, prefs)
+            let url = replaceUrls(user.website, prefs)
             icon "link"
             a(href=url): text shortLink(url)
 
       tdiv(class="profile-joindate"):
-        span(title=getJoinDateFull(profile)):
-          icon "calendar", getJoinDate(profile)
+        span(title=getJoinDateFull(user)):
+          icon "calendar", getJoinDate(user)
 
       tdiv(class="profile-card-extra-links"):
         ul(class="profile-statlist"):
-          renderStat(profile.tweets, "posts", text="Tweets")
-          renderStat(profile.following, "following")
-          renderStat(profile.followers, "followers")
-          renderStat(profile.likes, "likes")
+          renderStat(user.tweets, "posts", text="Tweets")
+          renderStat(user.following, "following")
+          renderStat(user.followers, "followers")
+          renderStat(user.likes, "likes")
 
-proc renderPhotoRail(profile: Profile; photoRail: PhotoRail): VNode =
-  let count = insertSep($profile.media, ',')
+proc renderPhotoRail(profile: Profile): VNode =
+  let count = insertSep($profile.user.media, ',')
   buildHtml(tdiv(class="photo-rail-card")):
     tdiv(class="photo-rail-header"):
-      a(href=(&"/{profile.username}/media")):
+      a(href=(&"/{profile.user.username}/media")):
         icon "picture", count & " Photos and videos"
 
     input(id="photo-rail-grid-toggle", `type`="checkbox")
@@ -76,20 +76,19 @@ proc renderPhotoRail(profile: Profile; photoRail: PhotoRail): VNode =
       icon "down"
 
     tdiv(class="photo-rail-grid"):
-      for i, photo in photoRail:
+      for i, photo in profile.photoRail:
         if i == 16: break
-        let col = if photo.color.len > 0: photo.color else: "#161616"
-        a(href=(&"/{profile.username}/status/{photo.tweetId}#m"),
-          style={backgroundColor: col}):
+        a(href=(&"/{profile.user.username}/status/{photo.tweetId}#m")):
           genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
 
-proc renderBanner(profile: Profile): VNode =
+proc renderBanner(banner: string): VNode =
   buildHtml():
-    if "#" in profile.banner:
-      tdiv(class="profile-banner-color", style={backgroundColor: profile.banner})
+    if banner.len == 0:
+      a()
+    elif banner.startsWith('#'):
+      a(style={backgroundColor: banner})
     else:
-      a(href=getPicUrl(profile.banner), target="_blank"):
-        genImg(profile.banner)
+      a(href=getPicUrl(banner), target="_blank"): genImg(banner)
 
 proc renderProtected(username: string): VNode =
   buildHtml(tdiv(class="timeline-container")):
@@ -97,21 +96,21 @@ proc renderProtected(username: string): VNode =
       h2: text "This account's tweets are protected."
       p: text &"Only confirmed followers have access to @{username}'s tweets."
 
-proc renderProfile*(profile: Profile; timeline: var Timeline;
-                    photoRail: PhotoRail; prefs: Prefs; path: string): VNode =
-  timeline.query.fromUser = @[profile.username]
+proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
+  profile.tweets.query.fromUser = @[profile.user.username]
+
   buildHtml(tdiv(class="profile-tabs")):
     if not prefs.hideBanner:
       tdiv(class="profile-banner"):
-        renderBanner(profile)
+        renderBanner(profile.user.banner)
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
     tdiv(class=(&"profile-tab{sticky}")):
-      renderProfileCard(profile, prefs)
-      if photoRail.len > 0:
-        renderPhotoRail(profile, photoRail)
+      renderUserCard(profile.user, prefs)
+      if profile.photoRail.len > 0:
+        renderPhotoRail(profile)
 
-    if profile.protected:
-      renderProtected(profile.username)
+    if profile.user.protected:
+      renderProtected(profile.user.username)
     else:
-      renderTweetSearch(timeline, prefs, path)
+      renderTweetSearch(profile.tweets, prefs, path, profile.pinned)

src/views/renderutils.nim

@@ -15,18 +15,18 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
     if text.len > 0:
       text " " & text
 
-proc linkUser*(profile: Profile, class=""): VNode =
+proc linkUser*(user: User, class=""): VNode =
   let
     isName = "username" notin class
-    href = "/" & profile.username
-    nameText = if isName: profile.fullname
-               else: "@" & profile.username
+    href = "/" & user.username
+    nameText = if isName: user.fullname
+               else: "@" & user.username
 
   buildHtml(a(href=href, class=class, title=nameText)):
     text nameText
-    if isName and profile.verified:
+    if isName and user.verified:
       icon "ok", class="verified-icon", title="Verified account"
-    if isName and profile.protected:
+    if isName and user.protected:
       text " "
       icon "lock", title="Protected account"
 
@@ -88,3 +88,9 @@ proc getTabClass*(query: Query; tab: QueryKind): string =
   result = "tab-item"
   if query.kind == tab:
     result &= " active"
+
+proc getAvatarClass*(prefs: Prefs): string =
+  if prefs.squareAvatars:
+    "avatar"
+  else:
+    "avatar round"

src/views/rss.nimf

@@ -60,7 +60,7 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 #let urlPrefix = getUrlPrefix(cfg)
 #var links: seq[string]
 #for t in tweets:
-#  let retweet = if t.retweet.isSome: t.profile.username else: ""
+#  let retweet = if t.retweet.isSome: t.user.username else: ""
 #  let tweet = if retweet.len > 0: t.retweet.get else: t
 #  let link = getLink(tweet)
 #  if link in links: continue
@@ -68,7 +68,7 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 #  links.add link
     <item>
       <title>${getTitle(tweet, retweet)}</title>
-      <dc:creator>@${tweet.profile.username}</dc:creator>
+      <dc:creator>@${tweet.user.username}</dc:creator>
       <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
       <pubDate>${getRfc822Time(tweet)}</pubDate>
       <guid>${urlPrefix & link}</guid>
@@ -77,32 +77,32 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 #end for
 #end proc
 #
-#proc renderTimelineRss*(timeline: Timeline; profile: Profile; cfg: Config; multi=false): string =
+#proc renderTimelineRss*(profile: Profile; cfg: Config; multi=false): string =
 #let urlPrefix = getUrlPrefix(cfg)
 #result = ""
-#let user = (if multi: "" else: "@") & profile.username
-#var title = profile.fullname
-#if not multi: title &= " / " & user
+#let handle = (if multi: "" else: "@") & profile.user.username
+#var title = profile.user.fullname
+#if not multi: title &= " / " & handle
 #end if
 #title = xmltree.escape(title).sanitizeXml
 <?xml version="1.0" encoding="UTF-8"?>
 <rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
   <channel>
-    <atom:link href="${urlPrefix}/${profile.username}/rss" rel="self" type="application/rss+xml" />
+    <atom:link href="${urlPrefix}/${profile.user.username}/rss" rel="self" type="application/rss+xml" />
     <title>${title}</title>
-    <link>${urlPrefix}/${profile.username}</link>
-    <description>${getDescription(user, cfg)}</description>
+    <link>${urlPrefix}/${profile.user.username}</link>
+    <description>${getDescription(handle, cfg)}</description>
     <language>en-us</language>
     <ttl>40</ttl>
     <image>
       <title>${title}</title>
-      <link>${urlPrefix}/${profile.username}</link>
-      <url>${urlPrefix}${getPicUrl(profile.getUserPic(style="_400x400"))}</url>
+      <link>${urlPrefix}/${profile.user.username}</link>
+      <url>${urlPrefix}${getPicUrl(profile.user.getUserPic(style="_400x400"))}</url>
       <width>128</width>
       <height>128</height>
     </image>
-#if timeline.content.len > 0:
-${renderRssTweets(timeline.content, cfg)}
+#if profile.tweets.content.len > 0:
+${renderRssTweets(profile.tweets.content, cfg)}
 #end if
   </channel>
 </rss>

src/views/search.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, sequtils, unicode, tables
+import strutils, strformat, sequtils, unicode, tables, options
 import karax/[karaxdsl, vdom]
 
 import renderutils, timeline
@@ -88,7 +88,8 @@ proc renderSearchPanel*(query: Query): VNode =
           span(class="search-title"): text "Near"
           genInput("near", "", query.near, placeholder="Location...")
 
-proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string): VNode =
+proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
+                        pinned=none(Tweet)): VNode =
   let query = results.query
   buildHtml(tdiv(class="timeline-container")):
     if query.fromUser.len > 1:
@@ -105,9 +106,9 @@ proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string): VNo
     if query.fromUser.len == 0:
       renderSearchTabs(query)
 
-    renderTimelineTweets(results, prefs, path)
+    renderTimelineTweets(results, prefs, path, pinned)
 
-proc renderUserSearch*(results: Result[Profile]; prefs: Prefs): VNode =
+proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode =
   buildHtml(tdiv(class="timeline-container")):
     tdiv(class="timeline-header"):
       form(`method`="get", action="/search", class="search-field"):

src/views/status.nim

@@ -10,24 +10,22 @@ proc renderEarlier(thread: Chain): VNode =
       text "earlier replies"
 
 proc renderMoreReplies(thread: Chain): VNode =
-  let num = if thread.more != -1: $thread.more & " " else: ""
-  let reply = if thread.more == 1: "reply" else: "replies"
   let link = getLink(thread.content[^1])
   buildHtml(tdiv(class="timeline-item more-replies")):
     if thread.content[^1].available:
       a(class="more-replies-text", href=link):
-        text $num & "more " & reply
+        text "more replies"
     else:
       a(class="more-replies-text"):
-        text $num & "more " & reply
+        text "more replies"
 
 proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode =
   buildHtml(tdiv(class="reply thread thread-line")):
     for i, tweet in thread.content:
-      let last = (i == thread.content.high and thread.more == 0)
+      let last = (i == thread.content.high and not thread.hasMore)
       renderTweet(tweet, prefs, path, index=i, last=last)
 
-    if thread.more != 0:
+    if thread.hasMore:
       renderMoreReplies(thread)
 
 proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode =
@@ -60,12 +58,12 @@ proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode
         tdiv(class="after-tweet thread-line"):
           let
             total = conv.after.content.high
-            more = conv.after.more
+            hasMore = conv.after.hasMore
           for i, tweet in conv.after.content:
             renderTweet(tweet, prefs, path, index=i,
-                        last=(i == total and more == 0), afterTweet=true)
+                        last=(i == total and not hasMore), afterTweet=true)
 
-          if more != 0:
+          if hasMore:
             renderMoreReplies(conv.after)
 
     if not prefs.hideReplies:

src/views/timeline.nim

@@ -57,13 +57,13 @@ proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet
     elif t.replyId == result[0].id:
       result.add t
 
-proc renderUser(user: Profile; prefs: Prefs): VNode =
+proc renderUser(user: User; prefs: Prefs): VNode =
   buildHtml(tdiv(class="timeline-item")):
     a(class="tweet-link", href=("/" & user.username))
     tdiv(class="tweet-body profile-result"):
       tdiv(class="tweet-header"):
         a(class="tweet-avatar", href=("/" & user.username)):
-          genImg(user.getUserPic("_bigger"), class="avatar")
+          genImg(user.getUserPic("_bigger"), class=prefs.getAvatarClass)
 
         tdiv(class="tweet-name-row"):
           tdiv(class="fullname-and-username"):
@@ -73,7 +73,7 @@ proc renderUser(user: Profile; prefs: Prefs): VNode =
       tdiv(class="tweet-content media-body", dir="auto"):
         verbatim replaceUrls(user.bio, prefs)
 
-proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNode =
+proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode =
   buildHtml(tdiv(class="timeline")):
     if not results.beginning:
       renderNewer(results.query, path)
@@ -89,11 +89,16 @@ proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNod
     else:
       renderNoMore()
 
-proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string): VNode =
+proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
+                           pinned=none(Tweet)): VNode =
   buildHtml(tdiv(class="timeline")):
     if not results.beginning:
       renderNewer(results.query, parseUri(path).path)
 
+    if pinned.isSome:
+      let tweet = get pinned
+      renderTweet(tweet, prefs, path, showThread=tweet.hasThread)
+
     if results.content.len == 0:
       if not results.beginning:
         renderNoMore()

src/views/tweet.nim

@@ -1,9 +1,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import strutils, sequtils, strformat, options
 import karax/[karaxdsl, vdom, vstyles]
+from jester import Request
 
 import renderutils
 import ".."/[types, utils, formatters]
+import general
 
 proc getSmallPic(url: string): string =
   result = url
@@ -11,10 +13,10 @@ proc getSmallPic(url: string): string =
     result &= ":small"
   result = getPicUrl(result)
 
-proc renderMiniAvatar(profile: Profile): VNode =
-  let url = getPicUrl(profile.getUserPic("_mini"))
+proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
+  let url = getPicUrl(user.getUserPic("_mini"))
   buildHtml():
-    img(class="avatar mini", src=url)
+    img(class=(prefs.getAvatarClass & " mini"), src=url)
 
 proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode =
   buildHtml(tdiv):
@@ -27,16 +29,16 @@ proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode =
         span: icon "pin", "Pinned Tweet"
 
     tdiv(class="tweet-header"):
-      a(class="tweet-avatar", href=("/" & tweet.profile.username)):
+      a(class="tweet-avatar", href=("/" & tweet.user.username)):
         var size = "_bigger"
-        if not prefs.autoplayGifs and tweet.profile.userPic.endsWith("gif"):
+        if not prefs.autoplayGifs and tweet.user.userPic.endsWith("gif"):
           size = "_400x400"
-        genImg(tweet.profile.getUserPic(size), class="avatar")
+        genImg(tweet.user.getUserPic(size), class=prefs.getAvatarClass)
 
       tdiv(class="tweet-name-row"):
         tdiv(class="fullname-and-username"):
-          linkUser(tweet.profile, class="fullname")
-          linkUser(tweet.profile, class="username")
+          linkUser(tweet.user, class="fullname")
+          linkUser(tweet.user, class="username")
 
         span(class="tweet-date"):
           a(href=getLink(tweet), title=tweet.getTime):
@@ -97,7 +99,7 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
           img(src=thumb)
           renderVideoDisabled(video, path)
         else:
-          let vid = video.variants.filterIt(it.videoType == video.playbackType)
+          let vid = video.variants.filterIt(it.contentType == video.playbackType)
           let source = getVidUrl(vid[0].url)
           case video.playbackType
           of mp4:
@@ -201,14 +203,14 @@ proc renderReply(tweet: Tweet): VNode =
       if i > 0: text " "
       a(href=("/" & u)): text "@" & u
 
-proc renderAttribution(profile: Profile): VNode =
-  buildHtml(a(class="attribution", href=("/" & profile.username))):
-    renderMiniAvatar(profile)
-    strong: text profile.fullname
-    if profile.verified:
+proc renderAttribution(user: User; prefs: Prefs): VNode =
+  buildHtml(a(class="attribution", href=("/" & user.username))):
+    renderMiniAvatar(user, prefs)
+    strong: text user.fullname
+    if user.verified:
       icon "ok", class="verified-icon", title="Verified account"
 
-proc renderMediaTags(tags: seq[Profile]): VNode =
+proc renderMediaTags(tags: seq[User]): VNode =
   buildHtml(tdiv(class="media-tag-block")):
     icon "user"
     for i, p in tags:
@@ -242,9 +244,9 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
 
     tdiv(class="tweet-name-row"):
       tdiv(class="fullname-and-username"):
-        renderMiniAvatar(quote.profile)
-        linkUser(quote.profile, class="fullname")
-        linkUser(quote.profile, class="username")
+        renderMiniAvatar(quote.user, prefs)
+        linkUser(quote.user, class="fullname")
+        linkUser(quote.user, class="username")
 
       span(class="tweet-date"):
         a(href=getLink(quote), title=quote.getTime):
@@ -299,7 +301,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
   var tweet = fullTweet
   if tweet.retweet.isSome:
     tweet = tweet.retweet.get
-    retweet = fullTweet.profile.fullname
+    retweet = fullTweet.user.fullname
 
   buildHtml(tdiv(class=("timeline-item " & divClass))):
     if not mainTweet:
@@ -310,7 +312,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       renderHeader(tweet, retweet, prefs)
 
       if not afterTweet and index == 0 and tweet.reply.len > 0 and
-         (tweet.reply.len > 1 or tweet.reply[0] != tweet.profile.username):
+         (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username):
         renderReply(tweet)
 
       var tweetClass = "tweet-content media-body"
@@ -321,7 +323,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet)
 
       if tweet.attribution.isSome:
-        renderAttribution(tweet.attribution.get())
+        renderAttribution(tweet.attribution.get(), prefs)
 
       if tweet.card.isSome:
         renderCard(tweet.card.get(), prefs, path)
@@ -353,3 +355,8 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if showThread:
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
           text "Show this thread"
+
+proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
+  buildHtml(tdiv(class="tweet-embed")):
+    renderHead(prefs, cfg, req)
+    renderTweet(tweet, prefs, path, mainTweet=true)

tests/test_profile.py

@@ -18,9 +18,8 @@ protected = [
 invalid = [['thisprofiledoesntexist'], ['%']]
 
 banner_color = [
-    ['TheTwoffice', '29, 161, 242'],
-    ['profiletest', '80, 176, 58'],
-    ['nim_lang', '24, 26, 36']
+    ['nim_lang', '22, 25, 32'],
+    ['rustlang', '35, 31, 32']
 ]
 
 banner_image = [
@@ -78,7 +77,7 @@ class ProfileTest(BaseTestCase):
     @parameterized.expand(banner_color)
     def test_banner_color(self, username, color):
         self.open_nitter(username)
-        banner = self.find_element(Profile.banner + '-color')
+        banner = self.find_element(Profile.banner + ' a')
         self.assertIn(color, banner.value_of_css_property('background-color'))
 
     @parameterized.expand(banner_image)

tools/rendermd.nim

@@ -0,0 +1,10 @@
+import std/[os, strutils]
+import markdown
+
+for file in walkFiles("public/md/*.md"):
+  let
+    html = markdown(readFile(file))
+    output = file.replace(".md", ".html")
+
+  output.writeFile(html)
+  echo "Rendered ", output