Merge branch 'master' into feature-notes

2023-04-09 11:04:12 +02:00 · 2023-04-09 11:04:12 +02:00 · 7f3426ce21
parent b24a5da469 95893eedaa
commit 7f3426ce21
41 changed files with 454 additions and 234 deletions
--- a/.github/workflows/build-docker.yml
+++ b/.github/workflows/build-docker.yml
@ -1,4 +1,4 @@
-name: CI/CD
+name: Docker

 on:
  push:
@ -8,31 +8,51 @@ on:
      - master

 jobs:
-  build-docker:
-    runs-on: ubuntu-latest
+  build-docker-amd64:
+    runs-on: buildjet-2vcpu-ubuntu-2204
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
      - name: Set up Docker Buildx
        id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
        with:
          version: latest
      - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
+      - name: Build and push AMD64 Docker image
+        uses: docker/build-push-action@v3
        with:
          context: .
          file: ./Dockerfile
          platforms: linux/amd64
          push: true
          tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
+  build-docker-arm64:
+    runs-on: buildjet-2vcpu-ubuntu-2204-arm
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push ARM64 Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./Dockerfile.arm64
+          platforms: linux/arm64
+          push: true
+          tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@ -0,0 +1,42 @@
+name: Run tests
+
+on:
+  push:
+    paths-ignore:
+      - "*.md"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Cache nimble
+        id: cache-nimble
+        uses: actions/cache@v3
+        with:
+          path: ~/.nimble
+          key: nimble-${{ hashFiles('*.nimble') }}
+          restore-keys: "nimble-"
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - uses: jiro4989/setup-nim-action@v1
+        with:
+          nim-version: "1.x"
+      - run: nimble build -d:release -Y
+      - run: pip install seleniumbase
+      - run: seleniumbase install chromedriver
+      - uses: supercharge/redis-github-action@1.5.0
+      - name: Prepare Nitter
+        run: |
+          sudo apt install libsass-dev -y
+          cp nitter.example.conf nitter.conf
+          nimble md
+          nimble scss
+      - name: Run tests
+        run: |
+          ./nitter &
+          pytest -n4 tests
--- a/4
+++ b/4
@ -1,4 +1,4 @@
-FROM nimlang/nim:1.6.2-alpine-regular as nim
+FROM nimlang/nim:1.6.10-alpine-regular as nim
 LABEL maintainer="setenforce@protonmail.com"

 RUN apk --no-cache add libsass-dev pcre
@ -20,4 +20,6 @@ COPY --from=nim /src/nitter/nitter ./
 COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
 COPY --from=nim /src/nitter/public ./public
 EXPOSE 8080
+RUN adduser -h /src/ -D -s /bin/sh nitter
+USER nitter
 CMD ./nitter
--- a/Dockerfile.arm64
+++ b/Dockerfile.arm64
@ -0,0 +1,23 @@
+FROM alpine:3.17 as nim
+LABEL maintainer="setenforce@protonmail.com"
+
+RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.8-r0" nimble pcre
+
+WORKDIR /src/nitter
+
+COPY nitter.nimble .
+RUN nimble install -y --depsOnly
+
+COPY . .
+RUN nimble build -d:danger -d:lto -d:strip \
+    && nimble scss \
+    && nimble md
+
+FROM alpine:3.17
+WORKDIR /src/
+RUN apk --no-cache add ca-certificates pcre openssl1.1-compat
+COPY --from=nim /src/nitter/nitter ./
+COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
+COPY --from=nim /src/nitter/public ./public
+EXPOSE 8080
+CMD ./nitter
--- a/README.md
+++ b/README.md
@ -109,7 +109,9 @@ performance reasons.

 ### Docker

-#### NOTE: For ARM64/ARM support, please use [unixfox's image](https://quay.io/repository/unixfox/nitter?tab=tags), more info [here](https://github.com/zedeus/nitter/issues/399#issuecomment-997263495)
+Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
+
+#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).

 To run Nitter with Docker, you'll need to install and run Redis separately
 before you can run the container. See below for how to also run Redis using
@ -122,6 +124,8 @@ docker build -t nitter:latest .
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
 ```

+Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
+
 A prebuilt Docker image is provided as well:

 ```bash
--- a/config.nims
+++ b/config.nims
@ -1,5 +1,6 @@
 --define:ssl
 --define:useStdLib
+--threads:off

 # workaround httpbeast file upload bug
 --assertions:off
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -8,7 +8,7 @@ services:
    ports:
      - "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
    volumes:
-      - ./nitter.conf:/src/nitter.conf:ro
+      - ./nitter.conf:/src/nitter.conf:Z,ro
    depends_on:
      - nitter-redis
    restart: unless-stopped
@ -17,6 +17,12 @@ services:
      interval: 30s
      timeout: 5s
      retries: 2
+    user: "998:998"
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL

  nitter-redis:
    image: redis:6-alpine
@ -30,6 +36,12 @@ services:
      interval: 30s
      timeout: 5s
      retries: 2
+    user: "999:1000"
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL

 volumes:
  nitter-redis:
--- a/nitter.example.conf
+++ b/nitter.example.conf
@ -37,9 +37,8 @@ tokenCount = 10
 [Preferences]
 theme = "Nitter"
 replaceTwitter = "nitter.net"
-replaceYouTube = "piped.kavin.rocks"
+replaceYouTube = "piped.video"
 replaceReddit = "teddit.net"
-replaceInstagram = ""
 proxyVideos = true
 hlsPlayback = false
 infiniteScroll = false
--- a/nitter.nimble
+++ b/nitter.nimble
@ -11,18 +11,18 @@ bin           = @["nitter"]
 # Dependencies

 requires "nim >= 1.4.8"
-requires "jester >= 0.5.0"
-requires "karax#5498909"
-requires "sass#e683aa1"
-requires "nimcrypto#a5742a9"
-requires "markdown#a661c26"
-requires "packedjson#d11d167"
-requires "supersnappy#2.1.1"
+requires "jester#baca3f"
+requires "karax#9ee695b"
+requires "sass#7dfdd03"
+requires "nimcrypto#4014ef9"
+requires "markdown#158efe3"
+requires "packedjson#9e6fbb6"
+requires "supersnappy#6c94198"
 requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
-requires "zippy#0.9.11"
-requires "flatty#0.2.3"
-requires "jsony#d0e69bd"
+requires "zippy#ca5989a"
+requires "flatty#e668085"
+requires "jsony#ea811be"


 # Tasks
--- a/public/js/hls.light.min.js
+++ b/public/js/hls.light.min.js
--- a/public/robots.txt
+++ b/public/robots.txt
@ -1,4 +1,5 @@
 User-agent: *
 Disallow: /
+Crawl-delay: 1
 User-agent: Twitterbot
 Disallow:
--- a/src/api.nim
+++ b/src/api.nim
@ -4,11 +4,22 @@ import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser

-proc getGraphUser*(id: string): Future[User] {.async.} =
+proc getGraphUser*(username: string): Future[User] {.async.} =
+  if username.len == 0: return
+  let
+    variables = """{
+      "screen_name": "$1",
+      "withSafetyModeUserFields": false,
+      "withSuperFollowsUserFields": false
+    }""" % [username]
+    js = await fetchRaw(graphUser ? {"variables": variables}, Api.userScreenName)
+  result = parseGraphUser(js)
+
+proc getGraphUserById*(id: string): Future[User] {.async.} =
  if id.len == 0 or id.any(c => not c.isDigit): return
  let
-    variables = %*{"userId": id, "withSuperFollowsUserFields": true}
-    js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
+    variables = """{"userId": "$1", "withSuperFollowsUserFields": true}""" % [id]
+    js = await fetchRaw(graphUserById ? {"variables": variables}, Api.userRestId)
  result = parseGraphUser(js)

 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
@ -53,20 +64,6 @@ proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
    url = listTimeline ? ps
  result = parseTimeline(await fetch(url, Api.timeline), after)

-proc getUser*(username: string): Future[User] {.async.} =
-  if username.len == 0: return
-  let
-    ps = genParams({"screen_name": username})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json, username)
-
-proc getUserById*(userId: string): Future[User] {.async.} =
-  if userId.len == 0: return
-  let
-    ps = genParams({"user_id": userId})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json)
-
 proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
  if id.len == 0: return
  let
@ -110,16 +107,21 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
  except InternalError:
    return Result[T](beginning: true, query: query)

-proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
-  let url = tweet / (id & ".json") ? genParams(cursor=after)
-  result = parseConversation(await fetch(url, Api.tweet), id)
+proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = tweetVariables % [id, cursor]
+    params = {"variables": variables, "features": tweetFeatures}
+    js = await fetch(graphTweet ? params, Api.tweetDetail)
+  result = parseGraphConversation(js, id)

 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
-  result = (await getTweetImpl(id, after)).replies
+  result = (await getGraphTweet(id, after)).replies
  result.beginning = after.len == 0

 proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
-  result = await getTweetImpl(id)
+  result = await getGraphTweet(id)
  if after.len > 0:
    result.replies = await getReplies(id, after)

--- a/src/apiutils.nim
+++ b/src/apiutils.nim
@ -23,7 +23,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
    result &= ("count", count)
  if cursor.len > 0:
    # The raw cursor often has plus signs, which sometimes get turned into spaces,
-    # so we need to them back into a plus
+    # so we need to turn them back into a plus
    if " " in cursor:
      result &= ("cursor", cursor.replace(" ", "+"))
    else:
@ -61,12 +61,20 @@ template fetchImpl(result, fetchBody) {.dirty.} =
  try:
    var resp: AsyncResponse
    pool.use(genHeaders(token)):
-      resp = await c.get($url)
-      result = await resp.body
+      template getContent =
+        resp = await c.get($url)
+        result = await resp.body

-      if resp.status == $Http503:
-        badClient = true
-        raise newException(InternalError, result)
+      getContent()
+
+      # Twitter randomly returns 401 errors with an empty body quite often.
+      # Retrying the request usually works.
+      if resp.status == "401 Unauthorized" and result.len == 0:
+        getContent()
+
+    if resp.status == $Http503:
+      badClient = true
+      raise newException(InternalError, result)

    if result.len > 0:
      if resp.headers.getOrDefault("content-encoding") == "gzip":
--- a/src/consts.nim
+++ b/src/consts.nim
@ -19,7 +19,9 @@ const
  tweet* = timelineApi / "conversation"

  graphql = api / "graphql"
-  graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
+  graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
+  graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
+  graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
  graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
@ -58,3 +60,34 @@ const
  ## user:   "result_filter: user"
  ## photos: "result_filter: photos"
  ## videos: "result_filter: videos"
+
+  tweetVariables* = """{
+  "focalTweetId": "$1",
+  $2
+  "includePromotedContent": false,
+  "withBirdwatchNotes": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withSuperFollowsTweetFields": false,
+  "withSuperFollowsUserFields": false,
+  "withVoice": false,
+  "withV2Timeline": true
+}"""
+
+  tweetFeatures* = """{
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": false,
+  "standardized_nudges_misinfo": false,
+  "verified_phone_label_enabled": false,
+  "responsive_web_twitter_blue_verified_badge_is_enabled": false,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+  "view_counts_everywhere_api_enabled": false,
+  "responsive_web_edit_tweet_api_enabled": false,
+  "tweetypie_unmention_optimization_enabled": false,
+  "vibe_api_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
+  "responsive_web_text_conversations_enabled": false,
+  "responsive_web_enhance_cards_enabled": false,
+  "interactive_text_enabled": false
+}"""
--- a/src/experimental/parser/graphql.nim
+++ b/src/experimental/parser/graphql.nim
@ -1,9 +1,14 @@
+import options
 import jsony
 import user, ../types/[graphuser, graphlistmembers]
 from ../../types import User, Result, Query, QueryKind

 proc parseGraphUser*(json: string): User =
  let raw = json.fromJson(GraphUser)
+
+  if raw.data.user.result.reason.get("") == "Suspended":
+    return User(suspended: true)
+
  result = toUser raw.data.user.result.legacy
  result.id = raw.data.user.result.restId

--- a/src/experimental/parser/unifiedcard.nim
+++ b/src/experimental/parser/unifiedcard.nim
@ -66,6 +66,8 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
      durationMs: videoInfo.durationMillis,
      variants: videoInfo.variants
    )
+  of model3d:
+    result.title = "Unsupported 3D model ad"

 proc parseUnifiedCard*(json: string): Card =
  let card = json.fromJson(UnifiedCard)
@ -82,6 +84,8 @@ proc parseUnifiedCard*(json: string): Card =
      component.parseMedia(card, result)
    of buttonGroup:
      discard
+    of ComponentType.unknown:
+      echo "ERROR: Unknown component type: ", json

    case component.kind
    of twitterListDetails:
--- a/src/experimental/types/graphuser.nim
+++ b/src/experimental/types/graphuser.nim
@ -1,3 +1,4 @@
+import options
 import user

 type
@ -10,3 +11,4 @@ type
  UserResult = object
    legacy*: RawUser
    restId*: string
+    reason*: Option[string]
--- a/src/experimental/types/unifiedcard.nim
+++ b/src/experimental/types/unifiedcard.nim
@ -17,6 +17,7 @@ type
    twitterListDetails
    communityDetails
    mediaWithDetailsHorizontal
+    unknown

  Component* = object
    kind*: ComponentType
@ -47,7 +48,7 @@ type
    vanity*: string

  MediaType* = enum
-    photo, video
+    photo, video, model3d

  MediaEntity* = object
    kind*: MediaType
@ -77,3 +78,29 @@ converter fromText*(text: Text): string = text.content
 proc renameHook*(v: var HasTypeField; fieldName: var string) =
  if fieldName == "type":
    fieldName = "kind"
+
+proc enumHook*(s: string; v: var ComponentType) =
+  v = case s
+      of "details": details
+      of "media": media
+      of "swipeable_media": swipeableMedia
+      of "button_group": buttonGroup
+      of "app_store_details": appStoreDetails
+      of "twitter_list_details": twitterListDetails
+      of "community_details": communityDetails
+      of "media_with_details_horizontal": mediaWithDetailsHorizontal
+      else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
+
+proc enumHook*(s: string; v: var AppType) =
+  v = case s
+      of "android_app": androidApp
+      of "iphone_app": iPhoneApp
+      of "ipad_app": iPadApp
+      else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
+
+proc enumHook*(s: string; v: var MediaType) =
+  v = case s
+      of "video": video
+      of "photo": photo
+      of "model3d": model3d
+      else: echo "ERROR: Unknown enum value (MediaType): ", s; photo
--- a/src/formatters.nim
+++ b/src/formatters.nim
@ -12,8 +12,7 @@ let
  twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
  twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""

-  ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
-  igRegex = re"(www\.)?instagram\.com"
+  ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})

  rdRegex = re"(?<![.b])((www|np|new|amp|old)\.)?reddit.com"
  rdShortRegex = re"(?<![.b])redd\.it\/"
@ -58,15 +57,13 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =

  if prefs.replaceYouTube.len > 0 and "youtu" in result:
    result = result.replace(ytRegex, prefs.replaceYouTube)
-    if prefs.replaceYouTube in result:
-      result = result.replace("/c/", "/")

  if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
-    result = result.replace(tco, &"{https}{prefs.replaceTwitter}/t.co")
+    result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
    result = result.replace(cards, prefs.replaceTwitter & "/cards")
    result = result.replace(twRegex, prefs.replaceTwitter)
    result = result.replacef(twLinkRegex, a(
-      prefs.replaceTwitter & "$2", href = &"{https}{prefs.replaceTwitter}$1"))
+      prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))

  if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
    result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
@ -74,9 +71,6 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
    if prefs.replaceReddit in result and "/gallery/" in result:
      result = result.replace("/gallery/", "/comments/")

-  if prefs.replaceInstagram.len > 0 and "instagram.com" in result:
-    result = result.replace(igRegex, prefs.replaceInstagram)
-
  if absolute.len > 0 and "href" in result:
    result = result.replace("href=\"/", &"href=\"{absolute}/")

--- a/src/nitter.nim
+++ b/src/nitter.nim
@ -57,6 +57,7 @@ settings:
  port = Port(cfg.port)
  staticDir = cfg.staticDir
  bindAddr = cfg.address
+  reusePort = true

 routes:
  get "/":
--- a/src/parser.nim
+++ b/src/parser.nim
@ -45,7 +45,6 @@ proc parseGraphList*(js: JsonNode): List =
    banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
  )

-
 proc parsePoll(js: JsonNode): Poll =
  let vals = js{"binding_values"}
  # name format is pollNchoice_*
@ -73,8 +72,8 @@ proc parseGif(js: JsonNode): Gif =
 proc parseVideo(js: JsonNode): Video =
  result = Video(
    thumb: js{"media_url_https"}.getImageStr,
-    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
-    available: js{"ext_media_availability", "status"}.getStr == "available",
+    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt),
+    available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available",
    title: js{"ext_alt_text"}.getStr,
    durationMs: js{"video_info", "duration_millis"}.getInt
    # playbackType: mp4
@ -186,7 +185,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
     result.url.len == 0 or result.url.startsWith("card://"):
    result.url = getPicUrl(result.image)

-proc parseTweet(js: JsonNode): Tweet =
+proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
  if js.isNull: return
  result = Tweet(
    id: js{"id_str"}.getId,
@ -194,7 +193,6 @@ proc parseTweet(js: JsonNode): Tweet =
    replyId: js{"in_reply_to_status_id_str"}.getId,
    text: js{"full_text"}.getStr,
    time: js{"created_at"}.getTime,
-    source: getSource(js),
    hasThread: js{"self_thread"}.notNull,
    available: true,
    user: User(id: js{"user_id_str"}.getStr),
@ -208,6 +206,10 @@ proc parseTweet(js: JsonNode): Tweet =

  result.expandTweetEntities(js)

+  # fix for pinned threads
+  if result.hasThread and result.threadId == 0:
+    result.threadId = js{"self_thread", "id_str"}.getId
+
  if js{"is_quote_status"}.getBool:
    result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)

@ -215,7 +217,7 @@ proc parseTweet(js: JsonNode): Tweet =
    result.retweet = some Tweet(id: rt.getId)
    return

-  with jsCard, js{"card"}:
+  if jsCard.kind != JNull:
    let name = jsCard{"name"}.getStr
    if "poll" in name:
      if "image" in name:
@ -292,64 +294,18 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
    result.users[k] = parseUser(v, k)

  for k, v in tweets:
-    var tweet = parseTweet(v)
+    var tweet = parseTweet(v, v{"card"})
    if tweet.user.id in result.users:
      tweet.user = result.users[tweet.user.id]
    result.tweets[k] = tweet

-proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
-  result.thread = Chain()
-
-  let thread = js{"content", "item", "content", "conversationThread"}
-  with cursor, thread{"showMoreCursor"}:
-    result.thread.cursor = cursor{"value"}.getStr
-    result.thread.hasMore = true
-
-  for t in thread{"conversationComponents"}:
-    let content = t{"conversationTweetComponent", "tweet"}
-
-    if content{"displayType"}.getStr == "SelfThread":
-      result.self = true
-
-    var tweet = finalizeTweet(global, content{"id"}.getStr)
-    if not tweet.available:
-      tweet.tombstone = getTombstone(content{"tombstone"})
-    result.thread.content.add tweet
-
-proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
-  result = Conversation(replies: Result[Chain](beginning: true))
-  let global = parseGlobalObjects(? js)
-
-  let instructions = ? js{"timeline", "instructions"}
-  if instructions.len == 0:
-    return
-
-  for e in instructions[0]{"addEntries", "entries"}:
-    let entry = e{"entryId"}.getStr
-    if "tweet" in entry or "tombstone" in entry:
-      let tweet = finalizeTweet(global, e.getEntryId)
-      if $tweet.id != tweetId:
-        result.before.content.add tweet
-      else:
-        result.tweet = tweet
-    elif "conversationThread" in entry:
-      let (thread, self) = parseThread(e, global)
-      if thread.content.len > 0:
-        if self:
-          result.after = thread
-        else:
-          result.replies.content.add thread
-    elif "cursor-showMore" in entry:
-      result.replies.bottom = e.getCursor
-    elif "cursor-bottom" in entry:
-      result.replies.bottom = e.getCursor
-
 proc parseStatus*(js: JsonNode): Tweet =
  with e, js{"errors"}:
-    if e.getError == tweetNotFound:
+    if e.getError in {tweetNotFound, tweetUnavailable, tweetCensored, doesntExist,
+                      tweetNotAuthorized, suspended}:
      return

-  result = parseTweet(js)
+  result = parseTweet(js, js{"card"})
  if not result.isNil:
    result.user = parseUser(js{"user"})

@ -406,7 +362,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
 proc parsePhotoRail*(js: JsonNode): PhotoRail =
  for tweet in js:
    let
-      t = parseTweet(tweet)
+      t = parseTweet(tweet, js{"card"})
      url = if t.photos.len > 0: t.photos[0]
            elif t.video.isSome: get(t.video).thumb
            elif t.gif.isSome: get(t.gif).thumb
@ -416,6 +372,70 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =
    if url.len == 0: continue
    result.add GalleryPhoto(url: url, tweetId: $t.id)

+proc parseGraphTweet(js: JsonNode): Tweet =
+  if js.kind == JNull or js{"__typename"}.getStr == "TweetUnavailable":
+    return Tweet(available: false)
+
+  var jsCard = copy(js{"card", "legacy"})
+  if jsCard.kind != JNull:
+    var values = newJObject()
+    for val in jsCard["binding_values"]:
+      values[val["key"].getStr] = val["value"]
+    jsCard["binding_values"] = values
+
+  result = parseTweet(js{"legacy"}, jsCard)
+  result.user = parseUser(js{"core", "user_results", "result", "legacy"})
+
+  with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
+    result.expandNoteTweetEntities(noteTweet)
+
+  if result.quote.isSome:
+    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
+
+proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
+  let thread = js{"content", "items"}
+  for t in js{"content", "items"}:
+    let entryId = t{"entryId"}.getStr
+    if "cursor-showmore" in entryId:
+      let cursor = t{"item", "itemContent", "value"}
+      result.thread.cursor = cursor.getStr
+      result.thread.hasMore = true
+    elif "tweet" in entryId:
+      let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"})
+      result.thread.content.add tweet
+
+      if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
+        result.self = true
+
+proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
+  result = Conversation(replies: Result[Chain](beginning: true))
+
+  let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for e in instructions[0]{"entries"}:
+    let entryId = e{"entryId"}.getStr
+    # echo entryId
+    if entryId.startsWith("tweet"):
+      let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
+
+      if not tweet.available:
+        tweet.id = parseBiggestInt(entryId.getId())
+
+      if $tweet.id == tweetId:
+        result.tweet = tweet
+      else:
+        result.before.content.add tweet
+    elif entryId.startsWith("conversationthread"):
+      let (thread, self) = parseGraphThread(e)
+      if self:
+        result.after = thread
+      else:
+        result.replies.content.add thread
+    elif entryId.startsWith("cursor-bottom"):
+      result.replies.bottom = e{"content", "itemContent", "value"}.getStr
+
 proc parseGraphArticle*(js: JsonNode): Article =
  if not js{"errors"}.isNull:
    return
--- a/src/parserutils.nim
+++ b/src/parserutils.nim
@ -28,13 +28,13 @@ template `?`*(js: JsonNode): untyped =
  if j.isNull: return
  j

-template `with`*(ident, value, body): untyped =
-  block:
+template with*(ident, value, body): untyped =
+  if true:
    let ident {.inject.} = value
    if ident != nil: body

-template `with`*(ident; value: JsonNode; body): untyped =
-  block:
+template with*(ident; value: JsonNode; body): untyped =
+  if true:
    let ident {.inject.} = value
    if value.notNull: body

@ -133,10 +133,6 @@ proc getTombstone*(js: JsonNode): string =
  result = js{"tombstoneInfo", "richText", "text"}.getStr
  result.removeSuffix(" Learn more")

-proc getSource*(js: JsonNode): string =
-  let src = js{"source"}.getStr
-  result = src.substr(src.find('>') + 1, src.rfind('<') - 1)
-
 proc getMp4Resolution*(url: string): int =
  # parses the height out of a URL like this one:
  # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
@ -234,47 +230,37 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
  user.bio = user.bio.replacef(unRegex, unReplace)
                     .replacef(htRegex, htReplace)

-proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
-  let
-    orig = tweet.text.toRunes
-    textRange = js{"display_text_range"}
-    textSlice = textRange{0}.getInt .. textRange{1}.getInt
-    hasQuote = js{"is_quote_status"}.getBool
-    hasCard = tweet.card.isSome
-
-  var replyTo = ""
-  if tweet.replyId != 0:
-    with reply, js{"in_reply_to_screen_name"}:
-      tweet.reply.add reply.getStr
-      replyTo = reply.getStr
-
-  let ent = ? js{"entities"}
+proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
+                        replyTo=""; hasQuote=false) =
+  let hasCard = tweet.card.isSome

  var replacements = newSeq[ReplaceSlice]()

-  with urls, ent{"urls"}:
+  with urls, entities{"urls"}:
    for u in urls:
      let urlStr = u["url"].getStr
-      if urlStr.len == 0 or urlStr notin tweet.text:
+      if urlStr.len == 0 or urlStr notin text:
        continue
+
      replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
+
      if hasCard and u{"url"}.getStr == get(tweet.card).url:
        get(tweet.card).url = u{"expanded_url"}.getStr

-  with media, ent{"media"}:
+  with media, entities{"media"}:
    for m in media:
      replacements.extractUrls(m, textSlice.b, hideTwitter = true)

-  if "hashtags" in ent:
-    for hashtag in ent["hashtags"]:
+  if "hashtags" in entities:
+    for hashtag in entities["hashtags"]:
      replacements.extractHashtags(hashtag)

-  if "symbols" in ent:
-    for symbol in ent["symbols"]:
+  if "symbols" in entities:
+    for symbol in entities["symbols"]:
      replacements.extractHashtags(symbol)

-  if "user_mentions" in ent:
-    for mention in ent["user_mentions"]:
+  if "user_mentions" in entities:
+    for mention in entities["user_mentions"]:
      let
        name = mention{"screen_name"}.getStr
        slice = mention.extractSlice
@ -291,5 +277,27 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
  replacements.deduplicate
  replacements.sort(cmp)

-  tweet.text = orig.replacedWith(replacements, textSlice)
-                   .strip(leading=false)
+  tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
+
+proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entities"}
+    hasQuote = js{"is_quote_status"}.getBool
+    textRange = js{"display_text_range"}
+    textSlice = textRange{0}.getInt .. textRange{1}.getInt
+
+  var replyTo = ""
+  if tweet.replyId != 0:
+    with reply, js{"in_reply_to_screen_name"}:
+      replyTo = reply.getStr
+      tweet.reply.add replyTo
+
+  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote)
+
+proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entity_set"}
+    text = js{"text"}.getStr
+    textSlice = 0..text.runeLen
+
+  tweet.expandTextEntities(entities, text, textSlice)
--- a/src/prefs_impl.nim
+++ b/src/prefs_impl.nim
@ -83,7 +83,7 @@ genPrefs:
      "Enable mp4 video playback (only for gifs)"

    hlsPlayback(checkbox, false):
-      "Enable hls video streaming (requires JavaScript)"
+      "Enable HLS video streaming (requires JavaScript)"

    proxyVideos(checkbox, true):
      "Proxy video streaming through the server (might be slow)"
@ -107,10 +107,6 @@ genPrefs:
      "Reddit -> Teddit/Libreddit"
      placeholder: "Teddit hostname"

-    replaceInstagram(input, ""):
-      "Instagram -> Bibliogram"
-      placeholder: "Bibliogram hostname"
-
 iterator allPrefs*(): Pref =
  for k, v in prefList:
    for pref in v:
--- a/src/redis_cache.nim
+++ b/src/redis_cache.nim
@ -118,11 +118,11 @@ proc getUserId*(username: string): Future[string] {.async.} =
  pool.withAcquire(r):
    result = await r.hGet(name.uidKey, name)
    if result == redisNil:
-      let user = await getUser(username)
+      let user = await getGraphUser(username)
      if user.suspended:
        return "suspended"
      else:
-        await cacheUserId(name, user.id)
+        await all(cacheUserId(name, user.id), cache(user))
        return user.id

 proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
@ -130,8 +130,7 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
  if prof != redisNil:
    prof.deserialize(User)
  elif fetch:
-    let userId = await getUserId(username)
-    result = await getGraphUser(userId)
+    result = await getGraphUser(username)
    await cache(result)

 proc getCachedUsername*(userId: string): Future[string] {.async.} =
@ -142,9 +141,11 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
  if username != redisNil:
    result = username
  else:
-    let user = await getUserById(userId)
+    let user = await getGraphUserById(userId)
    result = user.username
    await setEx(key, baseCacheTime, result)
+    if result.len > 0 and user.id.len > 0:
+      await all(cacheUserId(result, user.id), cache(user))

 proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
  if id == 0: return
@ -153,7 +154,7 @@ proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
    tweet.deserialize(Tweet)
  else:
    result = await getStatus($id)
-    if result.isNil:
+    if not result.isNil:
      await cache(result)

 proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
--- a/src/routes/rss.nim
+++ b/src/routes/rss.nim
@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, tables, times, hashes, uri
+import asyncdispatch, tables, times, hashes, uri

 import jester

@ -10,6 +10,11 @@ include "../views/rss.nimf"

 export times, hashes

+proc redisKey*(page, name, cursor: string): string =
+  result = page & ":" & name
+  if cursor.len > 0:
+    result &= ":" & cursor
+
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
  var profile: Profile
  let
@ -42,8 +47,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
 template respRss*(rss, page) =
  if rss.cursor.len == 0:
    let info = case page
-               of "User": &""" "{@"name"}" """
-               of "List": &""" "{@"id"}" """
+               of "User": " \"" & @"name" & "\" "
+               of "List": " \"" & @"id" & "\" "
               else: " "

    resp Http404, showError(page & info & "not found", cfg)
@ -67,7 +72,7 @@ proc createRssRouter*(cfg: Config) =

      let
        cursor = getCursor()
-        key = &"search:{hash(genQueryUrl(query))}:cursor"
+        key = redisKey("search", $hash(genQueryUrl(query)), cursor)

      var rss = await getCachedRss(key)
      if rss.cursor.len > 0:
@ -84,9 +89,8 @@ proc createRssRouter*(cfg: Config) =
      cond cfg.enableRss
      cond '.' notin @"name"
      let
-        cursor = getCursor()
        name = @"name"
-        key = &"twitter:{name}:{cursor}"
+        key = redisKey("twitter", name, getCursor())

      var rss = await getCachedRss(key)
      if rss.cursor.len > 0:
@ -101,18 +105,20 @@ proc createRssRouter*(cfg: Config) =
      cond cfg.enableRss
      cond '.' notin @"name"
      cond @"tab" in ["with_replies", "media", "search"]
-      let name = @"name"
-      let query =
-        case @"tab"
-        of "with_replies": getReplyQuery(name)
-        of "media": getMediaQuery(name)
-        of "search": initQuery(params(request), name=name)
-        else: Query(fromUser: @[name])
+      let
+        name = @"name"
+        tab = @"tab"
+        query =
+          case tab
+          of "with_replies": getReplyQuery(name)
+          of "media": getMediaQuery(name)
+          of "search": initQuery(params(request), name=name)
+          else: Query(fromUser: @[name])

-      var key = &"""{@"tab"}:{@"name"}:"""
-      if @"tab" == "search":
-        key &= $hash(genQueryUrl(query)) & ":"
-      key &= getCursor()
+      let searchKey = if tab != "search": ""
+                      else: ":" & $hash(genQueryUrl(query))
+
+      let key = redisKey(tab, name & searchKey, getCursor())

      var rss = await getCachedRss(key)
      if rss.cursor.len > 0:
@ -132,28 +138,27 @@ proc createRssRouter*(cfg: Config) =
        cursor = getCursor()

      if list.id.len == 0:
-        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
+        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)

-      let url = &"/i/lists/{list.id}/rss"
+      let url = "/i/lists/" & list.id & "/rss"
      if cursor.len > 0:
-        redirect(&"{url}?cursor={encodeUrl(cursor, false)}")
+        redirect(url & "?cursor=" & encodeUrl(cursor, false))
      else:
        redirect(url)

    get "/i/lists/@id/rss":
      cond cfg.enableRss
      let
+        id = @"id"
        cursor = getCursor()
-        key =
-          if cursor.len == 0: "lists:" & @"id"
-          else: &"""lists:{@"id"}:{cursor}"""
+        key = redisKey("lists", id, cursor)

      var rss = await getCachedRss(key)
      if rss.cursor.len > 0:
        respRss(rss, "List")

      let
-        list = await getCachedList(id=(@"id"))
+        list = await getCachedList(id=id)
        timeline = await getListTimeline(list.id, cursor)
      rss.cursor = timeline.bottom
      rss.feed = renderListRss(timeline.content, list, cfg)
--- a/src/routes/search.nim
+++ b/src/routes/search.nim
@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, uri
+import strutils, uri

 import jester

@ -14,32 +14,34 @@ export search
 proc createSearchRouter*(cfg: Config) =
  router search:
    get "/search/?":
-      if @"q".len > 500:
+      let q = @"q"
+      if q.len > 500:
        resp Http400, showError("Search input too long.", cfg)

      let
        prefs = cookiePrefs()
        query = initQuery(params(request))
+        title = "Search" & (if q.len > 0: " (" & q & ")" else: "")

      case query.kind
      of users:
-        if "," in @"q":
-          redirect("/" & @"q")
+        if "," in q:
+          redirect("/" & q)
        let users = await getSearch[User](query, getCursor())
-        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
+        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
      of tweets:
        let
          tweets = await getSearch[Tweet](query, getCursor())
          rss = "/search/rss?" & genQueryUrl(query)
        resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
-                        request, cfg, prefs, rss=rss)
+                        request, cfg, prefs, title, rss=rss)
      else:
        resp Http404, showError("Invalid search", cfg)

    get "/hashtag/@hash":
-      redirect(&"""/search?q={encodeUrl("#" & @"hash")}""")
+      redirect("/search?q=" & encodeUrl("#" & @"hash"))

    get "/opensearch":
      let url = getUrlPrefix(cfg) & "/search?q="
      resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                    generateOpenSearchXML(cfg.title, cfg.hostname, url)
+                     generateOpenSearchXML(cfg.title, cfg.hostname, url)
--- a/src/routes/timeline.nim
+++ b/src/routes/timeline.nim
@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, sequtils, uri, options, times
+import asyncdispatch, strutils, sequtils, uri, options, times
 import jester, karax/vdom

 import router_utils
@ -102,7 +102,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
 template respTimeline*(timeline: typed) =
  let t = timeline
  if t.len == 0:
-    resp Http404, showError(&"""User "{@"name"}" not found""", cfg)
+    resp Http404, showError("User \"" & @"name" & "\" not found", cfg)
  resp t

 template respUserId*() =
--- a/src/sass/profile/card.scss
+++ b/src/sass/profile/card.scss
@ -73,9 +73,9 @@
        }
    }

-    .profile-joindate, .profile-location, profile-website {
+    .profile-joindate, .profile-location, .profile-website {
        color: var(--fg_faded);
-        margin: 2px 0;
+        margin: 1px 0;
        width: 100%;
    }
 }
--- a/src/sass/tweet/_base.scss
+++ b/src/sass/tweet/_base.scss
@ -100,6 +100,7 @@
 .avatar {
    &.round {
        border-radius: 50%;
+        -webkit-user-select: none;
    }
    
    &.mini {
@ -137,7 +138,6 @@
    }
 }

-
 .attribution {
    display: flex;
    pointer-events: all;
@ -200,6 +200,7 @@

 .tweet-stats {
    margin-bottom: -3px;
+    -webkit-user-select: none;
 }

 .tweet-stat {
@ -231,6 +232,7 @@
    left: 0;
    top: 0;
    position: absolute;
+    -webkit-user-select: none;

    &:hover {
        background-color: var(--bg_hover);
--- a/src/sass/tweet/thread.scss
+++ b/src/sass/tweet/thread.scss
@ -23,7 +23,6 @@
    font-size: 18px;
 }

-
@media(max-width: 600px) {
    .main-tweet .tweet-content {
        font-size: 16px;
--- a/src/sass/tweet/video.scss
+++ b/src/sass/tweet/video.scss
@ -3,7 +3,7 @@

 video {
    max-height: 100%;
-    max-width: 100%;
+    width: 100%;
 }

 .gallery-video {
--- a/src/tokens.nim
+++ b/src/tokens.nim
@ -41,7 +41,8 @@ proc getPoolJson*(): JsonNode =
      let
        maxReqs =
          case api
-          of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500
+          of Api.listMembers, Api.listBySlug, Api.list,
+             Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
          of Api.timeline: 187
          else: 180
        reqs = maxReqs - token.apis[api].remaining
--- a/src/types.nim
+++ b/src/types.nim
@ -9,6 +9,7 @@ type
  InternalError* = object of CatchableError

  Api* {.pure.} = enum
+    tweetDetail
    userShow
    timeline
    search
@ -17,6 +18,7 @@ type
    listBySlug
    listMembers
    userRestId
+    userScreenName
    status

  RateLimit* = object
@ -43,9 +45,12 @@ type
    invalidToken = 89
    listIdOrSlug = 112
    tweetNotFound = 144
+    tweetNotAuthorized = 179
    forbidden = 200
    badToken = 239
    noCsrf = 353
+    tweetUnavailable = 421
+    tweetCensored = 422

  User* = object
    id*: string
@ -240,6 +245,7 @@ type
    available*: bool
    tombstone*: string
    location*: string
+    # Unused, needed for backwards compat
    source*: string
    stats*: TweetStats
    retweet*: Option[Tweet]
--- a/src/views/general.nim
+++ b/src/views/general.nim
@ -81,7 +81,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";

    title:
      if titleText.len > 0:
-        text &"{titleText}|{cfg.title}"
+        text titleText & " | " & cfg.title
      else:
        text cfg.title

@ -98,9 +98,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
      link(rel="preload", type="image/png", href=bannerUrl, `as`="image")

    for url in images:
-      let suffix = if "400x400" in url or url.endsWith("placeholder.png"): ""
-                   else: "?name=small"
-      let preloadUrl = getPicUrl(url & suffix)
+      let preloadUrl = if "400x400" in url: getPicUrl(url)
+                       else: getSmallPic(url)
      link(rel="preload", type="image/png", href=preloadUrl, `as`="image")

      let image = getUrlPrefix(cfg) & getPicUrl(url)
--- a/src/views/profile.nim
+++ b/src/views/profile.nim
@ -50,7 +50,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
          span:
            let url = replaceUrls(user.website, prefs)
            icon "link"
-            a(href=url): text shortLink(url)
+            a(href=url): text url.shortLink

      tdiv(class="profile-joindate"):
        span(title=getJoinDateFull(user)):
@ -108,7 +108,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
        renderBanner(profile.user.banner)

    let sticky = if prefs.stickyProfile: " sticky" else: ""
-    tdiv(class=(&"profile-tab{sticky}")):
+    tdiv(class=("profile-tab" & sticky)):
      renderUserCard(profile.user, prefs)
      if profile.photoRail.len > 0:
        renderPhotoRail(profile)
--- a/src/views/renderutils.nim
+++ b/src/views/renderutils.nim
@ -3,6 +3,14 @@ import strutils, strformat
 import karax/[karaxdsl, vdom, vstyles]
 import ".."/[types, utils]

+const smallWebp* = "?name=small&format=webp"
+
+proc getSmallPic*(url: string): string =
+  result = url
+  if "?" notin url and not url.endsWith("placeholder.png"):
+    result &= smallWebp
+  result = getPicUrl(result)
+
 proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
  var c = "icon-" & icon
  if class.len > 0: c = &"{c} {class}"
@ -55,12 +63,12 @@ proc genCheckbox*(pref, label: string; state: bool): VNode =
    else: input(name=pref, `type`="checkbox")
    span(class="checkbox")

-proc genInput*(pref, label, state, placeholder: string; class=""): VNode =
+proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
  let p = placeholder
  buildHtml(tdiv(class=("pref-group pref-input " & class))):
    if label.len > 0:
      label(`for`=pref): text label
-    if state.len == 0:
+    if autofocus and state.len == 0:
      input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
    else:
      input(name=pref, `type`="text", placeholder=p, value=state)
--- a/src/views/search.nim
+++ b/src/views/search.nim
@ -88,7 +88,7 @@ proc renderSearchPanel*(query: Query): VNode =
            genDate("until", query.until)
        tdiv:
          span(class="search-title"): text "Near"
-          genInput("near", "", query.near, placeholder="Location...")
+          genInput("near", "", query.near, "Location...", autofocus=false)

 proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
                        pinned=none(Tweet)): VNode =
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@ -7,14 +7,7 @@ import renderutils
 import ".."/[types, utils, formatters]
 import general

-const
-  doctype = "<!DOCTYPE html>\n"
-
-proc getSmallPic(url: string): string =
-  result = url
-  if "?" notin url and not url.endsWith("placeholder.png"):
-    result &= "?name=small"
-  result = getPicUrl(result)
+const doctype = "<!DOCTYPE html>\n"

 proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
  let url = getPicUrl(user.getUserPic("_mini"))
@ -60,9 +53,8 @@ proc renderAlbum(tweet: Tweet): VNode =
          tdiv(class="attachment image"):
            let
              named = "name=" in photo
-              orig = photo
-              small = if named: photo else: photo & "?name=small"
-            a(href=getOrigPicUrl(orig), class="still-image", target="_blank"):
+              small = if named: photo else: photo & smallWebp
+            a(href=getOrigPicUrl(photo), class="still-image", target="_blank"):
              genImg(small)

 proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool =
@ -355,7 +347,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
        renderQuote(tweet.quote.get(), prefs, path)

      if mainTweet:
-        p(class="tweet-published"): text &"{getTime(tweet)} · {tweet.source}"
+        p(class="tweet-published"): text &"{getTime(tweet)}"

      if tweet.mediaTags.len > 0:
        renderMediaTags(tweet.mediaTags)
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@ -0,0 +1 @@
+seleniumbase
--- a/tests/test_card.py
+++ b/tests/test_card.py
@ -42,7 +42,7 @@ no_thumb = [

    ['nim_lang/status/1082989146040340480',
     'Nim in 2018: A short recap',
-     'Posted in r/programming by u/miran1',
+     'Posted by u/miran1 - 36 votes and 46 comments',
     'reddit.com']
 ]

--- a/tests/test_quote.py
+++ b/tests/test_quote.py
@ -3,7 +3,7 @@ from parameterized import parameterized

 text = [
    ['elonmusk/status/1138136540096319488',
-     'Trev Page', '@Model3Owners',
+     'TREV PAGE', '@Model3Owners',
     """As of March 58.4% of new car sales in Norway are electric. 

 What are we doing wrong? reuters.com/article/us-norwa…"""],