Merge branch 'master' into feature-notes

.github/workflows/build-docker.yml

@@ -1,4 +1,4 @@
-name: CI/CD
+name: Docker
 
 on:
   push:
@@ -8,31 +8,51 @@ on:
       - master
 
 jobs:
-  build-docker:
+  build-docker-amd64:
-    runs-on: ubuntu-latest
+    runs-on: buildjet-2vcpu-ubuntu-2204
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push
+      - name: Build and push AMD64 Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64
           push: true
           tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
+  build-docker-arm64:
+    runs-on: buildjet-2vcpu-ubuntu-2204-arm
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push ARM64 Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./Dockerfile.arm64
+          platforms: linux/arm64
+          push: true
+          tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64

.github/workflows/run-tests.yml

@@ -0,0 +1,42 @@
+name: Run tests
+
+on:
+  push:
+    paths-ignore:
+      - "*.md"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Cache nimble
+        id: cache-nimble
+        uses: actions/cache@v3
+        with:
+          path: ~/.nimble
+          key: nimble-${{ hashFiles('*.nimble') }}
+          restore-keys: "nimble-"
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - uses: jiro4989/setup-nim-action@v1
+        with:
+          nim-version: "1.x"
+      - run: nimble build -d:release -Y
+      - run: pip install seleniumbase
+      - run: seleniumbase install chromedriver
+      - uses: supercharge/redis-github-action@1.5.0
+      - name: Prepare Nitter
+        run: |
+          sudo apt install libsass-dev -y
+          cp nitter.example.conf nitter.conf
+          nimble md
+          nimble scss
+      - name: Run tests
+        run: |
+          ./nitter &
+          pytest -n4 tests

Dockerfile

@@ -1,4 +1,4 @@
-FROM nimlang/nim:1.6.2-alpine-regular as nim
+FROM nimlang/nim:1.6.10-alpine-regular as nim
 LABEL maintainer="setenforce@protonmail.com"
 
 RUN apk --no-cache add libsass-dev pcre
@@ -20,4 +20,6 @@ COPY --from=nim /src/nitter/nitter ./
 COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
 COPY --from=nim /src/nitter/public ./public
 EXPOSE 8080
+RUN adduser -h /src/ -D -s /bin/sh nitter
+USER nitter
 CMD ./nitter

Dockerfile.arm64

@@ -0,0 +1,23 @@
+FROM alpine:3.17 as nim
+LABEL maintainer="setenforce@protonmail.com"
+
+RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.8-r0" nimble pcre
+
+WORKDIR /src/nitter
+
+COPY nitter.nimble .
+RUN nimble install -y --depsOnly
+
+COPY . .
+RUN nimble build -d:danger -d:lto -d:strip \
+    && nimble scss \
+    && nimble md
+
+FROM alpine:3.17
+WORKDIR /src/
+RUN apk --no-cache add ca-certificates pcre openssl1.1-compat
+COPY --from=nim /src/nitter/nitter ./
+COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
+COPY --from=nim /src/nitter/public ./public
+EXPOSE 8080
+CMD ./nitter

README.md

@@ -109,7 +109,9 @@ performance reasons.
 
 ### Docker
 
-#### NOTE: For ARM64/ARM support, please use [unixfox's image](https://quay.io/repository/unixfox/nitter?tab=tags), more info [here](https://github.com/zedeus/nitter/issues/399#issuecomment-997263495)
+Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
+
+#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
 
 To run Nitter with Docker, you'll need to install and run Redis separately
 before you can run the container. See below for how to also run Redis using
@@ -122,6 +124,8 @@ docker build -t nitter:latest .
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
 ```
 
+Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
+
 A prebuilt Docker image is provided as well:
 
 ```bash

config.nims

@@ -1,5 +1,6 @@
 --define:ssl
 --define:useStdLib
+--threads:off
 
 # workaround httpbeast file upload bug
 --assertions:off

docker-compose.yml

@@ -8,7 +8,7 @@ services:
     ports:
       - "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
     volumes:
-      - ./nitter.conf:/src/nitter.conf:ro
+      - ./nitter.conf:/src/nitter.conf:Z,ro
     depends_on:
       - nitter-redis
     restart: unless-stopped
@@ -17,6 +17,12 @@ services:
       interval: 30s
       timeout: 5s
       retries: 2
+    user: "998:998"
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
 
   nitter-redis:
     image: redis:6-alpine
@@ -30,6 +36,12 @@ services:
       interval: 30s
       timeout: 5s
       retries: 2
+    user: "999:1000"
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
 
 volumes:
   nitter-redis:

nitter.example.conf

@@ -37,9 +37,8 @@ tokenCount = 10
 [Preferences]
 theme = "Nitter"
 replaceTwitter = "nitter.net"
-replaceYouTube = "piped.kavin.rocks"
+replaceYouTube = "piped.video"
 replaceReddit = "teddit.net"
-replaceInstagram = ""
 proxyVideos = true
 hlsPlayback = false
 infiniteScroll = false

nitter.nimble

@@ -11,18 +11,18 @@ bin           = @["nitter"]
 # Dependencies
 
 requires "nim >= 1.4.8"
-requires "jester >= 0.5.0"
+requires "jester#baca3f"
-requires "karax#5498909"
+requires "karax#9ee695b"
-requires "sass#e683aa1"
+requires "sass#7dfdd03"
-requires "nimcrypto#a5742a9"
+requires "nimcrypto#4014ef9"
-requires "markdown#a661c26"
+requires "markdown#158efe3"
-requires "packedjson#d11d167"
+requires "packedjson#9e6fbb6"
-requires "supersnappy#2.1.1"
+requires "supersnappy#6c94198"
 requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
-requires "zippy#0.9.11"
+requires "zippy#ca5989a"
-requires "flatty#0.2.3"
+requires "flatty#e668085"
-requires "jsony#d0e69bd"
+requires "jsony#ea811be"
 
 
 # Tasks

public/robots.txt

@@ -1,4 +1,5 @@
 User-agent: *
 Disallow: /
+Crawl-delay: 1
 User-agent: Twitterbot
 Disallow:

src/api.nim

@@ -4,11 +4,22 @@ import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
 
-proc getGraphUser*(id: string): Future[User] {.async.} =
+proc getGraphUser*(username: string): Future[User] {.async.} =
+  if username.len == 0: return
+  let
+    variables = """{
+      "screen_name": "$1",
+      "withSafetyModeUserFields": false,
+      "withSuperFollowsUserFields": false
+    }""" % [username]
+    js = await fetchRaw(graphUser ? {"variables": variables}, Api.userScreenName)
+  result = parseGraphUser(js)
+
+proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
-    variables = %*{"userId": id, "withSuperFollowsUserFields": true}
+    variables = """{"userId": "$1", "withSuperFollowsUserFields": true}""" % [id]
-    js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
+    js = await fetchRaw(graphUserById ? {"variables": variables}, Api.userRestId)
   result = parseGraphUser(js)
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
@@ -53,20 +64,6 @@ proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
     url = listTimeline ? ps
   result = parseTimeline(await fetch(url, Api.timeline), after)
 
-proc getUser*(username: string): Future[User] {.async.} =
-  if username.len == 0: return
-  let
-    ps = genParams({"screen_name": username})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json, username)
-
-proc getUserById*(userId: string): Future[User] {.async.} =
-  if userId.len == 0: return
-  let
-    ps = genParams({"user_id": userId})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json)
-
 proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
@@ -110,16 +107,21 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
   except InternalError:
     return Result[T](beginning: true, query: query)
 
-proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
+proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
-  let url = tweet / (id & ".json") ? genParams(cursor=after)
+  if id.len == 0: return
-  result = parseConversation(await fetch(url, Api.tweet), id)
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = tweetVariables % [id, cursor]
+    params = {"variables": variables, "features": tweetFeatures}
+    js = await fetch(graphTweet ? params, Api.tweetDetail)
+  result = parseGraphConversation(js, id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
-  result = (await getTweetImpl(id, after)).replies
+  result = (await getGraphTweet(id, after)).replies
   result.beginning = after.len == 0
 
 proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
-  result = await getTweetImpl(id)
+  result = await getGraphTweet(id)
   if after.len > 0:
     result.replies = await getReplies(id, after)
 

src/apiutils.nim

@@ -23,7 +23,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
     result &= ("count", count)
   if cursor.len > 0:
     # The raw cursor often has plus signs, which sometimes get turned into spaces,
-    # so we need to them back into a plus
+    # so we need to turn them back into a plus
     if " " in cursor:
       result &= ("cursor", cursor.replace(" ", "+"))
     else:
@@ -61,12 +61,20 @@ template fetchImpl(result, fetchBody) {.dirty.} =
   try:
     var resp: AsyncResponse
     pool.use(genHeaders(token)):
-      resp = await c.get($url)
+      template getContent =
-      result = await resp.body
+        resp = await c.get($url)
+        result = await resp.body
 
-      if resp.status == $Http503:
+      getContent()
-        badClient = true
+
-        raise newException(InternalError, result)
+      # Twitter randomly returns 401 errors with an empty body quite often.
+      # Retrying the request usually works.
+      if resp.status == "401 Unauthorized" and result.len == 0:
+        getContent()
+
+    if resp.status == $Http503:
+      badClient = true
+      raise newException(InternalError, result)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":

src/consts.nim

@@ -19,7 +19,9 @@ const
   tweet* = timelineApi / "conversation"
 
   graphql = api / "graphql"
-  graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
+  graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
+  graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
+  graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
   graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
   graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
   graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
@@ -58,3 +60,34 @@ const
   ## user:   "result_filter: user"
   ## photos: "result_filter: photos"
   ## videos: "result_filter: videos"
+
+  tweetVariables* = """{
+  "focalTweetId": "$1",
+  $2
+  "includePromotedContent": false,
+  "withBirdwatchNotes": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withSuperFollowsTweetFields": false,
+  "withSuperFollowsUserFields": false,
+  "withVoice": false,
+  "withV2Timeline": true
+}"""
+
+  tweetFeatures* = """{
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": false,
+  "standardized_nudges_misinfo": false,
+  "verified_phone_label_enabled": false,
+  "responsive_web_twitter_blue_verified_badge_is_enabled": false,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+  "view_counts_everywhere_api_enabled": false,
+  "responsive_web_edit_tweet_api_enabled": false,
+  "tweetypie_unmention_optimization_enabled": false,
+  "vibe_api_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
+  "responsive_web_text_conversations_enabled": false,
+  "responsive_web_enhance_cards_enabled": false,
+  "interactive_text_enabled": false
+}"""

src/experimental/parser/graphql.nim

@@ -1,9 +1,14 @@
+import options
 import jsony
 import user, ../types/[graphuser, graphlistmembers]
 from ../../types import User, Result, Query, QueryKind
 
 proc parseGraphUser*(json: string): User =
   let raw = json.fromJson(GraphUser)
+
+  if raw.data.user.result.reason.get("") == "Suspended":
+    return User(suspended: true)
+
   result = toUser raw.data.user.result.legacy
   result.id = raw.data.user.result.restId
 

src/experimental/parser/unifiedcard.nim

@@ -66,6 +66,8 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
       durationMs: videoInfo.durationMillis,
       variants: videoInfo.variants
     )
+  of model3d:
+    result.title = "Unsupported 3D model ad"
 
 proc parseUnifiedCard*(json: string): Card =
   let card = json.fromJson(UnifiedCard)
@@ -82,6 +84,8 @@ proc parseUnifiedCard*(json: string): Card =
       component.parseMedia(card, result)
     of buttonGroup:
       discard
+    of ComponentType.unknown:
+      echo "ERROR: Unknown component type: ", json
 
     case component.kind
     of twitterListDetails:

src/experimental/types/graphuser.nim

@@ -1,3 +1,4 @@
+import options
 import user
 
 type
@@ -10,3 +11,4 @@ type
   UserResult = object
     legacy*: RawUser
     restId*: string
+    reason*: Option[string]

src/experimental/types/unifiedcard.nim

@@ -17,6 +17,7 @@ type
     twitterListDetails
     communityDetails
     mediaWithDetailsHorizontal
+    unknown
 
   Component* = object
     kind*: ComponentType
@@ -47,7 +48,7 @@ type
     vanity*: string
 
   MediaType* = enum
-    photo, video
+    photo, video, model3d
 
   MediaEntity* = object
     kind*: MediaType
@@ -77,3 +78,29 @@ converter fromText*(text: Text): string = text.content
 proc renameHook*(v: var HasTypeField; fieldName: var string) =
   if fieldName == "type":
     fieldName = "kind"
+
+proc enumHook*(s: string; v: var ComponentType) =
+  v = case s
+      of "details": details
+      of "media": media
+      of "swipeable_media": swipeableMedia
+      of "button_group": buttonGroup
+      of "app_store_details": appStoreDetails
+      of "twitter_list_details": twitterListDetails
+      of "community_details": communityDetails
+      of "media_with_details_horizontal": mediaWithDetailsHorizontal
+      else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
+
+proc enumHook*(s: string; v: var AppType) =
+  v = case s
+      of "android_app": androidApp
+      of "iphone_app": iPhoneApp
+      of "ipad_app": iPadApp
+      else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
+
+proc enumHook*(s: string; v: var MediaType) =
+  v = case s
+      of "video": video
+      of "photo": photo
+      of "model3d": model3d
+      else: echo "ERROR: Unknown enum value (MediaType): ", s; photo

src/formatters.nim

@@ -12,8 +12,7 @@ let
   twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
   twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
 
-  ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
+  ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
-  igRegex = re"(www\.)?instagram\.com"
 
   rdRegex = re"(?<![.b])((www|np|new|amp|old)\.)?reddit.com"
   rdShortRegex = re"(?<![.b])redd\.it\/"
@@ -58,15 +57,13 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
 
   if prefs.replaceYouTube.len > 0 and "youtu" in result:
     result = result.replace(ytRegex, prefs.replaceYouTube)
-    if prefs.replaceYouTube in result:
-      result = result.replace("/c/", "/")
 
   if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
-    result = result.replace(tco, &"{https}{prefs.replaceTwitter}/t.co")
+    result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
     result = result.replace(cards, prefs.replaceTwitter & "/cards")
     result = result.replace(twRegex, prefs.replaceTwitter)
     result = result.replacef(twLinkRegex, a(
-      prefs.replaceTwitter & "$2", href = &"{https}{prefs.replaceTwitter}$1"))
+      prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
     result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
@@ -74,9 +71,6 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
     if prefs.replaceReddit in result and "/gallery/" in result:
       result = result.replace("/gallery/", "/comments/")
 
-  if prefs.replaceInstagram.len > 0 and "instagram.com" in result:
-    result = result.replace(igRegex, prefs.replaceInstagram)
-
   if absolute.len > 0 and "href" in result:
     result = result.replace("href=\"/", &"href=\"{absolute}/")
 

src/nitter.nim

@@ -57,6 +57,7 @@ settings:
   port = Port(cfg.port)
   staticDir = cfg.staticDir
   bindAddr = cfg.address
+  reusePort = true
 
 routes:
   get "/":

src/parser.nim

@@ -45,7 +45,6 @@ proc parseGraphList*(js: JsonNode): List =
     banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
   )
 
-
 proc parsePoll(js: JsonNode): Poll =
   let vals = js{"binding_values"}
   # name format is pollNchoice_*
@@ -73,8 +72,8 @@ proc parseGif(js: JsonNode): Gif =
 proc parseVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"media_url_https"}.getImageStr,
-    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
+    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt),
-    available: js{"ext_media_availability", "status"}.getStr == "available",
+    available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available",
     title: js{"ext_alt_text"}.getStr,
     durationMs: js{"video_info", "duration_millis"}.getInt
     # playbackType: mp4
@@ -186,7 +185,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
      result.url.len == 0 or result.url.startsWith("card://"):
     result.url = getPicUrl(result.image)
 
-proc parseTweet(js: JsonNode): Tweet =
+proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
   if js.isNull: return
   result = Tweet(
     id: js{"id_str"}.getId,
@@ -194,7 +193,6 @@ proc parseTweet(js: JsonNode): Tweet =
     replyId: js{"in_reply_to_status_id_str"}.getId,
     text: js{"full_text"}.getStr,
     time: js{"created_at"}.getTime,
-    source: getSource(js),
     hasThread: js{"self_thread"}.notNull,
     available: true,
     user: User(id: js{"user_id_str"}.getStr),
@@ -208,6 +206,10 @@ proc parseTweet(js: JsonNode): Tweet =
 
   result.expandTweetEntities(js)
 
+  # fix for pinned threads
+  if result.hasThread and result.threadId == 0:
+    result.threadId = js{"self_thread", "id_str"}.getId
+
   if js{"is_quote_status"}.getBool:
     result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
 
@@ -215,7 +217,7 @@ proc parseTweet(js: JsonNode): Tweet =
     result.retweet = some Tweet(id: rt.getId)
     return
 
-  with jsCard, js{"card"}:
+  if jsCard.kind != JNull:
     let name = jsCard{"name"}.getStr
     if "poll" in name:
       if "image" in name:
@@ -292,64 +294,18 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
     result.users[k] = parseUser(v, k)
 
   for k, v in tweets:
-    var tweet = parseTweet(v)
+    var tweet = parseTweet(v, v{"card"})
     if tweet.user.id in result.users:
       tweet.user = result.users[tweet.user.id]
     result.tweets[k] = tweet
 
-proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
-  result.thread = Chain()
-
-  let thread = js{"content", "item", "content", "conversationThread"}
-  with cursor, thread{"showMoreCursor"}:
-    result.thread.cursor = cursor{"value"}.getStr
-    result.thread.hasMore = true
-
-  for t in thread{"conversationComponents"}:
-    let content = t{"conversationTweetComponent", "tweet"}
-
-    if content{"displayType"}.getStr == "SelfThread":
-      result.self = true
-
-    var tweet = finalizeTweet(global, content{"id"}.getStr)
-    if not tweet.available:
-      tweet.tombstone = getTombstone(content{"tombstone"})
-    result.thread.content.add tweet
-
-proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
-  result = Conversation(replies: Result[Chain](beginning: true))
-  let global = parseGlobalObjects(? js)
-
-  let instructions = ? js{"timeline", "instructions"}
-  if instructions.len == 0:
-    return
-
-  for e in instructions[0]{"addEntries", "entries"}:
-    let entry = e{"entryId"}.getStr
-    if "tweet" in entry or "tombstone" in entry:
-      let tweet = finalizeTweet(global, e.getEntryId)
-      if $tweet.id != tweetId:
-        result.before.content.add tweet
-      else:
-        result.tweet = tweet
-    elif "conversationThread" in entry:
-      let (thread, self) = parseThread(e, global)
-      if thread.content.len > 0:
-        if self:
-          result.after = thread
-        else:
-          result.replies.content.add thread
-    elif "cursor-showMore" in entry:
-      result.replies.bottom = e.getCursor
-    elif "cursor-bottom" in entry:
-      result.replies.bottom = e.getCursor
-
 proc parseStatus*(js: JsonNode): Tweet =
   with e, js{"errors"}:
-    if e.getError == tweetNotFound:
+    if e.getError in {tweetNotFound, tweetUnavailable, tweetCensored, doesntExist,
+                      tweetNotAuthorized, suspended}:
       return
 
-  result = parseTweet(js)
+  result = parseTweet(js, js{"card"})
   if not result.isNil:
     result.user = parseUser(js{"user"})
 
@@ -406,7 +362,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
 proc parsePhotoRail*(js: JsonNode): PhotoRail =
   for tweet in js:
     let
-      t = parseTweet(tweet)
+      t = parseTweet(tweet, js{"card"})
       url = if t.photos.len > 0: t.photos[0]
             elif t.video.isSome: get(t.video).thumb
             elif t.gif.isSome: get(t.gif).thumb
@@ -416,6 +372,70 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =
     if url.len == 0: continue
     result.add GalleryPhoto(url: url, tweetId: $t.id)
 
+proc parseGraphTweet(js: JsonNode): Tweet =
+  if js.kind == JNull or js{"__typename"}.getStr == "TweetUnavailable":
+    return Tweet(available: false)
+
+  var jsCard = copy(js{"card", "legacy"})
+  if jsCard.kind != JNull:
+    var values = newJObject()
+    for val in jsCard["binding_values"]:
+      values[val["key"].getStr] = val["value"]
+    jsCard["binding_values"] = values
+
+  result = parseTweet(js{"legacy"}, jsCard)
+  result.user = parseUser(js{"core", "user_results", "result", "legacy"})
+
+  with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
+    result.expandNoteTweetEntities(noteTweet)
+
+  if result.quote.isSome:
+    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
+
+proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
+  let thread = js{"content", "items"}
+  for t in js{"content", "items"}:
+    let entryId = t{"entryId"}.getStr
+    if "cursor-showmore" in entryId:
+      let cursor = t{"item", "itemContent", "value"}
+      result.thread.cursor = cursor.getStr
+      result.thread.hasMore = true
+    elif "tweet" in entryId:
+      let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"})
+      result.thread.content.add tweet
+
+      if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
+        result.self = true
+
+proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
+  result = Conversation(replies: Result[Chain](beginning: true))
+
+  let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for e in instructions[0]{"entries"}:
+    let entryId = e{"entryId"}.getStr
+    # echo entryId
+    if entryId.startsWith("tweet"):
+      let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
+
+      if not tweet.available:
+        tweet.id = parseBiggestInt(entryId.getId())
+
+      if $tweet.id == tweetId:
+        result.tweet = tweet
+      else:
+        result.before.content.add tweet
+    elif entryId.startsWith("conversationthread"):
+      let (thread, self) = parseGraphThread(e)
+      if self:
+        result.after = thread
+      else:
+        result.replies.content.add thread
+    elif entryId.startsWith("cursor-bottom"):
+      result.replies.bottom = e{"content", "itemContent", "value"}.getStr
+
 proc parseGraphArticle*(js: JsonNode): Article =
   if not js{"errors"}.isNull:
     return

src/parserutils.nim

@@ -28,13 +28,13 @@ template `?`*(js: JsonNode): untyped =
   if j.isNull: return
   j
 
-template `with`*(ident, value, body): untyped =
+template with*(ident, value, body): untyped =
-  block:
+  if true:
     let ident {.inject.} = value
     if ident != nil: body
 
-template `with`*(ident; value: JsonNode; body): untyped =
+template with*(ident; value: JsonNode; body): untyped =
-  block:
+  if true:
     let ident {.inject.} = value
     if value.notNull: body
 
@@ -133,10 +133,6 @@ proc getTombstone*(js: JsonNode): string =
   result = js{"tombstoneInfo", "richText", "text"}.getStr
   result.removeSuffix(" Learn more")
 
-proc getSource*(js: JsonNode): string =
-  let src = js{"source"}.getStr
-  result = src.substr(src.find('>') + 1, src.rfind('<') - 1)
-
 proc getMp4Resolution*(url: string): int =
   # parses the height out of a URL like this one:
   # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
@@ -234,47 +230,37 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
   user.bio = user.bio.replacef(unRegex, unReplace)
                      .replacef(htRegex, htReplace)
 
-proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
+proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
-  let
+                        replyTo=""; hasQuote=false) =
-    orig = tweet.text.toRunes
+  let hasCard = tweet.card.isSome
-    textRange = js{"display_text_range"}
-    textSlice = textRange{0}.getInt .. textRange{1}.getInt
-    hasQuote = js{"is_quote_status"}.getBool
-    hasCard = tweet.card.isSome
-
-  var replyTo = ""
-  if tweet.replyId != 0:
-    with reply, js{"in_reply_to_screen_name"}:
-      tweet.reply.add reply.getStr
-      replyTo = reply.getStr
-
-  let ent = ? js{"entities"}
 
   var replacements = newSeq[ReplaceSlice]()
 
-  with urls, ent{"urls"}:
+  with urls, entities{"urls"}:
     for u in urls:
       let urlStr = u["url"].getStr
-      if urlStr.len == 0 or urlStr notin tweet.text:
+      if urlStr.len == 0 or urlStr notin text:
         continue
+
       replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
+
       if hasCard and u{"url"}.getStr == get(tweet.card).url:
         get(tweet.card).url = u{"expanded_url"}.getStr
 
-  with media, ent{"media"}:
+  with media, entities{"media"}:
     for m in media:
       replacements.extractUrls(m, textSlice.b, hideTwitter = true)
 
-  if "hashtags" in ent:
+  if "hashtags" in entities:
-    for hashtag in ent["hashtags"]:
+    for hashtag in entities["hashtags"]:
       replacements.extractHashtags(hashtag)
 
-  if "symbols" in ent:
+  if "symbols" in entities:
-    for symbol in ent["symbols"]:
+    for symbol in entities["symbols"]:
       replacements.extractHashtags(symbol)
 
-  if "user_mentions" in ent:
+  if "user_mentions" in entities:
-    for mention in ent["user_mentions"]:
+    for mention in entities["user_mentions"]:
       let
         name = mention{"screen_name"}.getStr
         slice = mention.extractSlice
@@ -291,5 +277,27 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
   replacements.deduplicate
   replacements.sort(cmp)
 
-  tweet.text = orig.replacedWith(replacements, textSlice)
+  tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
-                   .strip(leading=false)
+
+proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entities"}
+    hasQuote = js{"is_quote_status"}.getBool
+    textRange = js{"display_text_range"}
+    textSlice = textRange{0}.getInt .. textRange{1}.getInt
+
+  var replyTo = ""
+  if tweet.replyId != 0:
+    with reply, js{"in_reply_to_screen_name"}:
+      replyTo = reply.getStr
+      tweet.reply.add replyTo
+
+  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote)
+
+proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entity_set"}
+    text = js{"text"}.getStr
+    textSlice = 0..text.runeLen
+
+  tweet.expandTextEntities(entities, text, textSlice)

src/prefs_impl.nim

@@ -83,7 +83,7 @@ genPrefs:
       "Enable mp4 video playback (only for gifs)"
 
     hlsPlayback(checkbox, false):
-      "Enable hls video streaming (requires JavaScript)"
+      "Enable HLS video streaming (requires JavaScript)"
 
     proxyVideos(checkbox, true):
       "Proxy video streaming through the server (might be slow)"
@@ -107,10 +107,6 @@ genPrefs:
       "Reddit -> Teddit/Libreddit"
       placeholder: "Teddit hostname"
 
-    replaceInstagram(input, ""):
-      "Instagram -> Bibliogram"
-      placeholder: "Bibliogram hostname"
-
 iterator allPrefs*(): Pref =
   for k, v in prefList:
     for pref in v:

src/redis_cache.nim

@@ -118,11 +118,11 @@ proc getUserId*(username: string): Future[string] {.async.} =
   pool.withAcquire(r):
     result = await r.hGet(name.uidKey, name)
     if result == redisNil:
-      let user = await getUser(username)
+      let user = await getGraphUser(username)
       if user.suspended:
         return "suspended"
       else:
-        await cacheUserId(name, user.id)
+        await all(cacheUserId(name, user.id), cache(user))
         return user.id
 
 proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
@@ -130,8 +130,7 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
   if prof != redisNil:
     prof.deserialize(User)
   elif fetch:
-    let userId = await getUserId(username)
+    result = await getGraphUser(username)
-    result = await getGraphUser(userId)
     await cache(result)
 
 proc getCachedUsername*(userId: string): Future[string] {.async.} =
@@ -142,9 +141,11 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
   if username != redisNil:
     result = username
   else:
-    let user = await getUserById(userId)
+    let user = await getGraphUserById(userId)
     result = user.username
     await setEx(key, baseCacheTime, result)
+    if result.len > 0 and user.id.len > 0:
+      await all(cacheUserId(result, user.id), cache(user))
 
 proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
   if id == 0: return
@@ -153,7 +154,7 @@ proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
     tweet.deserialize(Tweet)
   else:
     result = await getStatus($id)
-    if result.isNil:
+    if not result.isNil:
       await cache(result)
 
 proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =

src/routes/rss.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, tables, times, hashes, uri
+import asyncdispatch, tables, times, hashes, uri
 
 import jester
 
@@ -10,6 +10,11 @@ include "../views/rss.nimf"
 
 export times, hashes
 
+proc redisKey*(page, name, cursor: string): string =
+  result = page & ":" & name
+  if cursor.len > 0:
+    result &= ":" & cursor
+
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
   var profile: Profile
   let
@@ -42,8 +47,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
 template respRss*(rss, page) =
   if rss.cursor.len == 0:
     let info = case page
-               of "User": &""" "{@"name"}" """
+               of "User": " \"" & @"name" & "\" "
-               of "List": &""" "{@"id"}" """
+               of "List": " \"" & @"id" & "\" "
                else: " "
 
     resp Http404, showError(page & info & "not found", cfg)
@@ -67,7 +72,7 @@ proc createRssRouter*(cfg: Config) =
 
       let
         cursor = getCursor()
-        key = &"search:{hash(genQueryUrl(query))}:cursor"
+        key = redisKey("search", $hash(genQueryUrl(query)), cursor)
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -84,9 +89,8 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       let
-        cursor = getCursor()
         name = @"name"
-        key = &"twitter:{name}:{cursor}"
+        key = redisKey("twitter", name, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -101,18 +105,20 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       cond @"tab" in ["with_replies", "media", "search"]
-      let name = @"name"
+      let
-      let query =
+        name = @"name"
-        case @"tab"
+        tab = @"tab"
-        of "with_replies": getReplyQuery(name)
+        query =
-        of "media": getMediaQuery(name)
+          case tab
-        of "search": initQuery(params(request), name=name)
+          of "with_replies": getReplyQuery(name)
-        else: Query(fromUser: @[name])
+          of "media": getMediaQuery(name)
+          of "search": initQuery(params(request), name=name)
+          else: Query(fromUser: @[name])
 
-      var key = &"""{@"tab"}:{@"name"}:"""
+      let searchKey = if tab != "search": ""
-      if @"tab" == "search":
+                      else: ":" & $hash(genQueryUrl(query))
-        key &= $hash(genQueryUrl(query)) & ":"
+
-      key &= getCursor()
+      let key = redisKey(tab, name & searchKey, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -132,28 +138,27 @@ proc createRssRouter*(cfg: Config) =
         cursor = getCursor()
 
       if list.id.len == 0:
-        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
+        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)
 
-      let url = &"/i/lists/{list.id}/rss"
+      let url = "/i/lists/" & list.id & "/rss"
       if cursor.len > 0:
-        redirect(&"{url}?cursor={encodeUrl(cursor, false)}")
+        redirect(url & "?cursor=" & encodeUrl(cursor, false))
       else:
         redirect(url)
 
     get "/i/lists/@id/rss":
       cond cfg.enableRss
       let
+        id = @"id"
         cursor = getCursor()
-        key =
+        key = redisKey("lists", id, cursor)
-          if cursor.len == 0: "lists:" & @"id"
-          else: &"""lists:{@"id"}:{cursor}"""
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "List")
 
       let
-        list = await getCachedList(id=(@"id"))
+        list = await getCachedList(id=id)
         timeline = await getListTimeline(list.id, cursor)
       rss.cursor = timeline.bottom
       rss.feed = renderListRss(timeline.content, list, cfg)

src/routes/search.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, uri
+import strutils, uri
 
 import jester
 
@@ -14,32 +14,34 @@ export search
 proc createSearchRouter*(cfg: Config) =
   router search:
     get "/search/?":
-      if @"q".len > 500:
+      let q = @"q"
+      if q.len > 500:
         resp Http400, showError("Search input too long.", cfg)
 
       let
         prefs = cookiePrefs()
         query = initQuery(params(request))
+        title = "Search" & (if q.len > 0: " (" & q & ")" else: "")
 
       case query.kind
       of users:
-        if "," in @"q":
+        if "," in q:
-          redirect("/" & @"q")
+          redirect("/" & q)
         let users = await getSearch[User](query, getCursor())
-        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
+        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
       of tweets:
         let
           tweets = await getSearch[Tweet](query, getCursor())
           rss = "/search/rss?" & genQueryUrl(query)
         resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
-                        request, cfg, prefs, rss=rss)
+                        request, cfg, prefs, title, rss=rss)
       else:
         resp Http404, showError("Invalid search", cfg)
 
     get "/hashtag/@hash":
-      redirect(&"""/search?q={encodeUrl("#" & @"hash")}""")
+      redirect("/search?q=" & encodeUrl("#" & @"hash"))
 
     get "/opensearch":
       let url = getUrlPrefix(cfg) & "/search?q="
       resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                    generateOpenSearchXML(cfg.title, cfg.hostname, url)
+                     generateOpenSearchXML(cfg.title, cfg.hostname, url)

src/routes/timeline.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, sequtils, uri, options, times
+import asyncdispatch, strutils, sequtils, uri, options, times
 import jester, karax/vdom
 
 import router_utils
@@ -102,7 +102,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
 template respTimeline*(timeline: typed) =
   let t = timeline
   if t.len == 0:
-    resp Http404, showError(&"""User "{@"name"}" not found""", cfg)
+    resp Http404, showError("User \"" & @"name" & "\" not found", cfg)
   resp t
 
 template respUserId*() =

src/sass/profile/card.scss

@@ -73,9 +73,9 @@
         }
     }
 
-    .profile-joindate, .profile-location, profile-website {
+    .profile-joindate, .profile-location, .profile-website {
         color: var(--fg_faded);
-        margin: 2px 0;
+        margin: 1px 0;
         width: 100%;
     }
 }

src/sass/tweet/_base.scss

@@ -100,6 +100,7 @@
 .avatar {
     &.round {
         border-radius: 50%;
+        -webkit-user-select: none;
     }
     
     &.mini {
@@ -137,7 +138,6 @@
     }
 }
 
-
 .attribution {
     display: flex;
     pointer-events: all;
@@ -200,6 +200,7 @@
 
 .tweet-stats {
     margin-bottom: -3px;
+    -webkit-user-select: none;
 }
 
 .tweet-stat {
@@ -231,6 +232,7 @@
     left: 0;
     top: 0;
     position: absolute;
+    -webkit-user-select: none;
 
     &:hover {
         background-color: var(--bg_hover);

src/sass/tweet/thread.scss

@@ -23,7 +23,6 @@
     font-size: 18px;
 }
 
-
 @media(max-width: 600px) {
     .main-tweet .tweet-content {
         font-size: 16px;

src/sass/tweet/video.scss

@@ -3,7 +3,7 @@
 
 video {
     max-height: 100%;
-    max-width: 100%;
+    width: 100%;
 }
 
 .gallery-video {

src/tokens.nim

@@ -41,7 +41,8 @@ proc getPoolJson*(): JsonNode =
       let
         maxReqs =
           case api
-          of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500
+          of Api.listMembers, Api.listBySlug, Api.list,
+             Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
           of Api.timeline: 187
           else: 180
         reqs = maxReqs - token.apis[api].remaining

src/types.nim

@@ -9,6 +9,7 @@ type
   InternalError* = object of CatchableError
 
   Api* {.pure.} = enum
+    tweetDetail
     userShow
     timeline
     search
@@ -17,6 +18,7 @@ type
     listBySlug
     listMembers
     userRestId
+    userScreenName
     status
 
   RateLimit* = object
@@ -43,9 +45,12 @@ type
     invalidToken = 89
     listIdOrSlug = 112
     tweetNotFound = 144
+    tweetNotAuthorized = 179
     forbidden = 200
     badToken = 239
     noCsrf = 353
+    tweetUnavailable = 421
+    tweetCensored = 422
 
   User* = object
     id*: string
@@ -240,6 +245,7 @@ type
     available*: bool
     tombstone*: string
     location*: string
+    # Unused, needed for backwards compat
     source*: string
     stats*: TweetStats
     retweet*: Option[Tweet]

src/views/general.nim

@@ -81,7 +81,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
 
     title:
       if titleText.len > 0:
-        text &"{titleText}|{cfg.title}"
+        text titleText & " | " & cfg.title
       else:
         text cfg.title
 
@@ -98,9 +98,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
       link(rel="preload", type="image/png", href=bannerUrl, `as`="image")
 
     for url in images:
-      let suffix = if "400x400" in url or url.endsWith("placeholder.png"): ""
+      let preloadUrl = if "400x400" in url: getPicUrl(url)
-                   else: "?name=small"
+                       else: getSmallPic(url)
-      let preloadUrl = getPicUrl(url & suffix)
       link(rel="preload", type="image/png", href=preloadUrl, `as`="image")
 
       let image = getUrlPrefix(cfg) & getPicUrl(url)

src/views/profile.nim

@@ -50,7 +50,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
           span:
             let url = replaceUrls(user.website, prefs)
             icon "link"
-            a(href=url): text shortLink(url)
+            a(href=url): text url.shortLink
 
       tdiv(class="profile-joindate"):
         span(title=getJoinDateFull(user)):
@@ -108,7 +108,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
         renderBanner(profile.user.banner)
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
-    tdiv(class=(&"profile-tab{sticky}")):
+    tdiv(class=("profile-tab" & sticky)):
       renderUserCard(profile.user, prefs)
       if profile.photoRail.len > 0:
         renderPhotoRail(profile)

src/views/renderutils.nim

@@ -3,6 +3,14 @@ import strutils, strformat
 import karax/[karaxdsl, vdom, vstyles]
 import ".."/[types, utils]
 
+const smallWebp* = "?name=small&format=webp"
+
+proc getSmallPic*(url: string): string =
+  result = url
+  if "?" notin url and not url.endsWith("placeholder.png"):
+    result &= smallWebp
+  result = getPicUrl(result)
+
 proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
   var c = "icon-" & icon
   if class.len > 0: c = &"{c} {class}"
@@ -55,12 +63,12 @@ proc genCheckbox*(pref, label: string; state: bool): VNode =
     else: input(name=pref, `type`="checkbox")
     span(class="checkbox")
 
-proc genInput*(pref, label, state, placeholder: string; class=""): VNode =
+proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
   let p = placeholder
   buildHtml(tdiv(class=("pref-group pref-input " & class))):
     if label.len > 0:
       label(`for`=pref): text label
-    if state.len == 0:
+    if autofocus and state.len == 0:
       input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
     else:
       input(name=pref, `type`="text", placeholder=p, value=state)

src/views/search.nim

@@ -88,7 +88,7 @@ proc renderSearchPanel*(query: Query): VNode =
             genDate("until", query.until)
         tdiv:
           span(class="search-title"): text "Near"
-          genInput("near", "", query.near, placeholder="Location...")
+          genInput("near", "", query.near, "Location...", autofocus=false)
 
 proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
                         pinned=none(Tweet)): VNode =

src/views/tweet.nim

@@ -7,14 +7,7 @@ import renderutils
 import ".."/[types, utils, formatters]
 import general
 
-const
+const doctype = "<!DOCTYPE html>\n"
-  doctype = "<!DOCTYPE html>\n"
-
-proc getSmallPic(url: string): string =
-  result = url
-  if "?" notin url and not url.endsWith("placeholder.png"):
-    result &= "?name=small"
-  result = getPicUrl(result)
 
 proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
   let url = getPicUrl(user.getUserPic("_mini"))
@@ -60,9 +53,8 @@ proc renderAlbum(tweet: Tweet): VNode =
           tdiv(class="attachment image"):
             let
               named = "name=" in photo
-              orig = photo
+              small = if named: photo else: photo & smallWebp
-              small = if named: photo else: photo & "?name=small"
+            a(href=getOrigPicUrl(photo), class="still-image", target="_blank"):
-            a(href=getOrigPicUrl(orig), class="still-image", target="_blank"):
               genImg(small)
 
 proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool =
@@ -355,7 +347,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         renderQuote(tweet.quote.get(), prefs, path)
 
       if mainTweet:
-        p(class="tweet-published"): text &"{getTime(tweet)} · {tweet.source}"
+        p(class="tweet-published"): text &"{getTime(tweet)}"
 
       if tweet.mediaTags.len > 0:
         renderMediaTags(tweet.mediaTags)

tests/requirements.txt

@@ -0,0 +1 @@
+seleniumbase

tests/test_card.py

@@ -42,7 +42,7 @@ no_thumb = [
 
     ['nim_lang/status/1082989146040340480',
      'Nim in 2018: A short recap',
-     'Posted in r/programming by u/miran1',
+     'Posted by u/miran1 - 36 votes and 46 comments',
      'reddit.com']
 ]
 

tests/test_quote.py

@@ -3,7 +3,7 @@ from parameterized import parameterized
 
 text = [
     ['elonmusk/status/1138136540096319488',
-     'Trev Page', '@Model3Owners',
+     'TREV PAGE', '@Model3Owners',
      """As of March 58.4% of new car sales in Norway are electric. 
 
 What are we doing wrong? reuters.com/article/us-norwa…"""],