update(nitter): merge with guest_accounts

.github/workflows/run-tests.yml

@@ -40,6 +40,9 @@ jobs:
           nimble md
           nimble scss
       - name: Run tests
+        env:
+          GUEST_ACCOUNTS: ${{ secrets.GUEST_ACCOUNTS }}
         run: |
+          echo $GUEST_ACCOUNTS > ./guest_accounts.json
           ./nitter &
           pytest -n4 tests

.gitignore

@@ -10,6 +10,7 @@ nitter
 /public/css/style.css
 /public/md/*.html
 nitter.conf
+guest_accounts.json*
 dump.rdb
 
 guest_accounts.json

src/experimental/parser/guestaccount.nim

@@ -0,0 +1,20 @@
+import jsony
+import ../types/guestaccount
+from ../../types import GuestAccount
+
+proc toGuestAccount(account: RawAccount): GuestAccount =
+  let id = account.oauthToken[0 ..< account.oauthToken.find('-')]
+  result = GuestAccount(
+    id: id,
+    oauthToken: account.oauthToken,
+    oauthSecret: account.oauthTokenSecret
+  )
+
+proc parseGuestAccount*(raw: string): GuestAccount =
+  let rawAccount = raw.fromJson(RawAccount)
+  result = rawAccount.toGuestAccount
+
+proc parseGuestAccounts*(path: string): seq[GuestAccount] =
+  let rawAccounts = readFile(path).fromJson(seq[RawAccount])
+  for account in rawAccounts:
+    result.add account.toGuestAccount

src/experimental/parser/user.nim

@@ -56,7 +56,7 @@ proc toUser*(raw: RawUser): User =
     tweets: raw.statusesCount,
     likes: raw.favouritesCount,
     media: raw.mediaCount,
-    verified: raw.verified,
+    verified: raw.verified or raw.extIsBlueVerified,
     protected: raw.protected,
     joinDate: parseTwitterDate(raw.createdAt),
     banner: getBanner(raw),

src/experimental/types/guestaccount.nim

@@ -0,0 +1,4 @@
+type
+  RawAccount* = object
+    oauthToken*: string
+    oauthTokenSecret*: string

src/experimental/types/user.nim

@@ -16,6 +16,7 @@ type
     statusesCount*: int
     mediaCount*: int
     verified*: bool
+    extIsBlueVerified*: bool
     protected*: bool
     profileLinkColor*: string
     profileBannerUrl*: string

src/nitter.nim

@@ -18,12 +18,11 @@ const issuesUrl = "https://github.com/zedeus/nitter/issues"
 
 let
   configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
-  (cfg*, fullCfg) = getConfig(configPath)
+  (cfg, fullCfg) = getConfig(configPath)
 
   accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json")
-  accounts = parseJson(readFile(accountsPath))
 
-initAccountPool(cfg, parseJson(readFile(accountsPath)))
+initAccountPool(cfg, accountsPath)
 
 if not cfg.enableDebug:
   # Silence Jester's query warning

src/parser.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, options, times, math, tables
+import strutils, options, times, math
 import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard

src/redis_cache.nim

@@ -85,7 +85,7 @@ proc cache*(data: List) {.async.} =
   await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
 
 proc cache*(data: PhotoRail; name: string) {.async.} =
-  await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
+  await setEx("pr:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
 
 proc cache*(data: User) {.async.} =
   if data.username.len == 0: return

src/sass/navbar.scss

@@ -70,8 +70,9 @@ nav {
 
 .lp {
     height: 14px;
-    margin-top: 2px;
-    display: block;
+    display: inline-block;
+    position: relative;
+    top: 2px;
     fill: var(--fg_nav);
 
     &:hover {

src/tokens.nim

@@ -1,6 +1,7 @@
 #SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, times, json, random, strutils, tables, sets
+import asyncdispatch, times, json, random, strutils, tables, sets, os
 import types
+import experimental/parser/guestaccount
 
 # max requests at a time per account to avoid race conditions
 const
@@ -140,12 +141,18 @@ proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) =
 
   account.apis[api] = RateLimit(remaining: remaining, reset: reset)
 
-proc initAccountPool*(cfg: Config; accounts: JsonNode) =
+proc initAccountPool*(cfg: Config; path: string) =
   enableLogging = cfg.enableDebug
 
-  for account in accounts:
-    accountPool.add GuestAccount(
-      id: account{"user", "id_str"}.getStr,
-      oauthToken: account{"oauth_token"}.getStr,
-      oauthSecret: account{"oauth_token_secret"}.getStr,
-    )
+  let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path
+
+  if fileExists(jsonlPath):
+    log "Parsing JSONL guest accounts file: ", jsonlPath
+    for line in jsonlPath.lines:
+      accountPool.add parseGuestAccount(line)
+  elif fileExists(path):
+    log "Parsing JSON guest accounts file: ", path
+    accountPool = parseGuestAccounts(path)
+  else:
+    echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests."
+    quit 1

tests/test_card.py

@@ -13,11 +13,6 @@ card = [
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
      'gist.github.com', True],
 
-    ['FluentAI/status/1116417904831029248',
-     'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
-     'One of the only ways to improve Alexa is to have human beings check it for errors',
-     'theverge.com', True],
-
     ['nim_lang/status/1082989146040340480',
      'Nim in 2018: A short recap',
      'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
@@ -25,6 +20,11 @@ card = [
 ]
 
 no_thumb = [
+    ['FluentAI/status/1116417904831029248',
+     'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
+     'One of the only ways to improve Alexa is to have human beings check it for errors',
+     'theverge.com'],
+
     ['Thom_Wolf/status/1122466524860702729',
      'facebookresearch/fairseq',
      'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',

tests/test_timeline.py

@@ -6,7 +6,7 @@ normal = [['jack'], ['elonmusk']]
 after = [['jack', '1681686036294803456'],
          ['elonmusk', '1681686036294803456']]
 
-no_more = [['mobile_test_8?cursor=1000']]
+no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']]
 
 empty = [['emptyuser'], ['mobile_test_10']]
 

tests/test_tweet_media.py

@@ -14,7 +14,7 @@ poll = [
 
 image = [
     ['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'],
-    ['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
+    #['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
 ]
 
 gif = [