Add fallback for sensitive profiles

src/api.nim

@@ -8,9 +8,31 @@ const base = parseUri("https://twitter.com/")
 const agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
 
 const timelineUrl = "i/profiles/show/$1/timeline/tweets?include_available_features=1&include_entities=1&include_new_items_bar=true"
-const profileUrl = "i/profiles/popup"
+const profilePopupUrl = "i/profiles/popup"
+const profileIntentUrl = "intent/user"
 const tweetUrl = "i/status/"
 
+proc fetchHtml(client: AsyncHttpClient; url: Uri; jsonKey = ""): Future[XmlNode] {.async.} =
+  var resp = ""
+  try:
+    resp = await client.getContent($url)
+  except:
+    return nil
+
+  if jsonKey.len > 0:
+    let json = parseJson(resp)[jsonKey].str
+    return parseHtml(json)
+  else:
+    return parseHtml(resp)
+
+proc getProfileFallback(username: string; client: AsyncHttpClient): Future[Profile] {.async.} =
+  let
+    params = {"screen_name": username}
+    url = base / profileIntentUrl ? params
+    html = await client.fetchHtml(url)
+
+  result = parseIntentProfile(html)
+
 proc getProfile*(username: string): Future[Profile] {.async.} =
   let client = newAsyncHttpClient()
   defer: client.close()
@@ -24,25 +46,19 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
     "Accept-Language": "en-US,en;q=0.9"
   })
 
-  let params = {
-    "screen_name": username,
-    "wants_hovercard": "true",
-    "_": $(epochTime().int)
-  }
-
-  let url = base / profileUrl ? params
-  var resp = ""
-
-  try:
-    resp = await client.getContent($url)
-  except:
-    return Profile()
-
   let
-    json = parseJson(resp)["html"].str
-    html = parseHtml(json)
+    params = {
+      "screen_name": username,
+      "wants_hovercard": "true",
+      "_": $(epochTime().int)
+    }
+    url = base / profilePopupUrl ? params
+    html = await client.fetchHtml(url, jsonKey="html")
 
-  result = parseProfile(html)
+  if not html.querySelector(".ProfileCard-sensitiveWarningContainer").isNil:
+    return await getProfileFallback(username, client)
+
+  result = parsePopupProfile(html)
 
 proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
   let client = newAsyncHttpClient()
@@ -61,18 +77,7 @@ proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
   if after != "":
     url &= "&max_position=" & after
 
-  var resp = ""
-  try:
-    resp = await client.getContent($(base / url))
-  except:
-    return
-
-  var json: string = ""
-  var html: XmlNode
-  json = parseJson(resp)["items_html"].str
-  html = parseHtml(json)
-
-  writeFile("epic.html", $html)
+  let html = await client.fetchHtml(base / url, jsonKey="items_html")
 
   result = parseTweets(html)
 
@@ -91,15 +96,8 @@ proc getTweet*(id: string): Future[Conversation] {.async.} =
     "x-previous-page-name": "profile"
   })
 
-  let url = base / tweetUrl / id
-
-  var resp: string = ""
-  try:
-    resp = await client.getContent($url)
-  except:
-    return Conversation()
-
-  var html: XmlNode
-  html = parseHtml(resp)
+  let
+    url = base / tweetUrl / id
+    html = await client.fetchHtml(url)
 
   result = parseConversation(html)

src/parser.nim

@@ -4,7 +4,7 @@ import nimquery, regex
 import ./types, ./formatters
 
 proc getAttr(node: XmlNode; attr: string; default=""): string =
-  if node.isNIl or node.attrs.isNil: return default
+  if node.isNil or node.attrs.isNil: return default
   return node.attrs.getOrDefault(attr)
 
 proc selectAttr(node: XmlNode; selector: string; attr: string; default=""): string =
@@ -15,16 +15,21 @@ proc selectText(node: XmlNode; selector: string): string =
   let res = node.querySelector(selector)
   result = if res == nil: "" else: res.innerText()
 
-proc parseProfile*(node: XmlNode): Profile =
+proc parsePopupProfile*(node: XmlNode): Profile =
   let profile = node.querySelector(".profile-card")
-  result.fullname = profile.selectText(".fullname").strip()
-  result.username = profile.selectText(".username").strip(chars={'@', ' '})
-  result.description = profile.selectText(".bio")
-  result.verified = profile.selectText(".Icon.Icon--verified").len > 0
-  result.protected = profile.selectText(".Icon.Icon--protected").len > 0
-  result.userpic = profile.selectAttr(".ProfileCard-avatarImage", "src").getUserpic()
-  result.banner = profile.selectAttr("svg > image", "xlink:href").replace("600x200", "1500x500")
-  if result.banner == "":
+  if profile.isNil: return
+
+  result = Profile(
+     fullname: profile.selectText(".fullname").strip(),
+     username: profile.selectText(".username").strip(chars={'@', ' '}),
+     description: profile.selectText(".bio"),
+     verified: profile.selectText(".Icon.Icon--verified").len > 0,
+     protected: profile.selectText(".Icon.Icon--protected").len > 0,
+     userpic: profile.selectAttr(".ProfileCard-avatarImage", "src").getUserpic(),
+     banner: profile.selectAttr("svg > image", "xlink:href").replace("600x200", "1500x500")
+  )
+
+  if result.banner.len == 0:
       result.banner = profile.selectAttr(".ProfileCard-bg", "style")
 
   let stats = profile.querySelectorAll(".ProfileCardStats-statLink")
@@ -35,12 +40,29 @@ proc parseProfile*(node: XmlNode): Profile =
     of "following": result.following = text
     else: result.tweets = text
 
-proc parseTweetProfile*(tweet: XmlNode): Profile =
+proc parseIntentProfile*(profile: XmlNode): Profile =
   result = Profile(
-    fullname: tweet.getAttr("data-name"),
-    username: tweet.getAttr("data-screen-name"),
-    userpic: tweet.selectAttr(".avatar", "src").getUserpic(),
-    verified: tweet.selectText(".Icon.Icon--verified").len > 0
+    fullname: profile.selectText("a.fn.url.alternate-context").strip(),
+    username: profile.selectText(".nickname").strip(chars={'@', ' '}),
+    userpic: profile.querySelector(".profile.summary").selectAttr("img.photo", "src").getUserPic(),
+    description: profile.selectText("p.note").strip(),
+    verified: not profile.querySelector("li.verified").isNil,
+    protected: not profile.querySelector("li.protected").isNil,
+    banner: "background-color: #161616",
+    tweets: "?"
+  )
+
+  for stat in profile.querySelectorAll("dd.count > a"):
+    case stat.getAttr("href").split("/")[^1]
+    of "followers": result.followers = stat.innerText()
+    of "following": result.following = stat.innerText()
+
+proc parseTweetProfile*(profile: XmlNode): Profile =
+  result = Profile(
+    fullname: profile.getAttr("data-name"),
+    username: profile.getAttr("data-screen-name"),
+    userpic: profile.selectAttr(".avatar", "src").getUserpic(),
+    verified: profile.selectText(".Icon.Icon--verified").len > 0
   )
 
 proc parseTweet*(tweet: XmlNode): Tweet =