From 310c5e936d690d5c8cfd0facd80b06f876939ef9 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 9 Jun 2020 15:04:38 +0200 Subject: [PATCH] Optional base64 support for proxy urls --- nitter.conf | 1 + src/config.nim | 9 ++++--- src/formatters.nim | 6 +++++ src/nitter.nim | 1 + src/routes/media.nim | 59 +++++++++++++++++++++++--------------------- src/types.nim | 1 + src/utils.nim | 23 +++++++++++------ 7 files changed, 61 insertions(+), 39 deletions(-) diff --git a/nitter.conf b/nitter.conf index d66a155..99b0d18 100644 --- a/nitter.conf +++ b/nitter.conf @@ -20,6 +20,7 @@ redisMaxConnections = 30 [Config] hmacKey = "secretkey" # random key for cryptographic signing of video urls +base64Media = false # use base64 encoding for proxied media urls tokenCount = 10 # minimum amount of usable tokens. tokens are used to authorize API requests, # but they expire after ~1 hour, and have a limit of 187 requests. diff --git a/src/config.nim b/src/config.nim index 5566598..89e81e3 100644 --- a/src/config.nim +++ b/src/config.nim @@ -20,6 +20,10 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = hostname: cfg.get("Server", "hostname", "nitter.net"), staticDir: cfg.get("Server", "staticDir", "./public"), + hmacKey: cfg.get("Config", "hmacKey", "secretkey"), + base64Media: cfg.get("Config", "base64Media", false), + minTokens: cfg.get("Config", "tokenCount", 10), + cacheDir: cfg.get("Cache", "directory", "/tmp/nitter"), listCacheTime: cfg.get("Cache", "listMinutes", 120), rssCacheTime: cfg.get("Cache", "rssMinutes", 10), @@ -27,10 +31,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = redisHost: cfg.get("Cache", "redisHost", "localhost"), redisPort: cfg.get("Cache", "redisPort", 6379), redisConns: cfg.get("Cache", "redisConnections", 20), - redisMaxConns: cfg.get("Cache", "redisMaxConnections", 30), - - hmacKey: cfg.get("Config", "hmacKey", "secretkey"), - minTokens: cfg.get("Config", "tokenCount", 10), + redisMaxConns: cfg.get("Cache", "redisMaxConnections", 30) ) return (conf, cfg) diff --git a/src/formatters.nim b/src/formatters.nim index d1b289f..5de8981 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -15,6 +15,7 @@ const nbsp = $Rune(0x000A0) wwwRegex = re"https?://(www[0-9]?\.)?" + m3u8Regex = re"""url="(.+.m3u8)"""" manifestRegex = re"(.+(.ts|.m3u8|.vmap))" userpicRegex = re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$" extRegex = re"(\.[A-z]+)$" @@ -52,6 +53,11 @@ proc replaceUrl*(url: string; prefs: Prefs; absolute=""): string = if absolute.len > 0: result = result.replace("href=\"/", "href=\"https://" & absolute & "/") +proc getM3u8Url*(content: string): string = + var m: RegexMatch + if content.find(m3u8Regex, m): + result = content[m.group(0)[0]] + proc proxifyVideo*(manifest: string; proxy: bool): string = proc cb(m: RegexMatch; s: string): string = result = "https://video.twimg.com" & s[m.group(0)[0]] diff --git a/src/nitter.nim b/src/nitter.nim index 56a2110..826b624 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -23,6 +23,7 @@ stdout.flushFile updateDefaultPrefs(fullCfg) setCacheTimes(cfg) setHmacKey(cfg.hmacKey) +setProxyEncoding(cfg.base64Media) waitFor initRedisPool(cfg) stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n" diff --git a/src/routes/media.nim b/src/routes/media.nim index 6ad562c..32a2802 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -1,17 +1,16 @@ -import uri, strutils, httpclient, os, hashes +import uri, strutils, httpclient, os, hashes, base64, re import asynchttpserver, asyncstreams, asyncfile, asyncnet -import jester, regex +import jester import router_utils import ".."/[types, formatters, agents, utils] import ../views/general export asynchttpserver, asyncstreams, asyncfile, asyncnet -export httpclient, os, strutils, asyncstreams, regex +export httpclient, os, strutils, asyncstreams, base64, re const - m3u8Regex* = re"""url="(.+.m3u8)"""" m3u8Mime* = "application/vnd.apple.mpegurl" maxAge* = "max-age=604800" @@ -60,13 +59,27 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} = finally: client.safeClose() +template check*(code): untyped = + if code != Http200: + resp code + else: + enableRawMode() + break route + +proc decoded*(req: jester.Request; index: int): string = + let + based = req.matches[0].len > 1 + encoded = req.matches[index] + if based: decode(encoded) + else: decodeUrl(encoded) + proc createMediaRouter*(cfg: Config) = router media: get "/pic/?": resp Http404 - get "/pic/@url": - var url = decodeUrl(@"url") + get re"^\/pic\/(enc)?\/?(.+)": + var url = decoded(request, 1) if "twimg.com" notin url: url.insert(twimg) if not url.startsWith(https): @@ -75,42 +88,32 @@ proc createMediaRouter*(cfg: Config) = let uri = parseUri(url) cond isTwitterUrl(uri) == true - enableRawMode() - let code = await proxyMedia(request, $uri) - if code == Http200: - enableRawMode() - break route - else: - resp code + let code = await proxyMedia(request, url) + check code - get "/video/@sig/@url": - cond "http" in @"url" - var url = decodeUrl(@"url") - let prefs = cookiePrefs() + get re"^\/video\/(enc)?\/?(.+)\/(.+)$": + let url = decoded(request, 2) + cond "http" in url - if getHmac(url) != @"sig": + if getHmac(url) != request.matches[1]: resp showError("Failed to verify signature", cfg) if ".mp4" in url or ".ts" in url: let code = await proxyMedia(request, url) - if code == Http200: - enableRawMode() - break route - else: - resp code + check code var content: string if ".vmap" in url: - var m: RegexMatch - content = await safeFetch(url, mediaAgent) - if content.find(m3u8Regex, m): - url = decodeUrl(content[m.group(0)[0]]) + let m3u8 = getM3u8Url(await safeFetch(url, mediaAgent)) + if m3u8.len > 0: content = await safeFetch(url, mediaAgent) else: resp Http404 if ".m3u8" in url: - let vid = await safeFetch(url, mediaAgent) + let + vid = await safeFetch(url, mediaAgent) + prefs = cookiePrefs() content = proxifyVideo(vid, prefs.proxyVideos) resp content, m3u8Mime diff --git a/src/types.nim b/src/types.nim index b8849b0..b664fc4 100644 --- a/src/types.nim +++ b/src/types.nim @@ -201,6 +201,7 @@ type staticDir*: string hmacKey*: string + base64Media*: bool minTokens*: int cacheDir*: string diff --git a/src/utils.nim b/src/utils.nim index 6dfd6a2..c143793 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -1,7 +1,9 @@ -import strutils, strformat, sequtils, uri, tables +import strutils, strformat, sequtils, uri, tables, base64 import nimcrypto, regex -var hmacKey = "secretkey" +var + hmacKey = "secretkey" + base64Media = false const https* = "https://" @@ -20,18 +22,25 @@ const proc setHmacKey*(key: string) = hmacKey = key +proc setProxyEncoding*(state: bool) = + base64Media = state + proc getHmac*(data: string): string = ($hmac(sha256, hmacKey, data))[0 .. 12] proc getVidUrl*(link: string): string = if link.len == 0: return - let - sig = getHmac(link) - url = encodeUrl(link) - &"/video/{sig}/{url}" + let sig = getHmac(link) + if base64Media: + &"/video/enc/{sig}/{encode(link, safe=true)}" + else: + &"/video/{sig}/{encodeUrl(link)}" proc getPicUrl*(link: string): string = - &"/pic/{encodeUrl(link)}" + if base64Media: + &"/pic/enc/{encode(link, safe=true)}" + else: + &"/pic/{encodeUrl(link)}" proc cleanFilename*(filename: string): string = const reg = re"[^A-Za-z0-9._-]"