From 294df19f1b137d103d281bb8645791723d075f7b Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Mon, 17 Oct 2022 18:09:06 +0200 Subject: [PATCH] Replace rss-parser with fetch and the native DomParser (#2726) --- package.json | 1 - src/renderer/App.js | 29 +-- .../views/Subscriptions/Subscriptions.js | 191 +++++++++--------- yarn.lock | 25 +-- 4 files changed, 111 insertions(+), 135 deletions(-) diff --git a/package.json b/package.json index ecb38cb6a..544a94bd3 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,6 @@ "nedb-promises": "^6.2.1", "opml-to-json": "^1.0.1", "process": "^0.11.10", - "rss-parser": "^3.12.0", "socks-proxy-agent": "^6.0.0", "video.js": "7.18.1", "videojs-contrib-quality-levels": "^2.1.0", diff --git a/src/renderer/App.js b/src/renderer/App.js index 634b81e00..5f67ffde4 100644 --- a/src/renderer/App.js +++ b/src/renderer/App.js @@ -10,7 +10,6 @@ import FtButton from './components/ft-button/ft-button.vue' import FtToast from './components/ft-toast/ft-toast.vue' import FtProgressBar from './components/ft-progress-bar/ft-progress-bar.vue' import { marked } from 'marked' -import Parser from 'rss-parser' import { IpcChannels } from '../constants' import packageDetails from '../../package.json' import { showToast } from './helpers/utils' @@ -229,26 +228,30 @@ export default Vue.extend({ checkForNewBlogPosts: function () { if (this.checkForBlogPosts) { - const parser = new Parser() - const feedUrl = 'https://write.as/freetube/feed/' let lastAppWasRunning = localStorage.getItem('lastAppWasRunning') if (lastAppWasRunning !== null) { lastAppWasRunning = new Date(lastAppWasRunning) } - parser.parseURL(feedUrl).then((response) => { - const latestBlog = response.items[0] - const latestPubDate = new Date(latestBlog.pubDate) + fetch('https://write.as/freetube/feed/') + .then(response => response.text()) + .then(response => { + const xmlDom = new DOMParser().parseFromString(response, 'application/xml') - if (lastAppWasRunning === null || latestPubDate > lastAppWasRunning) { - this.blogBannerMessage = this.$t('A new blog is now available, {blogTitle}. Click to view more', { blogTitle: latestBlog.title }) - this.latestBlogUrl = latestBlog.link - this.showBlogBanner = true - } + const latestBlog = xmlDom.querySelector('item') + const latestPubDate = new Date(latestBlog.querySelector('pubDate').textContent) - localStorage.setItem('lastAppWasRunning', new Date()) - }) + if (lastAppWasRunning === null || latestPubDate > lastAppWasRunning) { + const title = latestBlog.querySelector('title').textContent + + this.blogBannerMessage = this.$t('A new blog is now available, {blogTitle}. Click to view more', { blogTitle: title }) + this.latestBlogUrl = latestBlog.querySelector('link').textContent + this.showBlogBanner = true + } + + localStorage.setItem('lastAppWasRunning', new Date()) + }) } }, diff --git a/src/renderer/views/Subscriptions/Subscriptions.js b/src/renderer/views/Subscriptions/Subscriptions.js index 80b40d522..79c180e9b 100644 --- a/src/renderer/views/Subscriptions/Subscriptions.js +++ b/src/renderer/views/Subscriptions/Subscriptions.js @@ -9,7 +9,6 @@ import FtElementList from '../../components/ft-element-list/ft-element-list.vue' import FtChannelBubble from '../../components/ft-channel-bubble/ft-channel-bubble.vue' import ytch from 'yt-channel-info' -import Parser from 'rss-parser' import { MAIN_PROFILE_ID } from '../../../constants' import { calculatePublishedDate, showToast } from '../../helpers/utils' @@ -299,62 +298,40 @@ export default Vue.extend({ }) }, - getChannelVideosLocalRSS: function (channel, failedAttempts = 0) { - return new Promise((resolve, reject) => { - const parser = new Parser() - const feedUrl = `https://www.youtube.com/feeds/videos.xml?channel_id=${channel.id}` + getChannelVideosLocalRSS: async function (channel, failedAttempts = 0) { + const feedUrl = `https://www.youtube.com/feeds/videos.xml?channel_id=${channel.id}` - parser.parseURL(feedUrl).then(async (feed) => { - const items = await Promise.all(feed.items.map((video) => { - video.authorId = channel.id - video.videoId = video.id.replace('yt:video:', '') - video.type = 'video' - video.lengthSeconds = '0:00' - video.isRSS = true + try { + const response = await fetch(feedUrl) - video.publishedDate = new Date(video.pubDate) + if (response.status === 404) { + this.errorChannels.push(channel) + return [] + } - if (video.publishedDate.toString() === 'Invalid Date') { - video.publishedDate = new Date(video.isoDate) - } - - video.publishedText = video.publishedDate.toLocaleString() - - return video - })) - - resolve(items) - }).catch((err) => { - console.error(err) - if (err.toString().match(/404/)) { - this.errorChannels.push(channel) - resolve([]) - } else { - const errorMessage = this.$t('Local API Error (Click to copy)') - showToast(`${errorMessage}: ${err}`, 10000, () => { - this.copyToClipboard({ content: err }) - }) - switch (failedAttempts) { - case 0: - resolve(this.getChannelVideosLocalScraper(channel, failedAttempts + 1)) - break - case 1: - if (this.backendFallback) { - showToast(this.$t('Falling back to Invidious API')) - resolve(this.getChannelVideosInvidiousRSS(channel, failedAttempts + 1)) - } else { - resolve([]) - } - break - case 2: - resolve(this.getChannelVideosLocalScraper(channel, failedAttempts + 1)) - break - default: - resolve([]) - } - } + return await this.parseYouTubeRSSFeed(await response.text(), channel.id) + } catch (error) { + console.error(error) + const errorMessage = this.$t('Local API Error (Click to copy)') + showToast(`${errorMessage}: ${error}`, 10000, () => { + this.copyToClipboard({ content: error }) }) - }) + switch (failedAttempts) { + case 0: + return this.getChannelVideosLocalScraper(channel, failedAttempts + 1) + case 1: + if (this.backendFallback) { + this.showToast(this.$t('Falling back to Invidious API')) + return this.getChannelVideosInvidiousRSS(channel, failedAttempts + 1) + } else { + return [] + } + case 2: + return this.getChannelVideosLocalScraper(channel, failedAttempts + 1) + default: + return [] + } + } }, getChannelVideosInvidiousScraper: function (channel, failedAttempts = 0) { @@ -398,54 +375,72 @@ export default Vue.extend({ }) }, - getChannelVideosInvidiousRSS: function (channel, failedAttempts = 0) { - return new Promise((resolve, reject) => { - const parser = new Parser() - const feedUrl = `${this.currentInvidiousInstance}/feed/channel/${channel.id}` + getChannelVideosInvidiousRSS: async function (channel, failedAttempts = 0) { + const feedUrl = `${this.currentInvidiousInstance}/feed/channel/${channel.id}` - parser.parseURL(feedUrl).then(async (feed) => { - resolve(await Promise.all(feed.items.map((video) => { - video.authorId = channel.id - video.videoId = video.id.replace('yt:video:', '') - video.type = 'video' - video.publishedDate = new Date(video.pubDate) - video.publishedText = video.publishedDate.toLocaleString() - video.lengthSeconds = '0:00' - video.isRSS = true + try { + const response = await fetch(feedUrl) - return video - }))) - }).catch((err) => { - console.error(err) - const errorMessage = this.$t('Invidious API Error (Click to copy)') - showToast(`${errorMessage}: ${err}`, 10000, () => { - this.copyToClipboard({ content: err }) - }) - if (err.toString().match(/500/)) { - this.errorChannels.push(channel) - resolve([]) - } else { - switch (failedAttempts) { - case 0: - resolve(this.getChannelVideosInvidiousScraper(channel, failedAttempts + 1)) - break - case 1: - if (this.backendFallback) { - showToast(this.$t('Falling back to the local API')) - resolve(this.getChannelVideosLocalRSS(channel, failedAttempts + 1)) - } else { - resolve([]) - } - break - case 2: - resolve(this.getChannelVideosInvidiousScraper(channel, failedAttempts + 1)) - break - default: - resolve([]) - } - } + if (response.status === 500) { + this.errorChannels.push(channel) + return [] + } + + return await this.parseYouTubeRSSFeed(await response.text(), channel.id) + } catch (error) { + console.error(error) + const errorMessage = this.$t('Invidious API Error (Click to copy)') + showToast(`${errorMessage}: ${error}`, 10000, () => { + this.copyToClipboard({ content: error }) }) - }) + switch (failedAttempts) { + case 0: + return this.getChannelVideosInvidiousScraper(channel, failedAttempts + 1) + case 1: + if (this.backendFallback) { + this.showToast(this.$t('Falling back to the local API')) + return this.getChannelVideosLocalRSS(channel, failedAttempts + 1) + } else { + return [] + } + case 2: + return this.getChannelVideosInvidiousScraper(channel, failedAttempts + 1) + default: + return [] + } + } + }, + + async parseYouTubeRSSFeed(rssString, channelId) { + const xmlDom = new DOMParser().parseFromString(rssString, 'application/xml') + + const channelName = xmlDom.querySelector('author > name').textContent + const entries = xmlDom.querySelectorAll('entry') + + const promises = [] + + for (const entry of entries) { + promises.push(this.parseRSSEntry(entry, channelId, channelName)) + } + + return await Promise.all(promises) + }, + + async parseRSSEntry(entry, channelId, channelName) { + const published = new Date(entry.querySelector('published').textContent) + return { + authorId: channelId, + author: channelName, + // querySelector doesn't support xml namespaces so we have to use getElementsByTagName here + videoId: entry.getElementsByTagName('yt:videoId')[0].textContent, + title: entry.querySelector('title').textContent, + publishedDate: published, + publishedText: published.toLocaleString(), + viewCount: entry.getElementsByTagName('media:statistics')[0]?.getAttribute('views') || null, + type: 'video', + lengthSeconds: '0:00', + isRSS: true + } }, increaseLimit: function () { diff --git a/yarn.lock b/yarn.lock index 0afd4371c..c9211c00f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3553,7 +3553,7 @@ enquirer@^2.3.5: dependencies: ansi-colors "^4.1.1" -entities@^2.0.0, entities@^2.0.3: +entities@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== @@ -7027,14 +7027,6 @@ roarr@^2.15.3: semver-compare "^1.0.0" sprintf-js "^1.1.2" -rss-parser@^3.12.0: - version "3.12.0" - resolved "https://registry.yarnpkg.com/rss-parser/-/rss-parser-3.12.0.tgz#b8888699ea46304a74363fbd8144671b2997984c" - integrity sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A== - dependencies: - entities "^2.0.3" - xml2js "^0.4.19" - run-parallel@^1.1.9: version "1.2.0" resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" @@ -7104,7 +7096,7 @@ sass@^1.54.9: immutable "^4.0.0" source-map-js ">=0.6.2 <2.0.0" -sax@>=0.6.0, sax@^1.1.3, sax@^1.2.4: +sax@^1.1.3, sax@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -8482,14 +8474,6 @@ xml-name-validator@^4.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835" integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw== -xml2js@^0.4.19: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - xmlbuilder@>=11.0.1: version "15.1.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5" @@ -8500,11 +8484,6 @@ xmlbuilder@^9.0.7: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= -xmlbuilder@~11.0.0: - version "11.0.1" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" - integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== - xtend@^4.0.0, xtend@^4.0.1, xtend@^4.0.2, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"