From b7846493ef07b364cd81f2c6fa2a3f81b2ee6aa8 Mon Sep 17 00:00:00 2001 From: Preston Date: Wed, 20 Jan 2021 10:54:44 -0500 Subject: [PATCH] Update comment scraper logic --- package-lock.json | 24 +-- package.json | 2 +- .../watch-video-comments.js | 157 ++++++++---------- .../watch-video-comments.vue | 6 +- 4 files changed, 85 insertions(+), 104 deletions(-) diff --git a/package-lock.json b/package-lock.json index a12e9ff0c..f00129dcf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9014,8 +9014,7 @@ "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" }, "highlight.js": { "version": "9.18.3", @@ -9167,11 +9166,6 @@ "util.promisify": "1.0.0" } }, - "html2json": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/html2json/-/html2json-1.0.2.tgz", - "integrity": "sha1-ydbSAvplQCOGwgKzRc9RvOgO0e8=" - }, "htmlparser2": { "version": "3.10.1", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", @@ -12586,6 +12580,14 @@ } } }, + "node-html-parser": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-2.0.2.tgz", + "integrity": "sha512-N2000Ho9dkabvRZcyiwm6zOpdiAzxAxcJ0Z0WNoh/yXHG0YCuiK2WpNQfN+9vheLNY/h/It11Gk7uwT4QTfk9Q==", + "requires": { + "he": "1.2.0" + } + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -19191,12 +19193,12 @@ } }, "yt-comment-scraper": { - "version": "1.3.11", - "resolved": "https://registry.npmjs.org/yt-comment-scraper/-/yt-comment-scraper-1.3.11.tgz", - "integrity": "sha512-F7a1/BAjASsX3MheAqO1uvY0sPfstLKmkHETiFyDe3Hp+qUMyniuPmAvXrZPOanvBHyb6sH5+d3wVj/S9dD/ug==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yt-comment-scraper/-/yt-comment-scraper-2.0.0.tgz", + "integrity": "sha512-goa9Z5REPXRKddKO6MJUzaa/cBgdQPj/akIGbyb1R5KnOgl4bLy6d4nD1pVUhLkP0Z8aEqwJAMmYpPBjeXTOIg==", "requires": { "axios": "^0.21.1", - "html2json": "^1.0.2" + "node-html-parser": "^2.0.2" }, "dependencies": { "axios": { diff --git a/package.json b/package.json index 7a12076bb..40caf0174 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "youtube-chat": "^1.1.0", "youtube-suggest": "^1.1.0", "yt-channel-info": "^1.2.0", - "yt-comment-scraper": "^1.3.11", + "yt-comment-scraper": "^2.0.0", "yt-dash-manifest-generator": "^1.1.0", "yt-trending-scraper": "1.1.0", "yt-xml2vtt": "^1.2.0", diff --git a/src/renderer/components/watch-video-comments/watch-video-comments.js b/src/renderer/components/watch-video-comments/watch-video-comments.js index b696370f9..da403115f 100644 --- a/src/renderer/components/watch-video-comments/watch-video-comments.js +++ b/src/renderer/components/watch-video-comments/watch-video-comments.js @@ -5,10 +5,7 @@ import FtLoader from '../../components/ft-loader/ft-loader.vue' import FtSelect from '../../components/ft-select/ft-select.vue' import FtTimestampCatcher from '../../components/ft-timestamp-catcher/ft-timestamp-catcher.vue' import autolinker from 'autolinker' -import { fork } from 'child_process' -import path from 'path' -// eslint-disable-next-line -import commentControllerRelativePath from 'file-loader!../../../process/comment-module-controller.js' +import ytcm from 'yt-comment-scraper' export default Vue.extend({ name: 'WatchVideoComments', @@ -93,8 +90,9 @@ export default Vue.extend({ this.sortNewest = !this.sortNewest switch (this.backendPreference) { case 'local': - console.log('In handle') - this.sortingChanged = true + this.isLoading = true + this.commentData = [] + this.nextPageToken = undefined this.getCommentDataLocal() break case 'invidious': @@ -122,7 +120,6 @@ export default Vue.extend({ this.showToast({ message: this.$t('Comments.There are no more comments for this video') }) - this.getCommentData() } else { this.getCommentData() } @@ -144,88 +141,72 @@ export default Vue.extend({ }, getCommentDataLocal: function () { - // we need the path from the working directory to fork correctly - if (this.commentProcess === null) { - let modulePath - if (this.isDev) { - modulePath = '../../../process/comment-module-controller.js' - } else { - modulePath = commentControllerRelativePath - } - - this.commentProcess = fork(path.join(__dirname, modulePath)) - - this.commentProcess.on('message', (msg) => { - if (msg.error === null) { - const commentJSON = JSON.parse(msg.comments) - if (commentJSON === null) { - this.showToast({ - message: this.$t('Comments.No more comments available'), - time: 7000, - action: () => { - } - }) - this.isLoading = false - } else { - // console.log(msg.comments) - const commentData = commentJSON.map((comment) => { - comment.showReplies = false - comment.dataType = 'local' - this.toLocalePublicationString({ - publishText: (comment.time + ' ago'), - templateString: this.$t('Video.Publicationtemplate'), - timeStrings: this.$t('Video.Published'), - liveStreamString: this.$t('Video.Watching'), - upcomingString: this.$t('Video.Published.Upcoming'), - isLive: false, - isUpcoming: false, - isRSS: false - }).then((data) => { - comment.time = data - }).catch((error) => { - console.error(error) - }) - if (this.hideCommentLikes) { - comment.likes = null - } - comment.text = autolinker.link(comment.text) - comment.replies.forEach((reply) => { - reply.text = autolinker.link(reply.text) - }) - return comment - }) - if (this.sortingChanged) { - this.commentData = [] - this.sortingChanged = false - } - this.commentData = this.commentData.concat(commentData) - this.isLoading = false - this.showComments = true - this.nextPageToken = '' - } - } else { - console.log(msg.error) - const errorMessage = this.$t('Local API Error (Click to copy)') - this.showToast({ - message: `${errorMessage}: ${msg.error}`, - time: 10000, - action: () => { - navigator.clipboard.writeText(msg.error) - } - }) - if (this.backendFallback && this.backendPreference === 'local') { - this.showToast({ - message: this.$t('Falling back to Invidious API') - }) - this.getCommentDataInvidious() - } else { - this.isLoading = false - } - } - }) + const payload = { + videoId: this.id, + setCookie: false, + sortByNewest: this.sortNewest, + continuation: this.nextPageToken ? this.nextPageToken : undefined } - this.commentProcess.send({ id: this.id, sortNewest: this.sortNewest }) + ytcm.getComments(payload).then((response) => { + console.log(response) + const commentData = response.comments.map((comment) => { + comment.showReplies = false + comment.dataType = 'local' + this.toLocalePublicationString({ + publishText: (comment.time + ' ago'), + templateString: this.$t('Video.Publicationtemplate'), + timeStrings: this.$t('Video.Published'), + liveStreamString: this.$t('Video.Watching'), + upcomingString: this.$t('Video.Published.Upcoming'), + isLive: false, + isUpcoming: false, + isRSS: false + }).then((data) => { + comment.time = data + }).catch((error) => { + console.error(error) + }) + if (this.hideCommentLikes) { + comment.likes = null + } + comment.text = autolinker.link(comment.text) + + if (comment.numReplies > 0) { + comment.replies.forEach((reply) => { + reply.text = autolinker.link(reply.text) + }) + } + + return comment + }) + if (this.sortingChanged) { + this.commentData = [] + this.sortingChanged = false + } + this.commentData = this.commentData.concat(commentData) + this.isLoading = false + this.showComments = true + this.nextPageToken = response.continuation + }).catch((err) => { + console.log(err) + const errorMessage = this.$t('Local API Error (Click to copy)') + this.showToast({ + message: `${errorMessage}: ${err}`, + time: 10000, + action: () => { + navigator.clipboard.writeText(err) + } + }) + if (this.backendFallback && this.backendPreference === 'local') { + this.showToast({ + message: this.$t('Falling back to Invidious API') + }) + this.getCommentDataInvidious() + } else { + this.isLoading = false + } + }) }, getCommentDataInvidious: function () { @@ -267,7 +248,6 @@ export default Vue.extend({ return comment }) - console.log(commentData) this.commentData = this.commentData.concat(commentData) this.nextPageToken = response.continuation this.isLoading = false @@ -327,7 +307,6 @@ export default Vue.extend({ return comment }) - console.log(commentData) this.commentData[index].replies = commentData this.commentData[index].showReplies = true this.isLoading = false diff --git a/src/renderer/components/watch-video-comments/watch-video-comments.vue b/src/renderer/components/watch-video-comments/watch-video-comments.vue index bb2ff828a..5c490d716 100644 --- a/src/renderer/components/watch-video-comments/watch-video-comments.vue +++ b/src/renderer/components/watch-video-comments/watch-video-comments.vue @@ -1,8 +1,5 @@