Merge pull request #674 from GilgusMaximus/development

Comment scraper child process [Help needed]
This commit is contained in:
Luca Hohmann 2020-10-24 13:44:16 +02:00 committed by GitHub
commit a99bfd6a47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 129 additions and 69 deletions

View File

@ -111,6 +111,7 @@ const config = {
fs: 'empty',
net: 'empty',
tls: 'empty',
child_process: 'empty'
},
plugins: [
// new WriteFilePlugin(),

View File

@ -0,0 +1,23 @@
const ModuleScraper = require('yt-comment-scraper')
let scraper = null
let currentSort = null
let currentVideoId = null
process.on('message', (msg) => {
if (msg === 'end') {
process.exit(0)
}
if (msg.id !== currentVideoId || msg.sortNewest !== currentSort) {
if (scraper !== null) {
scraper.cleanupStatics()
}
currentSort = msg.sortNewest
currentVideoId = msg.id
scraper = new ModuleScraper(true, currentSort)
}
scraper.scrape_next_page_youtube_comments(currentVideoId).then((comments) => {
process.send({ comments: JSON.stringify(comments), error: null })
}).catch((error) => {
process.send({ comments: null, error: error })
})
})

View File

@ -5,7 +5,10 @@ import FtLoader from '../../components/ft-loader/ft-loader.vue'
import FtSelect from '../../components/ft-select/ft-select.vue'
import FtTimestampCatcher from '../../components/ft-timestamp-catcher/ft-timestamp-catcher.vue'
import autolinker from 'autolinker'
import CommentScraper from 'yt-comment-scraper'
import { fork } from 'child_process'
import path from 'path'
// eslint-disable-next-line
import commentControllerRelativePath from 'file-loader!../../../process/comment-module-controller.js'
export default Vue.extend({
name: 'WatchVideoComments',
@ -32,10 +35,16 @@ export default Vue.extend({
commentScraper: null,
nextPageToken: null,
commentData: [],
sortNewest: false
sortNewest: false,
commentProcess: null,
sortingChanged: false
}
},
computed: {
isDev: function () {
return process.env.NODE_ENV === 'development'
},
backendPreference: function () {
return this.$store.getters.getBackendPreference
},
@ -69,6 +78,12 @@ export default Vue.extend({
return (this.sortNewest) ? 'newest' : 'top'
}
},
beforeDestroy: function () {
if (this.commentProcess !== null) {
this.commentProcess.send('end')
}
},
methods: {
onTimestamp: function (timestamp) {
this.$emit('timestamp-event', timestamp)
@ -78,7 +93,9 @@ export default Vue.extend({
this.sortNewest = !this.sortNewest
switch (this.backendPreference) {
case 'local':
this.getCommentDataLocal(true)
console.log('In handle')
this.sortingChanged = true
this.getCommentDataLocal()
break
case 'invidious':
this.isLoading = true
@ -88,11 +105,11 @@ export default Vue.extend({
}
},
getCommentData: function (sortChanged = false) {
getCommentData: function () {
this.isLoading = true
switch (this.backendPreference) {
case 'local':
this.getCommentDataLocal(sortChanged)
this.getCommentDataLocal()
break
case 'invidious':
this.getCommentDataInvidious(this.nextPageToken)
@ -126,13 +143,24 @@ export default Vue.extend({
}
},
getCommentDataLocal: function (sortChanged = false) {
if (this.commentScraper === null || sortChanged === true) {
this.commentScraper = new CommentScraper(false, this.sortNewest)
this.commentData = []
getCommentDataLocal: function () {
// we need the path from the working directory to fork correctly
if (this.commentProcess === null) {
let modulePath
if (this.isDev) {
modulePath = '../../../process/comment-module-controller.js'
} else {
modulePath = commentControllerRelativePath
}
this.commentScraper.scrape_next_page_youtube_comments(this.id).then((response) => {
if (response === null) {
this.commentProcess = fork(path.join(__dirname, modulePath), ['args'], {
stdio: ['pipe', 'pipe', 'pipe', 'ipc']
})
this.commentProcess.on('message', (msg) => {
if (msg.error === null) {
const commentJSON = JSON.parse(msg.comments)
if (commentJSON === null) {
this.showToast({
message: this.$t('Comments.No more comments available'),
time: 7000,
@ -140,10 +168,9 @@ export default Vue.extend({
}
})
this.isLoading = false
return
}
console.log(response)
const commentData = response.map((comment) => {
} else {
// console.log(msg.comments)
const commentData = commentJSON.map((comment) => {
comment.showReplies = false
comment.dataType = 'local'
this.toLocalePublicationString({
@ -169,18 +196,23 @@ export default Vue.extend({
})
return comment
})
if (this.sortingChanged) {
this.commentData = []
this.sortingChanged = false
}
this.commentData = this.commentData.concat(commentData)
this.isLoading = false
this.showComments = true
this.nextPageToken = ''
}).catch((err) => {
console.log(err)
}
} else {
console.log(msg.error)
const errorMessage = this.$t('Local API Error (Click to copy)')
this.showToast({
message: `${errorMessage}: ${err}`,
message: `${errorMessage}: ${msg.error}`,
time: 10000,
action: () => {
navigator.clipboard.writeText(err)
navigator.clipboard.writeText(msg.error)
}
})
if (this.backendFallback && this.backendPreference === 'local') {
@ -191,7 +223,11 @@ export default Vue.extend({
} else {
this.isLoading = false
}
}
})
}
this.commentProcess.send({ id: this.id, sortNewest: this.sortNewest })
},
getCommentDataInvidious: function () {