Added implementation of child process for comment scraping

This commit is contained in:
Luca 2020-10-15 11:51:11 +02:00
parent 91e10a4716
commit 5b83c45d21
2 changed files with 114 additions and 68 deletions

View File

@ -0,0 +1,24 @@
const ModuleScraper = require('yt-comment-scraper')
const fs = require('fs')
let scraper = null
let currentSort = null
let currentVideoId = null
fs.writeFileSync('D:/Workspace/textELEC.txt', 'hallololol')
process.on('message', (msg) => {
if (msg === 'end') {
process.exit(0)
}
if (msg.id !== currentVideoId || msg.sortNewest !== currentSort) {
if (scraper !== null) {
scraper.cleanupStatics()
}
currentSort = msg.sortNewest
currentVideoId = msg.id
scraper = new ModuleScraper(true, currentSort)
}
scraper.scrape_next_page_youtube_comments(currentVideoId).then((comments) => {
process.send({ comments: JSON.stringify(comments), error: null })
}).catch((error) => {
process.send({ comments: null, error: error })
})
})

View File

@ -5,7 +5,8 @@ import FtLoader from '../../components/ft-loader/ft-loader.vue'
import FtSelect from '../../components/ft-select/ft-select.vue'
import FtTimestampCatcher from '../../components/ft-timestamp-catcher/ft-timestamp-catcher.vue'
import autolinker from 'autolinker'
import CommentScraper from 'yt-comment-scraper'
import { fork } from 'child_process'
import path from 'path'
export default Vue.extend({
name: 'WatchVideoComments',
@ -28,7 +29,9 @@ export default Vue.extend({
commentScraper: null,
nextPageToken: null,
commentData: [],
sortNewest: false
sortNewest: false,
commentProcess: null,
sortingChanged: false
}
},
computed: {
@ -65,6 +68,12 @@ export default Vue.extend({
return (this.sortNewest) ? 'newest' : 'top'
}
},
beforeDestroy: function () {
if (this.commentProcess !== null) {
this.commentProcess.send('end')
}
},
methods: {
onTimestamp: function (timestamp) {
this.$emit('timestamp-event', timestamp)
@ -74,7 +83,9 @@ export default Vue.extend({
this.sortNewest = !this.sortNewest
switch (this.backendPreference) {
case 'local':
this.getCommentDataLocal(true)
console.log('In handle')
this.sortingChanged = true
this.getCommentDataLocal()
break
case 'invidious':
this.isLoading = true
@ -84,11 +95,11 @@ export default Vue.extend({
}
},
getCommentData: function (sortChanged = false) {
getCommentData: function () {
this.isLoading = true
switch (this.backendPreference) {
case 'local':
this.getCommentDataLocal(sortChanged)
this.getCommentDataLocal()
break
case 'invidious':
this.getCommentDataInvidious(this.nextPageToken)
@ -122,13 +133,17 @@ export default Vue.extend({
}
},
getCommentDataLocal: function (sortChanged = false) {
if (this.commentScraper === null || sortChanged === true) {
this.commentScraper = new CommentScraper(false, this.sortNewest)
this.commentData = []
}
this.commentScraper.scrape_next_page_youtube_comments(this.id).then((response) => {
if (response === null) {
getCommentDataLocal: function () {
// we need the path from the working directory to fork correctly
if (this.commentProcess === null) {
this.commentProcess = fork('./src/process/comment-module-controller.js', ['args'], {
stdio: 'pipe',
cwd: path.join(__dirname, '../../')
})
this.commentProcess.on('message', (msg) => {
if (msg.error === null) {
const commentJSON = JSON.parse(msg.comments)
if (commentJSON === null) {
this.showToast({
message: this.$t('Comments.No more comments available'),
time: 7000,
@ -136,10 +151,9 @@ export default Vue.extend({
}
})
this.isLoading = false
return
}
console.log(response)
const commentData = response.map((comment) => {
} else {
// console.log(msg.comments)
const commentData = commentJSON.map((comment) => {
comment.showReplies = false
comment.dataType = 'local'
this.toLocalePublicationString({
@ -165,18 +179,23 @@ export default Vue.extend({
})
return comment
})
if (this.sortingChanged) {
this.commentData = []
this.sortingChanged = false
}
this.commentData = this.commentData.concat(commentData)
this.isLoading = false
this.showComments = true
this.nextPageToken = ''
}).catch((err) => {
console.log(err)
}
} else {
console.log(msg.error)
const errorMessage = this.$t('Local API Error (Click to copy)')
this.showToast({
message: `${errorMessage}: ${err}`,
message: `${errorMessage}: ${msg.error}`,
time: 10000,
action: () => {
navigator.clipboard.writeText(err)
navigator.clipboard.writeText(msg.error)
}
})
if (this.backendFallback && this.backendPreference === 'local') {
@ -187,7 +206,10 @@ export default Vue.extend({
} else {
this.isLoading = false
}
}
})
}
this.commentProcess.send({ id: this.id, sortNewest: this.sortNewest })
},
getCommentDataInvidious: function () {