Migrate search to YouTube.js (#3028)

* Migrate search to YouTube.js

* Fix linting issue
This commit is contained in:
absidue 2023-01-05 04:54:08 +01:00 committed by GitHub
parent b157495552
commit 156176aca8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 248 additions and 270 deletions

View File

@ -62,13 +62,6 @@ const config = {
'./dist/**/*',
'!dist/web/*',
'!node_modules/**/*',
// renderer
'node_modules/{miniget,ytsr}/**/*',
'!**/README.md',
'!**/*.js.map',
'!**/*.d.ts',
],
dmg: {
contents: [

View File

@ -32,10 +32,6 @@ const config = {
path: path.join(__dirname, '../dist'),
filename: '[name].js',
},
// webpack spits out errors while inlining ytsr as
// they dynamically import their package.json file to extract the bug report URL
// the error: "Critical dependency: the request of a dependency is an expression"
externals: ['ytsr'],
module: {
rules: [
{

View File

@ -24,8 +24,7 @@ const config = {
},
externals: {
electron: '{}',
'youtubei.js': '{}',
ytsr: '{}'
'youtubei.js': '{}'
},
module: {
rules: [

View File

@ -83,8 +83,7 @@
"youtubei.js": "^2.7.0",
"yt-channel-info": "^3.2.1",
"yt-dash-manifest-generator": "1.1.0",
"ytdl-core": "https://github.com/absidue/node-ytdl-core#fix-likes-extraction",
"ytsr": "^3.8.0"
"ytdl-core": "https://github.com/absidue/node-ytdl-core#fix-likes-extraction"
},
"devDependencies": {
"@babel/core": "^7.20.7",

View File

@ -20,6 +20,7 @@ export default Vue.extend({
channelName: '',
subscriberCount: 0,
videoCount: '',
handle: null,
uploadedTime: '',
description: ''
}
@ -39,7 +40,7 @@ export default Vue.extend({
}
},
mounted: function () {
if (typeof (this.data.avatars) !== 'undefined') {
if (this.data.dataSource === 'local' || typeof (this.data.avatars) !== 'undefined') {
this.parseLocalData()
} else {
this.parseInvidiousData()
@ -47,7 +48,7 @@ export default Vue.extend({
},
methods: {
parseLocalData: function () {
this.thumbnail = this.data.bestAvatar.url
this.thumbnail = this.data.thumbnail ?? this.data.bestAvatar.url
if (!this.thumbnail.includes('https:')) {
this.thumbnail = `https:${this.thumbnail}`
@ -66,6 +67,10 @@ export default Vue.extend({
this.videoCount = Intl.NumberFormat(this.currentLocale).format(this.data.videos)
}
if (this.data.handle) {
this.handle = this.data.handle
}
this.description = this.data.descriptionShort
},

View File

@ -1 +1,6 @@
@use '../../scss-partials/_ft-list-item';
.handle {
color: inherit;
text-decoration: none;
}

View File

@ -31,7 +31,15 @@
>
{{ subscriberCount }} subscribers -
</span>
<router-link
v-if="handle !== null"
class="handle"
:to="`/channel/${id}`"
>
{{ handle }}
</router-link>
<span
v-else
class="videoCount"
>
{{ videoCount }} videos

View File

@ -56,7 +56,10 @@ export default Vue.extend({
}
},
mounted: function () {
if (typeof (this.data.owner) === 'object') {
// temporary until we've migrated the whole local API to youtubei.js
if (this.data.dataSource === 'local') {
this.parseLocalDataNew()
} else if (typeof (this.data.owner) === 'object') {
this.parseLocalData()
} else {
this.parseInvidiousData()
@ -98,6 +101,17 @@ export default Vue.extend({
this.videoCount = this.data.length
},
// TODO: after the local API is fully switched to YouTube.js
// cleanup the old local API stuff
parseLocalDataNew: function () {
this.title = this.data.title
this.thumbnail = this.data.thumbnail
this.channelName = this.data.channelName
this.channelLink = this.data.channelId
this.playlistLink = this.data.playlistId
this.videoCount = this.data.videoCount
},
...mapActions([
'openInExternalPlayer'
])

View File

@ -385,7 +385,9 @@ export default Vue.extend({
this.isPremium = this.data.premium || false
this.viewCount = this.data.viewCount
if (typeof (this.data.premiereTimestamp) !== 'undefined') {
if (typeof this.data.premiereDate !== 'undefined') {
this.publishedText = this.data.premiereDate.toLocaleString()
} else if (typeof (this.data.premiereTimestamp) !== 'undefined') {
this.publishedText = new Date(this.data.premiereTimestamp * 1000).toLocaleString()
} else {
this.publishedText = this.data.publishedText

View File

@ -27,9 +27,12 @@
<div
v-if="isLive || duration !== '0:00'"
class="videoDuration"
:class="{ live: isLive }"
:class="{
live: isLive,
upcoming: isUpcoming
}"
>
{{ isLive ? $t("Video.Live") : duration }}
{{ isLive ? $t("Video.Live") : (isUpcoming ? $t("Video.Upcoming") : duration) }}
</div>
<ft-icon-button
v-if="externalPlayer !== ''"

View File

@ -15,9 +15,10 @@ import { extractNumberFromString, getUserDataPath } from '../utils'
* @param {object} options
* @param {boolean} options.withPlayer set to true to get an Innertube instance that can decode the streaming URLs
* @param {string|undefined} options.location the geolocation to pass to YouTube get different content
* @param {boolean} options.safetyMode whether to hide mature content
* @returns the Innertube instance
*/
async function createInnertube(options = { withPlayer: false, location: undefined }) {
async function createInnertube(options = { withPlayer: false, location: undefined, safetyMode: false }) {
let cache
if (options.withPlayer) {
const userData = await getUserDataPath()
@ -27,6 +28,7 @@ async function createInnertube(options = { withPlayer: false, location: undefine
return await Innertube.create({
retrieve_player: !!options.withPlayer,
location: options.location,
enable_safety_mode: !!options.safetyMode,
// use browser fetch
fetch: (input, init) => fetch(input, init),
cache
@ -74,7 +76,7 @@ export async function getLocalTrending(location, tab, instance) {
const results = resultsInstance.videos
.filter((video) => video.type === 'Video')
.map(parseLocalListVideo)
.map(parseListVideo)
return {
results,
@ -82,6 +84,54 @@ export async function getLocalTrending(location, tab, instance) {
}
}
/**
* @param {string} query
* @param {object} filters
* @param {boolean} safetyMode
*/
export async function getLocalSearchResults(query, filters, safetyMode) {
const innertube = await createInnertube({ safetyMode })
const response = await innertube.search(query, convertSearchFilters(filters))
return handleSearchResponse(response)
}
/**
* @typedef {import('youtubei.js/dist/src/parser/youtube/Search').default} Search
*/
/**
* @param {Search} continuationData
*/
export async function getLocalSearchContinuation(continuationData) {
const response = await continuationData.getContinuation()
return handleSearchResponse(response)
}
/**
* @param {Search} response
*/
function handleSearchResponse(response) {
if (!response.results) {
return {
results: [],
continuationData: null
}
}
const results = response.results
.filter((item) => {
return item.type === 'Video' || item.type === 'Channel' || item.type === 'Playlist'
})
.map((item) => parseListItem(item))
return {
results,
continuationData: response.has_continuation ? response : null
}
}
/**
* @typedef {import('youtubei.js/dist/src/parser/classes/PlaylistVideo').default} PlaylistVideo
*/
@ -106,7 +156,7 @@ export function parseLocalPlaylistVideo(video) {
/**
* @param {Video} video
*/
function parseLocalListVideo(video) {
function parseListVideo(video) {
return {
type: 'video',
videoId: video.id,
@ -117,6 +167,96 @@ function parseLocalListVideo(video) {
viewCount: extractNumberFromString(video.view_count.text),
publishedText: video.published.text,
lengthSeconds: isNaN(video.duration.seconds) ? '' : video.duration.seconds,
liveNow: video.is_live
liveNow: video.is_live,
isUpcoming: video.is_upcoming || video.is_premiere,
premiereDate: video.upcoming
}
}
/**
* @typedef {import('youtubei.js/dist/src/parser/helpers').YTNode} YTNode
* @typedef {import('youtubei.js/dist/src/parser/classes/Channel').default} Channel
* @typedef {import('youtubei.js/dist/src/parser/classes/Playlist').default} Playlist
*/
/**
* @param {YTNode} item
*/
function parseListItem(item) {
switch (item.type) {
case 'Video':
return parseListVideo(item)
case 'Channel': {
/** @type {Channel} */
const channel = item
// see upstream TODO: https://github.com/LuanRT/YouTube.js/blob/main/src/parser/classes/Channel.ts#L33
// according to https://github.com/iv-org/invidious/issues/3514#issuecomment-1368080392
// the response can be the new or old one, so we currently need to handle both here
let subscribers
let videos = null
let handle = null
if (channel.subscribers.text.startsWith('@')) {
subscribers = channel.videos.text
handle = channel.subscribers.text
} else {
subscribers = channel.subscribers.text
videos = channel.videos.text
}
return {
type: 'channel',
dataSource: 'local',
thumbnail: channel.author.best_thumbnail?.url,
name: channel.author.name,
channelID: channel.author.id,
subscribers,
videos,
handle,
descriptionShort: channel.description_snippet.text
}
}
case 'Playlist': {
/** @type {Playlist} */
const playlist = item
return {
type: 'playlist',
dataSource: 'local',
title: playlist.title,
thumbnail: playlist.thumbnails[0].url,
channelName: playlist.author.name,
channelId: playlist.author.id,
playlistId: playlist.id,
videoCount: extractNumberFromString(playlist.video_count.text)
}
}
}
}
function convertSearchFilters(filters) {
const convertedFilters = {}
// some of the fields have different names and
// others have empty strings that we don't want to pass to youtubei.js
if (filters) {
if (filters.sortBy) {
convertedFilters.sort_by = filters.sortBy
}
if (filters.time) {
convertedFilters.upload_date = filters.time
}
if (filters.type) {
convertedFilters.type = filters.type
}
if (filters.duration) {
convertedFilters.type = filters.duration
}
}
return convertedFilters
}

View File

@ -481,7 +481,7 @@ export async function getPicturesPath() {
export function extractNumberFromString(str) {
if (typeof str === 'string') {
return parseInt(str.replace(/\D+/, ''))
return parseInt(str.replaceAll(/\D+/g, ''))
} else {
return NaN
}

View File

@ -287,6 +287,7 @@ $watched-transition-duration: 0.5s;
}
.videoWatched,
.live {
.live,
.upcoming {
text-transform: uppercase;
}

View File

@ -1,15 +1,10 @@
import ytdl from 'ytdl-core'
import ytsr from 'ytsr'
import { SocksProxyAgent } from 'socks-proxy-agent'
import { HttpsProxyAgent } from 'https-proxy-agent'
import { HttpProxyAgent } from 'http-proxy-agent'
import { searchFiltersMatch } from '../../helpers/utils'
const state = {
isYtSearchRunning: false
}
const state = {}
const getters = {}
@ -41,156 +36,6 @@ function createProxyAgent(protocol, hostname, port) {
}
const actions = {
ytSearch ({ commit, dispatch, rootState }, payload) {
return new Promise((resolve, reject) => {
if (state.isYtSearchRunning) {
resolve(false)
}
if (typeof payload.options.nextpageRef !== 'undefined') {
const continuation = payload.options.nextpageRef
const nextPageResults = ytsr.continueReq(continuation)
resolve(nextPageResults)
return
}
const defaultFilters = {
sortBy: 'relevance',
time: '',
type: 'all',
duration: ''
}
const settings = rootState.settings
if (settings.useProxy) {
const agent = createProxyAgent(settings.proxyProtocol, settings.proxyHostname, settings.proxyPort)
payload.options.requestOptions = { agent }
}
commit('toggleIsYtSearchRunning')
if (!searchFiltersMatch(defaultFilters, rootState.utils.searchSettings)) {
dispatch('ytSearchGetFilters', payload).then((filter) => {
if (typeof (payload.options.nextpageRef) === 'undefined' && filter !== payload.query) {
payload.options.nextpageRef = filter
}
const query = filter || payload.query
ytsr(query, payload.options).then((result) => {
resolve(result)
}).catch((err) => {
console.error(err)
reject(err)
}).finally(() => {
commit('toggleIsYtSearchRunning')
})
}).catch((err) => {
console.error(err)
commit('toggleIsYtSearchRunning')
reject(err)
})
} else {
ytsr(payload.query, payload.options).then((result) => {
resolve(result)
}).catch((err) => {
console.error(err)
reject(err)
}).finally(() => {
commit('toggleIsYtSearchRunning')
})
}
})
},
async ytSearchGetFilters ({ rootState }, payload) {
let options = null
let agent = null
const settings = rootState.settings
if (settings.useProxy) {
agent = createProxyAgent(settings.proxyProtocol, settings.proxyHostname, settings.proxyPort)
}
options = {
requestOptions: { agent }
}
let filter = await ytsr.getFilters(payload.query, options)
let filterUrl = null
let searchSettings = payload.searchSettings
if (typeof (searchSettings) === 'undefined') {
searchSettings = rootState.utils.searchSettings
}
if (searchSettings.sortBy !== 'relevance') {
let filterValue
switch (searchSettings.sortBy) {
case 'rating':
filterValue = 'Rating'
break
case 'upload_date':
filterValue = 'Upload date'
break
case 'view_count':
filterValue = 'View count'
break
}
filterUrl = filter.get('Sort by').get(filterValue).url
filter = await ytsr.getFilters(filterUrl, options)
}
if (searchSettings.duration !== '') {
let filterValue = null
if (searchSettings.duration === 'short') {
filterValue = 'Under 4 minutes'
} else if (searchSettings.duration === 'long') {
filterValue = 'Over 20 minutes'
}
filterUrl = filter.get('Duration').get(filterValue).url
filter = await ytsr.getFilters(filterUrl, options)
}
if (searchSettings.time !== '') {
let filterValue = null
switch (searchSettings.time) {
case 'hour':
filterValue = 'Last hour'
break
case 'today':
filterValue = 'Today'
break
case 'week':
filterValue = 'This week'
break
case 'month':
filterValue = 'This month'
break
case 'year':
filterValue = 'This year'
break
}
filterUrl = filter.get('Upload date').get(filterValue).url
filter = await ytsr.getFilters(filterUrl, options)
}
if (searchSettings.type !== 'all') {
const filterValue = searchSettings.type.charAt(0).toUpperCase() + searchSettings.type.slice(1)
filterUrl = filter.get('Type').get(filterValue).url
filter = await ytsr.getFilters(filterUrl, options)
}
return new Promise((resolve, reject) => {
resolve(filterUrl)
})
},
ytGetVideoInformation ({ rootState }, videoId) {
return new Promise((resolve, reject) => {
let agent = null
@ -212,11 +57,7 @@ const actions = {
}
}
const mutations = {
toggleIsYtSearchRunning (state) {
state.isYtSearchRunning = !state.isYtSearchRunning
}
}
const mutations = {}
export default {
state,

View File

@ -3,8 +3,8 @@ import { mapActions } from 'vuex'
import FtLoader from '../../components/ft-loader/ft-loader.vue'
import FtCard from '../../components/ft-card/ft-card.vue'
import FtElementList from '../../components/ft-element-list/ft-element-list.vue'
import { timeToSeconds } from 'youtubei.js/dist/src/utils/Utils'
import { copyToClipboard, searchFiltersMatch, showToast } from '../../helpers/utils'
import { getLocalSearchContinuation, getLocalSearchResults } from '../../helpers/api/local'
export default Vue.extend({
name: 'Search',
@ -20,7 +20,7 @@ export default Vue.extend({
amountOfResults: 0,
query: '',
searchPage: 1,
nextPageRef: '',
nextPageRef: null,
lastSearchQuery: '',
searchSettings: {},
shownResults: []
@ -65,7 +65,6 @@ export default Vue.extend({
const payload = {
query: query,
nextPage: false,
options: {},
searchSettings: searchSettings
}
@ -87,7 +86,6 @@ export default Vue.extend({
const payload = {
query: this.query,
nextPage: false,
options: {},
searchSettings: this.searchSettings
}
@ -122,86 +120,32 @@ export default Vue.extend({
}
},
performSearchLocal: function (payload) {
if (!payload.nextPage) {
this.isLoading = true
payload.options.pages = 1
}
performSearchLocal: async function (payload) {
this.isLoading = true
payload.options.safeSearch = this.showFamilyFriendlyOnly
try {
const { results, continuationData } = await getLocalSearchResults(payload.query, payload.searchSettings, this.showFamilyFriendlyOnly)
this.ytSearch(payload).then((result) => {
if (!result) {
if (results.length === 0) {
return
}
this.apiUsed = 'local'
this.amountOfResults = result.results
this.shownResults = results
this.nextPageRef = continuationData
const returnData = result.items.filter((item) => {
if (typeof item !== 'undefined') {
return item.type === 'video' || item.type === 'channel' || item.type === 'playlist'
}
return null
})
const dataToShow = []
returnData.forEach((video) => {
if (video.type === 'video') {
const authId = video.author.channelID
const publishDate = video.uploadedAt
let videoDuration = video.duration
const videoId = video.id
if (videoDuration !== null && videoDuration !== '' && videoDuration !== 'LIVE' && videoDuration !== 'UPCOMING' && videoDuration !== 'PREMIERE') {
videoDuration = timeToSeconds(video.duration)
}
dataToShow.push(
{
videoId: videoId,
title: video.title,
type: 'video',
author: video.author.name,
authorId: authId,
authorUrl: video.author.url,
videoThumbnails: video.thumbnail,
description: video.description,
viewCount: video.views,
published: publishDate,
publishedText: publishDate,
lengthSeconds: videoDuration,
liveNow: video.isLive || videoDuration === 'LIVE',
paid: false,
premium: false,
isUpcoming: videoDuration === 'UPCOMING' || videoDuration === 'PREMIERE',
timeText: videoDuration
}
)
} else {
dataToShow.push(video)
}
})
if (payload.nextPage) {
this.shownResults = this.shownResults.concat(dataToShow)
} else {
this.shownResults = dataToShow
}
this.nextPageRef = result.continuation
this.isLoading = false
const historyPayload = {
query: payload.query,
data: this.shownResults,
searchSettings: this.searchSettings,
nextPageRef: result.continuation,
amountOfResults: result.results
nextPageRef: this.nextPageRef
}
this.$store.commit('addToSessionSearchHistory', historyPayload)
}).catch((err) => {
} catch (err) {
console.error(err)
const errorMessage = this.$t('Local API Error (Click to copy)')
showToast(`${errorMessage}: ${err}`, 10000, () => {
@ -213,7 +157,43 @@ export default Vue.extend({
} else {
this.isLoading = false
}
})
}
},
getNextpageLocal: async function (payload) {
try {
const { results, continuationData } = getLocalSearchContinuation(payload.options.nextPageRef)
if (results.length === 0) {
return
}
this.apiUsed = 'local'
this.shownResults = this.shownResults.concat(results)
this.nextPageRef = continuationData
const historyPayload = {
query: payload.query,
data: this.shownResults,
searchSettings: this.searchSettings,
nextPageRef: this.nextPageRef
}
this.$store.commit('addToSessionSearchHistory', historyPayload)
} catch (err) {
console.error(err)
const errorMessage = this.$t('Local API Error (Click to copy)')
showToast(`${errorMessage}: ${err}`, 10000, () => {
copyToClipboard(err)
})
if (this.backendPreference === 'local' && this.backendFallback) {
showToast(this.$t('Falling back to Invidious API'))
this.performSearchInvidious(payload)
} else {
this.isLoading = false
}
}
},
performSearchInvidious: function (payload) {
@ -281,19 +261,18 @@ export default Vue.extend({
nextPage: function () {
const payload = {
query: this.query,
nextPage: true,
searchSettings: this.searchSettings,
options: {
nextpageRef: this.nextPageRef
nextPageRef: this.nextPageRef
}
}
if (this.apiUsed === 'local') {
if (this.amountOfResults <= this.shownResults.length) {
showToast(this.$t('Search Filters.There are no more results for this search'))
} else {
if (this.nextPageRef !== null) {
showToast(this.$t('Search Filters["Fetching results. Please wait"]'))
this.performSearchLocal(payload)
this.getNextpageLocal(payload)
} else {
showToast(this.$t('Search Filters.There are no more results for this search'))
}
} else {
showToast(this.$t('Search Filters["Fetching results. Please wait"]'))
@ -319,7 +298,6 @@ export default Vue.extend({
},
...mapActions([
'ytSearch',
'invidiousAPICall'
])
}

View File

@ -566,6 +566,7 @@ Video:
# As in a Live Video
Premieres on: Premieres on
Premieres: Premieres
Upcoming: Upcoming
Live: Live
Live Now: Live Now
Live Chat: Live Chat

View File

@ -9281,10 +9281,3 @@ ytdl-core@^3.2.2:
m3u8stream "^0.8.6"
miniget "^4.2.2"
sax "^1.1.3"
ytsr@^3.8.0:
version "3.8.0"
resolved "https://registry.yarnpkg.com/ytsr/-/ytsr-3.8.0.tgz#49a8e5dc413f41515fc3d79d93ee3e073d10e772"
integrity sha512-R+RfYXvBBMAr2e4OxrQ5SBv5x/Mdhmcj1Q8TH0f2HK5d2jbhHOtK4BdzPvLriA6MDoMwqqX04GD8Rpf9UNtSTg==
dependencies:
miniget "^4.2.2"