Merge d2e94b1a45 into 89f535e265

Authored by: bashonly

.github/workflows/core.yml

@@ -53,7 +53,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install test requirements
-      run: python3 ./devscripts/install_deps.py --include dev --include curl_cffi
+      run: python3 ./devscripts/install_deps.py --include dev --include curl-cffi
     - name: Run tests
       continue-on-error: False
       run: |

yt_dlp/extractor/_extractors.py

@@ -873,6 +873,11 @@ from .japandiet import (
     SangiinIE,
 )
 from .jeuxvideo import JeuxVideoIE
+from .jiocinema import (
+    JioCinemaTVIE,
+    JioCinemaMovieIE,
+    JioCinemaTVSeasonIE
+)
 from .jiosaavn import (
     JioSaavnSongIE,
     JioSaavnAlbumIE,

yt_dlp/extractor/crunchyroll.py

@@ -24,11 +24,15 @@ class CrunchyrollBaseIE(InfoExtractor):
     _BASE_URL = 'https://www.crunchyroll.com'
     _API_BASE = 'https://api.crunchyroll.com'
     _NETRC_MACHINE = 'crunchyroll'
+    _REFRESH_TOKEN = None
     _AUTH_HEADERS = None
+    _AUTH_EXPIRY = None
     _API_ENDPOINT = None
-    _BASIC_AUTH = None
+    _BASIC_AUTH = 'Basic ' + base64.b64encode(':'.join((
+        't-kdgp2h8c3jub8fn0fq',
+        'yfLDfMfrYvKXh4JXS1LEI2cCqu1v5Wan',
+    )).encode()).decode()
     _IS_PREMIUM = None
-    _CLIENT_ID = ('cr_web', 'noaihdevm_6iyg0a8l0q')
     _LOCALE_LOOKUP = {
         'ar': 'ar-SA',
         'de': 'de-DE',
@@ -43,69 +47,74 @@ class CrunchyrollBaseIE(InfoExtractor):
         'hi': 'hi-IN',
     }
 
-    @property
-    def is_logged_in(self):
-        return bool(self._get_cookies(self._BASE_URL).get('etp_rt'))
+    def _set_auth_info(self, response):
+        CrunchyrollBaseIE._IS_PREMIUM = 'cr_premium' in traverse_obj(response, ('access_token', {jwt_decode_hs256}, 'benefits', ...))
+        CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': response['token_type'] + ' ' + response['access_token']}
+        CrunchyrollBaseIE._AUTH_EXPIRY = time_seconds(seconds=traverse_obj(response, ('expires_in', {float_or_none}), default=300) - 10)
+
+    def _request_token(self, headers, data, note='Requesting token', errnote='Failed to request token'):
+        try:  # TODO: Add impersonation support here
+            return self._download_json(
+                f'{self._BASE_URL}/auth/v1/token', None, note=note, errnote=errnote,
+                headers=headers, data=urlencode_postdata(data))
+        except ExtractorError as error:
+            if not isinstance(error.cause, HTTPError) or error.cause.status != 403:
+                raise
+            raise ExtractorError(
+                'Request blocked by Cloudflare; navigate to Crunchyroll in your browser, '
+                'then pass the fresh cookies (with --cookies-from-browser or --cookies) '
+                'and your browser\'s User-Agent (with --user-agent)', expected=True)
 
     def _perform_login(self, username, password):
-        if self.is_logged_in:
+        if not CrunchyrollBaseIE._REFRESH_TOKEN:
+            CrunchyrollBaseIE._REFRESH_TOKEN = self.cache.load(self._NETRC_MACHINE, username)
+        if CrunchyrollBaseIE._REFRESH_TOKEN:
             return
 
-        upsell_response = self._download_json(
-            f'{self._API_BASE}/get_upsell_data.0.json', None, 'Getting session id',
-            query={
-                'sess_id': 1,
-                'device_id': 'whatvalueshouldbeforweb',
-                'device_type': 'com.crunchyroll.static',
-                'access_token': 'giKq5eY27ny3cqz',
-                'referer': f'{self._BASE_URL}/welcome/login'
-            })
-        if upsell_response['code'] != 'ok':
-            raise ExtractorError('Could not get session id')
-        session_id = upsell_response['data']['session_id']
-
-        login_response = self._download_json(
-            f'{self._API_BASE}/login.1.json', None, 'Logging in',
-            data=urlencode_postdata({
-                'account': username,
-                'password': password,
-                'session_id': session_id
-            }))
-        if login_response['code'] != 'ok':
-            raise ExtractorError('Login failed. Server message: %s' % login_response['message'], expected=True)
-        if not self.is_logged_in:
-            raise ExtractorError('Login succeeded but did not set etp_rt cookie')
-
-    def _update_auth(self):
-        if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_REFRESH > time_seconds():
-            return
-
-        if not CrunchyrollBaseIE._BASIC_AUTH:
-            cx_api_param = self._CLIENT_ID[self.is_logged_in]
-            self.write_debug(f'Using cxApiParam={cx_api_param}')
-            CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode()
-
-        auth_headers = {'Authorization': CrunchyrollBaseIE._BASIC_AUTH}
-        if self.is_logged_in:
-            grant_type = 'etp_rt_cookie'
-        else:
-            grant_type = 'client_id'
-            auth_headers['ETP-Anonymous-ID'] = uuid.uuid4()
         try:
-            auth_response = self._download_json(
-                f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
-                headers=auth_headers, data=f'grant_type={grant_type}'.encode())
+            login_response = self._request_token(
+                headers={'Authorization': self._BASIC_AUTH}, data={
+                    'username': username,
+                    'password': password,
+                    'grant_type': 'password',
+                    'scope': 'offline_access',
+                }, note='Logging in', errnote='Failed to log in')
         except ExtractorError as error:
-            if isinstance(error.cause, HTTPError) and error.cause.status == 403:
-                raise ExtractorError(
-                    'Request blocked by Cloudflare; navigate to Crunchyroll in your browser, '
-                    'then pass the fresh cookies (with --cookies-from-browser or --cookies) '
-                    'and your browser\'s User-Agent (with --user-agent)', expected=True)
+            if isinstance(error.cause, HTTPError) and error.cause.status == 401:
+                raise ExtractorError('Invalid username and/or password', expected=True)
             raise
 
-        CrunchyrollBaseIE._IS_PREMIUM = 'cr_premium' in traverse_obj(auth_response, ('access_token', {jwt_decode_hs256}, 'benefits', ...))
-        CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']}
-        CrunchyrollBaseIE._AUTH_REFRESH = time_seconds(seconds=traverse_obj(auth_response, ('expires_in', {float_or_none}), default=300) - 10)
+        CrunchyrollBaseIE._REFRESH_TOKEN = login_response['refresh_token']
+        self.cache.store(self._NETRC_MACHINE, username, CrunchyrollBaseIE._REFRESH_TOKEN)
+        self._set_auth_info(login_response)
+
+    def _update_auth(self):
+        if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_EXPIRY > time_seconds():
+            return
+
+        auth_headers = {'Authorization': self._BASIC_AUTH}
+        if CrunchyrollBaseIE._REFRESH_TOKEN:
+            data = {
+                'refresh_token': CrunchyrollBaseIE._REFRESH_TOKEN,
+                'grant_type': 'refresh_token',
+                'scope': 'offline_access',
+            }
+        else:
+            data = {'grant_type': 'client_id'}
+            auth_headers['ETP-Anonymous-ID'] = uuid.uuid4()
+        try:
+            auth_response = self._request_token(auth_headers, data)
+        except ExtractorError as error:
+            username, password = self._get_login_info()
+            if not username or not isinstance(error.cause, HTTPError) or error.cause.status != 400:
+                raise
+            self.to_screen('Refresh token has expired. Re-logging in')
+            CrunchyrollBaseIE._REFRESH_TOKEN = None
+            self.cache.store(self._NETRC_MACHINE, username, None)
+            self._perform_login(username, password)
+            return
+
+        self._set_auth_info(auth_response)
 
     def _locale_from_language(self, language):
         config_locale = self._configuration_arg('metadata', ie_key=CrunchyrollBetaIE, casesense=True)
@@ -168,7 +177,8 @@ class CrunchyrollBaseIE(InfoExtractor):
         self._update_auth()
         stream_response = self._download_json(
             f'https://cr-play-service.prd.crunchyrollsvc.com/v1/{identifier}/console/switch/play',
-            display_id, note='Downloading stream info', headers=CrunchyrollBaseIE._AUTH_HEADERS)
+            display_id, note='Downloading stream info', errnote='Failed to download stream info',
+            headers=CrunchyrollBaseIE._AUTH_HEADERS)
 
         available_formats = {'': ('', '', stream_response['url'])}
         for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])):
@@ -383,9 +393,9 @@ class CrunchyrollBetaIE(CrunchyrollCmsBaseIE):
 
         if not self._IS_PREMIUM and traverse_obj(response, (f'{object_type}_metadata', 'is_premium_only')):
             message = f'This {object_type} is for premium members only'
-            if self.is_logged_in:
+            if CrunchyrollBaseIE._REFRESH_TOKEN:
                 raise ExtractorError(message, expected=True)
-            self.raise_login_required(message)
+            self.raise_login_required(message, method='password')
 
         result['formats'], result['subtitles'] = self._extract_stream(internal_id)
 
@@ -575,9 +585,9 @@ class CrunchyrollMusicIE(CrunchyrollBaseIE):
 
         if not self._IS_PREMIUM and response.get('isPremiumOnly'):
             message = f'This {response.get("type") or "media"} is for premium members only'
-            if self.is_logged_in:
+            if CrunchyrollBaseIE._REFRESH_TOKEN:
                 raise ExtractorError(message, expected=True)
-            self.raise_login_required(message)
+            self.raise_login_required(message, method='password')
 
         result = self._transform_music_response(response)
         result['formats'], _ = self._extract_stream(f'music/{internal_id}', internal_id)

yt_dlp/extractor/jiocinema.py

@@ -0,0 +1,246 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+import json
+import math
+import re
+import uuid
+
+
+class JioCinemaBaseIE(InfoExtractor):
+    deviceId = None
+    auth_token = None
+
+    def _get_auth_token(self):
+        token_endpoint = "https://auth-jiocinema.voot.com/tokenservice/apis/v4/guest"
+
+        payload = {
+            "appName": "RJIL_JioCinema",
+            "deviceType": "phone",
+            "os": "ios",
+            "deviceId": self.deviceId,
+            "freshLaunch": True,
+            "adId": self.deviceId,
+            "appVersion": "23.11.22.0-dc532c79"
+        }
+
+        token_response = self._download_json(url_or_request=token_endpoint, video_id=None, note='Fetching Auth Token', data=bytes(json.dumps(payload), encoding='utf8'))
+
+        if "authToken" in token_response:
+            return token_response["authToken"]
+
+        return None
+
+    def _get_media_metadata(self, media_id):
+        media_metadata = {}
+        metadata_endpoint_template = 'https://content-jiovoot.voot.com/psapi/voot/v1/voot-web/content/query/asset-details?&ids=include:{media_id}&responseType=common&devicePlatformType=desktop'
+
+        metadata_endpoint = metadata_endpoint_template.format(media_id=media_id)
+
+        media_metadata_response = self._download_json(metadata_endpoint, None, 'Fetching Episode Metadata')
+
+        if media_metadata_response and 'result' in media_metadata_response and media_metadata_response['result']:
+            media_metadata_response = media_metadata_response['result'][0]
+
+        if media_metadata_response['showName']:
+            media_metadata['series'] = media_metadata_response['showName']
+        media_metadata['title'] = media_metadata_response['fullTitle']
+        media_metadata['description'] = media_metadata_response['fullSynopsis']
+        if media_metadata_response['season']:
+            media_metadata['season_number'] = int(media_metadata_response['season'])
+        if media_metadata_response['episode']:
+            media_metadata['episode_number'] = int(media_metadata_response['episode'])
+        # media_metadata['subtitles'] = media_metadata_response['subtitles']
+
+        return media_metadata
+
+    def _get_stream_url(self, auth_token, media_id):
+        stream_endpoint = 'https://apis-jiovoot.voot.com/playbackjv/v4/{media_id}'
+
+        stream_endpoint = stream_endpoint.format(media_id=media_id)
+
+        headers = {
+            'accesstoken': auth_token,
+            'x-platform': 'androidweb'
+        }
+
+        payload = {
+            "4k": True,
+            "ageGroup": "18+",
+            "appVersion": "3.4.0",
+            "bitrateProfile": "xhdpi",
+            "capability": {
+                "drmCapability": {
+                    "aesSupport": "yes",
+                    "fairPlayDrmSupport": "yes",
+                    "playreadyDrmSupport": "none",
+                    "widevineDRMSupport": "yes"
+                },
+                "frameRateCapability": [
+                    {
+                        "frameRateSupport": "60fps",
+                        "videoQuality": "2160p"
+                    }
+                ]
+            },
+            "continueWatchingRequired": True,
+            "dolby": False,
+            "downloadRequest": False,
+            "hevc": True,
+            "kidsSafe": False,
+            "manufacturer": "Mac OS",
+            "model": "Mac OS",
+            "multiAudioRequired": True,
+            "osVersion": "10.15.7",
+            "parentalPinValid": True
+        }
+
+        stream_response = self._download_json(url_or_request=stream_endpoint, video_id=None, note='Extracting Stream URL', data=bytes(json.dumps(payload), encoding='utf8'), headers=headers)
+
+        mpd_url = None
+        m3u8_url = None
+
+        for url_data in stream_response['data']['playbackUrls']:
+            if url_data['encryption'] == 'widevine':
+                mpd_url = url_data['url']
+            elif url_data['encryption'] == 'aes128':
+                m3u8_url = url_data['url']
+
+        return mpd_url, m3u8_url
+
+    def _real_initialize(self):
+        super()._real_initialize()
+        if self.deviceId is None:
+            self.deviceId = str(uuid.uuid4())
+
+        self.auth_token = self._get_auth_token()
+
+    def _real_extract(self, url):
+        media_name, season_number, episode_name, media_id = re.match(self._VALID_URL, url).groups()
+
+        if not self.auth_token:
+            return
+
+        media_metadata = self._get_media_metadata(media_id)
+        mpd_url, m3u8_url = self._get_stream_url(self.auth_token, media_id)
+
+        formats = []
+        formats.extend(self._extract_mpd_formats(mpd_url, media_id))
+        formats.extend(self._extract_m3u8_formats(m3u8_url, media_id))
+
+        response_dict = {
+            'id': media_id,
+            'formats': formats,
+        }
+        response_dict.update(media_metadata)
+        return response_dict
+
+
+class JioCinemaTVIE(JioCinemaBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?jiocinema\.com/tv-shows/(?P<media_name>[a-zA-Z0-9\-]*)/(?P<season_number>[0-9]*)/(?P<episode_name>[a-zA-Z0-9\-]*)/(?P<media_id>[a-zA-Z0-9\-]*)'
+    _TEST = {
+        'url': 'https://www.jiocinema.com/tv-shows/cedric/1/i-love-school/3769333',
+        'info_dict': {
+            'id': '3769333',
+            'ext': 'mp4',
+            'title': 'I Love School',
+            'description': 'md5:71cc843e4ec65f62c6fd33cf38920198',
+            'series': 'Cedric',
+            'season_number': 1,
+            'episode_number': 1,
+            'episode': 'Episode 1',
+            'season': 'Season 1',
+        },
+        'params': {
+            'skip_download': True,
+            'format': 'best'
+        }
+    }
+
+
+class JioCinemaMovieIE(JioCinemaBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?jiocinema\.com/movies/(?P<media_name>[a-zA-Z0-9\-]*)/?(?P<season_number>[0-9]*)?/?(?P<episode_name>[a-zA-Z0-9\-]*)?/(?P<media_id>[a-zA-Z0-9\-]*)'
+    _TEST = {
+        'url': 'https://www.jiocinema.com/movies/i-choose-you-pokemon-the-movie/3777402',
+        'info_dict': {
+            'id': '3777402',
+            'ext': 'mp4',
+            'title': 'I Choose You! - Pokemon the Movie',
+            'description': 'md5:c224c04ce664ac2f41d85e2fb1d49b2f'
+        },
+        'params': {
+            'skip_download': True,
+            'format': 'best'
+        }
+    }
+
+
+class JioCinemaTVSeasonIE(JioCinemaBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?jiocinema\.com/tv-shows/(?P<playlist_name>[a-zA-Z0-9\-]*)/(?P<playlist_id>[a-zA-Z0-9\-]*)$'
+    _TEST = {
+        'url': 'https://www.jiocinema.com/tv-shows/cedric/3768448',
+        'info_dict': {
+            'id': '3769333',
+            'ext': 'mp4',
+            'title': 'I Love School',
+            'description': 'md5:71cc843e4ec65f62c6fd33cf38920198',
+            'series': 'Cedric',
+            'season_number': 1,
+            'episode_number': 1,
+            'episode': 'Episode 1',
+            'season': 'Season 1',
+        },
+        'params': {
+            'skip_download': True,
+            'format': 'best'
+        }
+    }
+
+    def _real_extract(self, url):
+        playlist_entries = []
+        playlist_name, playlist_id = re.match(self._VALID_URL, url).groups()
+        season_data_url_template = 'https://content-jiovoot.voot.com/psapi/voot/v1/voot-web/view/show/{playlist_id}?responseType=common&devicePlatformType=desktop&layoutCohort=default'
+        season_data_url = season_data_url_template.format(playlist_id=playlist_id)
+        season_data_response = self._download_json(season_data_url, None, 'Fetching Season Info')
+        season_id_list = self._extract_season_id_list(season_data_response)
+        playlist_entries = self._fetch_episode_data(season_id_list)
+        return self.playlist_result(playlist_entries)
+
+    def _extract_season_id_list(self, season_data_response):
+        season_id_list = []
+
+        for tray in season_data_response['trays']:
+            if tray['title'] == 'Episodes':
+                for season in tray['trayTabs']:
+                    season_id_list.append(season['id'])
+                break
+        return season_id_list
+
+    def _fetch_episode_data(self, season_id_list):
+        episode_url_list = []
+        for season_id in season_id_list:
+            episode_list_template = 'https://content-jiovoot.voot.com/psapi/voot/v1/voot-web/content/generic/series-wise-episode?sort=episode%3Aasc&id={season_id}&responseType=common&page={page_number}'
+            page_number = 1
+            total_pages = 1
+
+            episode_details_list_url = episode_list_template.format(season_id=season_id, page_number=page_number)
+
+            paginated_episode_list = self._download_json(episode_details_list_url, None, 'Fetching Episode List')
+            episode_url_list.extend(self._extract_episode_urls(paginated_episode_list))
+            total_pages = math.ceil(paginated_episode_list['totalAsset'] / 10)
+
+            page_number += 1
+            while page_number <= total_pages:
+                episode_details_list_url = episode_list_template.format(season_id=season_id, page_number=page_number)
+                paginated_episode_list = self._download_json(episode_details_list_url, None, 'Fetching Episode List')
+                episode_url_list.extend(self._extract_episode_urls(paginated_episode_list))
+                page_number += 1
+        return episode_url_list
+
+    def _extract_episode_urls(self, episode_response_list):
+        episode_url_list = []
+        for episode in episode_response_list['result']:
+            episode_url_list.append(self.url_result(episode['slug']))
+        return episode_url_list

yt_dlp/extractor/stacommu.py

@@ -174,7 +174,7 @@ class TheaterComplexTownBaseIE(StacommuBaseIE):
 
 
 class TheaterComplexTownVODIE(TheaterComplexTownBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:en/)?videos/episodes/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:(?:en|ja)/)?videos/episodes/(?P<id>\w+)'
     IE_NAME = 'theatercomplextown:vod'
     _TESTS = [{
         'url': 'https://www.theater-complex.town/videos/episodes/hoxqidYNoAn7bP92DN6p78',
@@ -195,6 +195,9 @@ class TheaterComplexTownVODIE(TheaterComplexTownBaseIE):
     }, {
         'url': 'https://www.theater-complex.town/en/videos/episodes/6QT7XYwM9dJz5Gf9VB6K5y',
         'only_matching': True,
+    }, {
+        'url': 'https://www.theater-complex.town/ja/videos/episodes/hoxqidYNoAn7bP92DN6p78',
+        'only_matching': True,
     }]
 
     _API_PATH = 'videoEpisodes'
@@ -204,7 +207,7 @@ class TheaterComplexTownVODIE(TheaterComplexTownBaseIE):
 
 
 class TheaterComplexTownPPVIE(TheaterComplexTownBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:en/)?ppv/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?theater-complex\.town/(?:(?:en|ja)/)?ppv/(?P<id>\w+)'
     IE_NAME = 'theatercomplextown:ppv'
     _TESTS = [{
         'url': 'https://www.theater-complex.town/ppv/wytW3X7khrjJBUpKuV3jen',
@@ -223,6 +226,9 @@ class TheaterComplexTownPPVIE(TheaterComplexTownBaseIE):
     }, {
         'url': 'https://www.theater-complex.town/en/ppv/wytW3X7khrjJBUpKuV3jen',
         'only_matching': True,
+    }, {
+        'url': 'https://www.theater-complex.town/ja/ppv/qwUVmLmGEiZ3ZW6it9uGys',
+        'only_matching': True,
     }]
 
     _API_PATH = 'events'

yt_dlp/extractor/wrestleuniverse.py

@@ -147,7 +147,7 @@ class WrestleUniverseBaseIE(InfoExtractor):
         metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
         if not metadata:
             webpage = self._download_webpage(url, video_id)
-            nextjs_data = self._search_nextjs_data(webpage, video_id)
+            nextjs_data = self._search_nextjs_data(webpage, video_id, fatal=False)
             metadata = traverse_obj(nextjs_data, (
                 'props', 'pageProps', *variadic(props_keys, (str, bytes, dict, set)), {dict})) or {}
         return metadata