From 9b8ee23b99de91f9e463050baddfd76fa6580ad6 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 21 Apr 2022 00:35:57 +0530 Subject: [PATCH] [dependencies] Create module with all dependency imports --- test/test_aes.py | 6 +- yt_dlp/YoutubeDL.py | 20 ++----- yt_dlp/aes.py | 9 +-- yt_dlp/compat/__init__.py | 21 ------- yt_dlp/compat/_legacy.py | 3 + yt_dlp/cookies.py | 35 +++--------- yt_dlp/dependencies.py | 77 ++++++++++++++++++++++++++ yt_dlp/downloader/hls.py | 5 +- yt_dlp/downloader/websocket.py | 10 +--- yt_dlp/extractor/fc2.py | 4 +- yt_dlp/extractor/twitcasting.py | 4 +- yt_dlp/postprocessor/embedthumbnail.py | 20 +++---- yt_dlp/utils.py | 25 +++------ 13 files changed, 127 insertions(+), 112 deletions(-) create mode 100644 yt_dlp/dependencies.py diff --git a/test/test_aes.py b/test/test_aes.py index 1c1238c8b..c934104e3 100644 --- a/test/test_aes.py +++ b/test/test_aes.py @@ -23,7 +23,7 @@ from yt_dlp.aes import ( aes_gcm_decrypt_and_verify, aes_gcm_decrypt_and_verify_bytes, ) -from yt_dlp.compat import compat_pycrypto_AES +from yt_dlp.dependencies import Cryptodome_AES from yt_dlp.utils import bytes_to_intlist, intlist_to_bytes # the encrypted data can be generate with 'devscripts/generate_aes_testdata.py' @@ -45,7 +45,7 @@ class TestAES(unittest.TestCase): data = b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\x27\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd' decrypted = intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(data), self.key, self.iv)) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) - if compat_pycrypto_AES: + if Cryptodome_AES: decrypted = aes_cbc_decrypt_bytes(data, intlist_to_bytes(self.key), intlist_to_bytes(self.iv)) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) @@ -75,7 +75,7 @@ class TestAES(unittest.TestCase): decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify( bytes_to_intlist(data), self.key, bytes_to_intlist(authentication_tag), self.iv[:12])) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) - if compat_pycrypto_AES: + if Cryptodome_AES: decrypted = aes_gcm_decrypt_and_verify_bytes( data, intlist_to_bytes(self.key), authentication_tag, intlist_to_bytes(self.iv[:12])) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 155b5a063..9acd88171 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -27,10 +27,8 @@ from string import ascii_letters from .cache import Cache from .compat import ( - compat_brotli, compat_get_terminal_size, compat_os_name, - compat_pycrypto_AES, compat_shlex_quote, compat_str, compat_urllib_error, @@ -109,7 +107,6 @@ from .utils import ( format_field, formatSeconds, get_domain, - has_certifi, int_or_none, iri_to_uri, join_nonempty, @@ -3656,20 +3653,11 @@ class YoutubeDL: ) or 'none' write_debug('exe versions: %s' % exe_str) - from .cookies import SECRETSTORAGE_AVAILABLE, SQLITE_AVAILABLE - from .downloader.websocket import has_websockets - from .postprocessor.embedthumbnail import has_mutagen + from .dependencies import available_dependencies - lib_str = join_nonempty( - compat_brotli and compat_brotli.__name__, - has_certifi and 'certifi', - compat_pycrypto_AES and compat_pycrypto_AES.__name__.split('.')[0], - SECRETSTORAGE_AVAILABLE and 'secretstorage', - has_mutagen and 'mutagen', - SQLITE_AVAILABLE and 'sqlite', - has_websockets and 'websockets', - delim=', ') or 'none' - write_debug('Optional libraries: %s' % lib_str) + write_debug('Optional libraries: %s' % (', '.join(sorted({ + module.__name__.split('.')[0] for module in available_dependencies.values() + })) or 'none')) self._setup_opener() proxy_map = {} diff --git a/yt_dlp/aes.py b/yt_dlp/aes.py index 603f3d187..ba3baf3de 100644 --- a/yt_dlp/aes.py +++ b/yt_dlp/aes.py @@ -1,16 +1,17 @@ from math import ceil -from .compat import compat_b64decode, compat_ord, compat_pycrypto_AES +from .compat import compat_b64decode, compat_ord +from .dependencies import Cryptodome_AES from .utils import bytes_to_intlist, intlist_to_bytes -if compat_pycrypto_AES: +if Cryptodome_AES: def aes_cbc_decrypt_bytes(data, key, iv): """ Decrypt bytes with AES-CBC using pycryptodome """ - return compat_pycrypto_AES.new(key, compat_pycrypto_AES.MODE_CBC, iv).decrypt(data) + return Cryptodome_AES.new(key, Cryptodome_AES.MODE_CBC, iv).decrypt(data) def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce): """ Decrypt bytes with AES-GCM using pycryptodome """ - return compat_pycrypto_AES.new(key, compat_pycrypto_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag) + return Cryptodome_AES.new(key, Cryptodome_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag) else: def aes_cbc_decrypt_bytes(data, key, iv): diff --git a/yt_dlp/compat/__init__.py b/yt_dlp/compat/__init__.py index 7a0e82992..56a65bb6c 100644 --- a/yt_dlp/compat/__init__.py +++ b/yt_dlp/compat/__init__.py @@ -54,11 +54,6 @@ else: compat_realpath = os.path.realpath -try: - import websockets as compat_websockets -except ImportError: - compat_websockets = None - # Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl # See https://github.com/yt-dlp/yt-dlp/issues/792 # https://docs.python.org/3/library/os.path.html#os.path.expanduser @@ -78,22 +73,6 @@ else: compat_expanduser = os.path.expanduser -try: - from Cryptodome.Cipher import AES as compat_pycrypto_AES -except ImportError: - try: - from Crypto.Cipher import AES as compat_pycrypto_AES - except ImportError: - compat_pycrypto_AES = None - -try: - import brotlicffi as compat_brotli -except ImportError: - try: - import brotli as compat_brotli - except ImportError: - compat_brotli = None - WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py index f185b7e2f..ce24760e5 100644 --- a/yt_dlp/compat/_legacy.py +++ b/yt_dlp/compat/_legacy.py @@ -17,6 +17,9 @@ from subprocess import DEVNULL from .asyncio import run as compat_asyncio_run # noqa: F401 from .re import Pattern as compat_Pattern # noqa: F401 from .re import match as compat_Match # noqa: F401 +from ..dependencies import Cryptodome_AES as compat_pycrypto_AES # noqa: F401 +from ..dependencies import brotli as compat_brotli # noqa: F401 +from ..dependencies import websockets as compat_websockets # noqa: F401 # compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py index 8a4baa5bb..621c91e86 100644 --- a/yt_dlp/cookies.py +++ b/yt_dlp/cookies.py @@ -17,31 +17,14 @@ from .aes import ( unpad_pkcs7, ) from .compat import compat_b64decode, compat_cookiejar_Cookie +from .dependencies import ( + _SECRETSTORAGE_UNAVAILABLE_REASON, + secretstorage, + sqlite3, +) from .minicurses import MultilinePrinter, QuietMultilinePrinter from .utils import Popen, YoutubeDLCookieJar, error_to_str, expand_path -try: - import sqlite3 - SQLITE_AVAILABLE = True -except ImportError: - # although sqlite3 is part of the standard library, it is possible to compile python without - # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544 - SQLITE_AVAILABLE = False - - -try: - import secretstorage - SECRETSTORAGE_AVAILABLE = True -except ImportError: - SECRETSTORAGE_AVAILABLE = False - SECRETSTORAGE_UNAVAILABLE_REASON = ( - 'as the `secretstorage` module is not installed. ' - 'Please install by running `python3 -m pip install secretstorage`.') -except Exception as _err: - SECRETSTORAGE_AVAILABLE = False - SECRETSTORAGE_UNAVAILABLE_REASON = f'as the `secretstorage` module could not be initialized. {_err}' - - CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'} SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'} @@ -122,7 +105,7 @@ def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), def _extract_firefox_cookies(profile, logger): logger.info('Extracting cookies from firefox') - if not SQLITE_AVAILABLE: + if not sqlite3: logger.warning('Cannot extract cookies from firefox without sqlite3 support. ' 'Please use a python interpreter compiled with sqlite3 support') return YoutubeDLCookieJar() @@ -236,7 +219,7 @@ def _get_chromium_based_browser_settings(browser_name): def _extract_chrome_cookies(browser_name, profile, keyring, logger): logger.info(f'Extracting cookies from {browser_name}') - if not SQLITE_AVAILABLE: + if not sqlite3: logger.warning(f'Cannot extract cookies from {browser_name} without sqlite3 support. ' 'Please use a python interpreter compiled with sqlite3 support') return YoutubeDLCookieJar() @@ -806,8 +789,8 @@ def _get_kwallet_password(browser_keyring_name, logger): def _get_gnome_keyring_password(browser_keyring_name, logger): - if not SECRETSTORAGE_AVAILABLE: - logger.error(f'secretstorage not available {SECRETSTORAGE_UNAVAILABLE_REASON}') + if not secretstorage: + logger.error(f'secretstorage not available {_SECRETSTORAGE_UNAVAILABLE_REASON}') return b'' # the Gnome keyring does not seem to organise keys in the same way as KWallet, # using `dbus-monitor` during startup, it can be observed that chromium lists all keys diff --git a/yt_dlp/dependencies.py b/yt_dlp/dependencies.py new file mode 100644 index 000000000..99cc6e29c --- /dev/null +++ b/yt_dlp/dependencies.py @@ -0,0 +1,77 @@ +# flake8: noqa: F401 + +try: + import brotlicffi as brotli +except ImportError: + try: + import brotli + except ImportError: + brotli = None + + +try: + import certifi +except ImportError: + certifi = None +else: + from os.path import exists as _path_exists + + # The certificate may not be bundled in executable + if not _path_exists(certifi.where()): + certifi = None + + +try: + from Cryptodome.Cipher import AES as Cryptodome_AES +except ImportError: + try: + from Crypto.Cipher import AES as Cryptodome_AES + except ImportError: + Cryptodome_AES = None + + +try: + import mutagen +except ImportError: + mutagen = None + + +secretstorage = None +try: + import secretstorage + _SECRETSTORAGE_UNAVAILABLE_REASON = None +except ImportError: + _SECRETSTORAGE_UNAVAILABLE_REASON = ( + 'as the `secretstorage` module is not installed. ' + 'Please install by running `python3 -m pip install secretstorage`') +except Exception as _err: + _SECRETSTORAGE_UNAVAILABLE_REASON = f'as the `secretstorage` module could not be initialized. {_err}' + + +try: + import sqlite3 +except ImportError: + # although sqlite3 is part of the standard library, it is possible to compile python without + # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544 + sqlite3 = None + + +try: + import websockets +except (ImportError, SyntaxError): + # websockets 3.10 on python 3.6 causes SyntaxError + # See https://github.com/yt-dlp/yt-dlp/issues/2633 + websockets = None + + +all_dependencies = {k: v for k, v in globals().items() if not k.startswith('_')} + + +available_dependencies = {k: v for k, v in all_dependencies.items() if v} + + +__all__ = [ + 'all_dependencies', + 'available_dependencies', + *all_dependencies.keys(), +] diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py index 2d65f48ae..694c843f3 100644 --- a/yt_dlp/downloader/hls.py +++ b/yt_dlp/downloader/hls.py @@ -5,7 +5,8 @@ import re from .external import FFmpegFD from .fragment import FragmentFD from .. import webvtt -from ..compat import compat_pycrypto_AES, compat_urlparse +from ..compat import compat_urlparse +from ..dependencies import Cryptodome_AES from ..downloader import get_suitable_downloader from ..utils import bug_reports_message, parse_m3u8_attributes, update_url_query @@ -60,7 +61,7 @@ class HlsFD(FragmentFD): s = urlh.read().decode('utf-8', 'ignore') can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None - if can_download and not compat_pycrypto_AES and '#EXT-X-KEY:METHOD=AES-128' in s: + if can_download and not Cryptodome_AES and '#EXT-X-KEY:METHOD=AES-128' in s: if FFmpegFD.available(): can_download, message = False, 'The stream has AES-128 encryption and pycryptodomex is not available' else: diff --git a/yt_dlp/downloader/websocket.py b/yt_dlp/downloader/websocket.py index 8465f9713..eb1b99b45 100644 --- a/yt_dlp/downloader/websocket.py +++ b/yt_dlp/downloader/websocket.py @@ -3,18 +3,10 @@ import os import signal import threading -try: - import websockets -except (ImportError, SyntaxError): - # websockets 3.10 on python 3.6 causes SyntaxError - # See https://github.com/yt-dlp/yt-dlp/issues/2633 - has_websockets = False -else: - has_websockets = True - from .common import FileDownloader from .external import FFmpegFD from ..compat import asyncio +from ..dependencies import websockets class FFmpegSinkFD(FileDownloader): diff --git a/yt_dlp/extractor/fc2.py b/yt_dlp/extractor/fc2.py index a4c9793bb..225677b00 100644 --- a/yt_dlp/extractor/fc2.py +++ b/yt_dlp/extractor/fc2.py @@ -4,10 +4,10 @@ from .common import InfoExtractor from ..compat import ( compat_parse_qs, ) +from ..dependencies import websockets from ..utils import ( ExtractorError, WebSocketsWrapper, - has_websockets, js_to_json, sanitized_Request, std_headers, @@ -170,7 +170,7 @@ class FC2LiveIE(InfoExtractor): }] def _real_extract(self, url): - if not has_websockets: + if not websockets: raise ExtractorError('websockets library is not available. Please install it.', expected=True) video_id = self._match_id(url) webpage = self._download_webpage('https://live.fc2.com/%s/' % video_id, video_id) diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py index 3d6a12265..07565383a 100644 --- a/yt_dlp/extractor/twitcasting.py +++ b/yt_dlp/extractor/twitcasting.py @@ -2,7 +2,7 @@ import itertools import re from .common import InfoExtractor -from ..downloader.websocket import has_websockets +from ..dependencies import websockets from ..utils import ( clean_html, ExtractorError, @@ -161,7 +161,7 @@ class TwitCastingIE(InfoExtractor): note='Downloading source quality m3u8', headers=self._M3U8_HEADERS, fatal=False)) - if has_websockets: + if websockets: qq = qualities(['base', 'mobilesource', 'main']) streams = traverse_obj(stream_server_data, ('llfmp4', 'streams')) or {} for mode, ws_url in streams.items(): diff --git a/yt_dlp/postprocessor/embedthumbnail.py b/yt_dlp/postprocessor/embedthumbnail.py index 5469f25e0..c5ea76893 100644 --- a/yt_dlp/postprocessor/embedthumbnail.py +++ b/yt_dlp/postprocessor/embedthumbnail.py @@ -4,17 +4,9 @@ import os import re import subprocess -try: - from mutagen.flac import FLAC, Picture - from mutagen.mp4 import MP4, MP4Cover - from mutagen.oggopus import OggOpus - from mutagen.oggvorbis import OggVorbis - has_mutagen = True -except ImportError: - has_mutagen = False - from .common import PostProcessor from .ffmpeg import FFmpegPostProcessor, FFmpegThumbnailsConvertorPP +from ..dependencies import mutagen from ..utils import ( Popen, PostProcessingError, @@ -26,6 +18,12 @@ from ..utils import ( shell_quote, ) +if mutagen: + from mutagen.flac import FLAC, Picture + from mutagen.mp4 import MP4, MP4Cover + from mutagen.oggopus import OggOpus + from mutagen.oggvorbis import OggVorbis + class EmbedThumbnailPPError(PostProcessingError): pass @@ -121,7 +119,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor): elif info['ext'] in ['m4a', 'mp4', 'mov']: prefer_atomicparsley = 'embed-thumbnail-atomicparsley' in self.get_param('compat_opts', []) # Method 1: Use mutagen - if not has_mutagen or prefer_atomicparsley: + if not mutagen or prefer_atomicparsley: success = False else: try: @@ -194,7 +192,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor): raise EmbedThumbnailPPError(f'Unable to embed using ffprobe & ffmpeg; {err}') elif info['ext'] in ['ogg', 'opus', 'flac']: - if not has_mutagen: + if not mutagen: raise EmbedThumbnailPPError('module mutagen was not found. Please install using `python -m pip install mutagen`') self._report_run('mutagen', filename) diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index ccea3c4e6..7f0c055ac 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -41,7 +41,6 @@ import zlib from .compat import ( asyncio, - compat_brotli, compat_chr, compat_cookiejar, compat_etree_fromstring, @@ -64,18 +63,10 @@ from .compat import ( compat_urllib_parse_urlparse, compat_urllib_request, compat_urlparse, - compat_websockets, ) +from .dependencies import brotli, certifi, websockets from .socks import ProxyType, sockssocket -try: - import certifi - - # The certificate may not be bundled in executable - has_certifi = os.path.exists(certifi.where()) -except ImportError: - has_certifi = False - def register_socks_protocols(): # "Register" SOCKS protocols @@ -138,7 +129,7 @@ def random_user_agent(): SUPPORTED_ENCODINGS = [ 'gzip', 'deflate' ] -if compat_brotli: +if brotli: SUPPORTED_ENCODINGS.append('br') std_headers = { @@ -1267,7 +1258,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler): def brotli(data): if not data: return data - return compat_brotli.decompress(data) + return brotli.decompress(data) def http_request(self, req): # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not @@ -5231,7 +5222,7 @@ class WebSocketsWrapper(): def __init__(self, url, headers=None, connect=True): self.loop = asyncio.events.new_event_loop() - self.conn = compat_websockets.connect( + self.conn = websockets.connect( url, extra_headers=headers, ping_interval=None, close_timeout=float('inf'), loop=self.loop, ping_timeout=float('inf')) if connect: @@ -5294,9 +5285,6 @@ class WebSocketsWrapper(): }) -has_websockets = bool(compat_websockets) - - def merge_headers(*dicts): """Merge dicts of http headers case insensitively, prioritizing the latter ones""" return {k.title(): v for k, v in itertools.chain.from_iterable(map(dict.items, dicts))} @@ -5312,3 +5300,8 @@ class classproperty: def Namespace(**kwargs): return collections.namedtuple('Namespace', kwargs)(**kwargs) + + +# Deprecated +has_certifi = bool(certifi) +has_websockets = bool(websockets)