From dca4021dd74f90d57dd3659ff84a5381feb4c9f2 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Wed, 8 Jul 2020 15:05:17 -0700 Subject: [PATCH 01/22] Add formatters module Add formatters module that contains a few basic built-in formatters for JSON, SRT, and an arbitrary plain TEXT. Mainly for examples of implementing your own. Module contains an abstract base class `TranscriptFormatter` so that anyone can create their own TranscriptFormatter class Module contains at runtime a `formats` which is basically an instance of a factory of classes (not sure if that is the correct term to use). But it itself is an instance that folks can add custom formatter classes that the API can make use of. --- youtube_transcript_api/formatters.py | 156 +++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 youtube_transcript_api/formatters.py diff --git a/youtube_transcript_api/formatters.py b/youtube_transcript_api/formatters.py new file mode 100644 index 0000000..a5f30cd --- /dev/null +++ b/youtube_transcript_api/formatters.py @@ -0,0 +1,156 @@ +from abc import ABCMeta +from abc import abstractclassmethod +from collections import defaultdict +import json +import re + +from xml.etree import ElementTree + +from ._html_unescaping import unescape + + +def parse_timecode(time): + """Converts a `time` into a formatted transcript timecode. + + :param time: a float representing time in seconds. + :type time: float + :return: a string formatted as a timecode, 'HH:MM:SS,MS' + :rtype str + + :example: + >>> parse_timecode(6.93) + '00:00:06,930' + """ + time = float(time) + hours, mins, secs = ( + str(int(time)//3600).rjust(2, '0'), + str(int(time)//60).rjust(2, '0'), + str(int(time)%60).rjust(2, '0'), + ) + ms = str(int(round((time - int(time))*1000, 2))).rjust(3, '0') + return f"{hours}:{mins}:{secs},{ms}" + + +class TranscriptFormatter(metaclass=ABCMeta): + """ + Abstract Base TranscriptFormatter class + + This class should be inherited from to create additional + custom transcript formatters. + + """ + HTML_TAG_REGEX = re.compile(r'<[^>]*>', re.IGNORECASE) + + @abstractclassmethod + def format(cls, transcript_data): + """Any subclass must implement this format class method. + + :param transcript_data: a list of transcripts, 1 or more. + :type transcript_data: list[list[dict], list[dict]] + :return: A list where each item is an individual transcript + as a string. + :rtype: list[str] + """ + pass + + +class JSONTranscriptFormatter(TranscriptFormatter): + """Formatter for outputting JSON data""" + @classmethod + def format(cls, transcript_data): + return [json.dumps(transcript_data)] if transcript_data else [] + + +class TextTranscriptFormatter(TranscriptFormatter): + """Formatter for outputting a Plain Text Format + + Converts the fetched transcript data into separated lines of + plain text separated by newline breaks (\n) with no timecodes. + + """ + @classmethod + def format(cls, transcript_data): + return ['\n'.join(line['text'] for transcript in transcript_data + for line in transcript)] + + +class SRTTranscriptFormatter(TranscriptFormatter): + """Formatter for outputting the SRT Format + + Converts the fetched transcript data into a simple .srt file format. + + """ + @classmethod + def format(cls, transcript_data): + contents = [] + for transcript in transcript_data: + content = [] + for frame, item in enumerate(transcript, start=1): + start_time = float(item.get('start')) + duration = float(item.get('dur', '0.0')) + + end_time = parse_timecode(start_time + duration) + start_time = parse_timecode(start_time) + + content.append("{frame}\n".format(frame=frame)) + content.append("{start_time} --> {end_time}\n".format( + start_time=start_time, end_time=end_time)) + content.append("{text}\n\n".format(text=item.get('text'))) + + contents.append(''.join(content)) + return ['\n\n'.join(contents)] + + +class TranscriptFormatterFactory: + """A Transcript Class Factory + + Allows for adding additional custom Transcript classes for the API + to use. Custom Transcript classes must inherit from the + TranscriptFormatter abstract base class. + """ + def __init__(self): + self._formatters = defaultdict(JSONTranscriptFormatter) + + def add_formatter(self, name, formatter_class): + """Allows for creating additional transcript formatters. + + + :param name: a name given to the `formatter_class` + :type name: str + :param formatter_class: a subclass of TranscriptFormatter + :type formatter_class: class + :rtype None + """ + if not issubclass(formatter_class, TranscriptFormatter): + raise TypeError( + f'{formatter_class} must be a subclass of TranscriptFormatter') + self._formatters.update({name:formatter_class}) + + def add_formatters(self, formatters_dict): + """Allow creation of multiple transcript formatters at a time. + + :param formatters_dict: key(s) are the string name to be given + to the formatter class, value for each key should be a subclass + of TranscriptFormatter. + :type formatters_dict: dict + :rtype None + """ + for name, formatter_class in formatters_dict.items(): + self.add_formatter(name, formatter_class) + + def get_formatter(self, name): + """Retrieve a formatter class by its assigned name. + + :param name: the string name given to the formatter class. + :type name: str + :return: a subclass of `TranscriptFormatter` + """ + return self._formatters[name] + + +formats = TranscriptFormatterFactory() +formats.add_formatters({ + 'json': JSONTranscriptFormatter, + 'srt': SRTTranscriptFormatter, + 'text': TextTranscriptFormatter +}) From ac75f906e6a747d0a4c03dae5ce5dbcacf9313e3 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Wed, 8 Jul 2020 15:11:54 -0700 Subject: [PATCH 02/22] Update test_api.py Fixed an encoding issue for Windows machines receiving an error due to Lines 24-27: `UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 119434: character maps to `. Adjusted test_api.py to fix an issue when the format=None isnt passed with mock test raising: `AssertionError: get_transcript('video_id_1', ['de', 'en'], None, None) call not found` Added `import json` to convert test data for json data into a string for 1 or many json transcripts to keep in line with how the formatters are eventually returned as strings. --- youtube_transcript_api/test/test_api.py | 33 ++++++++++++++----------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/youtube_transcript_api/test/test_api.py b/youtube_transcript_api/test/test_api.py index a081711..dc552d4 100644 --- a/youtube_transcript_api/test/test_api.py +++ b/youtube_transcript_api/test/test_api.py @@ -1,6 +1,6 @@ from unittest import TestCase from mock import patch - +import json import os import requests @@ -21,7 +21,10 @@ from youtube_transcript_api import ( def load_asset(filename): - with open('{dirname}/assets/{filename}'.format(dirname=os.path.dirname(__file__), filename=filename)) as file: + filepath = '{dirname}/assets/{filename}'.format( + dirname=os.path.dirname(__file__), filename=filename) + + with open(filepath, 'r', encoding='utf-8') as file: return file.read() @@ -47,11 +50,11 @@ class TestYouTubeTranscriptApi(TestCase): self.assertEqual( transcript, - [ + json.dumps([ {'text': 'Hey, this is just a test', 'start': 0.0, 'duration': 1.54}, {'text': 'this is not the original transcript', 'start': 1.54, 'duration': 4.16}, {'text': 'just something shorter, I made up for testing', 'start': 5.7, 'duration': 3.239} - ] + ]) ) def test_list_transcripts(self): @@ -158,15 +161,15 @@ class TestYouTubeTranscriptApi(TestCase): def test_get_transcript__with_proxy(self): proxies = {'http': '', 'https:': ''} transcript = YouTubeTranscriptApi.get_transcript( - 'GJLlxj_dtq8', proxies=proxies + 'GJLlxj_dtq8', proxies=proxies, format=None ) self.assertEqual( transcript, - [ + json.dumps([ {'text': 'Hey, this is just a test', 'start': 0.0, 'duration': 1.54}, {'text': 'this is not the original transcript', 'start': 1.54, 'duration': 4.16}, {'text': 'just something shorter, I made up for testing', 'start': 5.7, 'duration': 3.239} - ] + ]) ) def test_get_transcript__with_cookies(self): @@ -176,11 +179,11 @@ class TestYouTubeTranscriptApi(TestCase): self.assertEqual( transcript, - [ + json.dumps([ {'text': 'Hey, this is just a test', 'start': 0.0, 'duration': 1.54}, {'text': 'this is not the original transcript', 'start': 1.54, 'duration': 4.16}, {'text': 'just something shorter, I made up for testing', 'start': 5.7, 'duration': 3.239} - ] + ]) ) @patch('youtube_transcript_api.YouTubeTranscriptApi.get_transcript') @@ -191,8 +194,8 @@ class TestYouTubeTranscriptApi(TestCase): YouTubeTranscriptApi.get_transcripts([video_id_1, video_id_2], languages=languages) - mock_get_transcript.assert_any_call(video_id_1, languages, None, None) - mock_get_transcript.assert_any_call(video_id_2, languages, None, None) + mock_get_transcript.assert_any_call(video_id_1, languages, None, None, format=None) + mock_get_transcript.assert_any_call(video_id_2, languages, None, None, format=None) self.assertEqual(mock_get_transcript.call_count, 2) @patch('youtube_transcript_api.YouTubeTranscriptApi.get_transcript', side_effect=Exception('Error')) @@ -207,20 +210,20 @@ class TestYouTubeTranscriptApi(TestCase): YouTubeTranscriptApi.get_transcripts(['video_id_1', 'video_id_2'], continue_after_error=True) - mock_get_transcript.assert_any_call(video_id_1, ('en',), None, None) - mock_get_transcript.assert_any_call(video_id_2, ('en',), None, None) + mock_get_transcript.assert_any_call(video_id_1, ('en',), None, None, format=None) + mock_get_transcript.assert_any_call(video_id_2, ('en',), None, None, format=None) @patch('youtube_transcript_api.YouTubeTranscriptApi.get_transcript') def test_get_transcripts__with_cookies(self, mock_get_transcript): cookies = '/example_cookies.txt' YouTubeTranscriptApi.get_transcripts(['GJLlxj_dtq8'], cookies=cookies) - mock_get_transcript.assert_any_call('GJLlxj_dtq8', ('en',), None, cookies) + mock_get_transcript.assert_any_call('GJLlxj_dtq8', ('en',), None, cookies, format=None) @patch('youtube_transcript_api.YouTubeTranscriptApi.get_transcript') def test_get_transcripts__with_proxies(self, mock_get_transcript): proxies = {'http': '', 'https:': ''} YouTubeTranscriptApi.get_transcripts(['GJLlxj_dtq8'], proxies=proxies) - mock_get_transcript.assert_any_call('GJLlxj_dtq8', ('en',), proxies, None) + mock_get_transcript.assert_any_call('GJLlxj_dtq8', ('en',), proxies, None, format=None) def test_load_cookies(self): dirname, filename = os.path.split(os.path.abspath(__file__)) From 0e6fae250453e23baa200812ccfb260f0b77b59a Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Wed, 8 Jul 2020 15:14:32 -0700 Subject: [PATCH 03/22] replace CLI json arg with format in test_cli.py Replaced every occurance of `parsed_args.json` with `parsed_args.format` as well as the CLI arguments to match for testing. --- youtube_transcript_api/test/test_cli.py | 45 ++++++++++++------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/youtube_transcript_api/test/test_cli.py b/youtube_transcript_api/test/test_cli.py index 158cd35..ba74c77 100644 --- a/youtube_transcript_api/test/test_cli.py +++ b/youtube_transcript_api/test/test_cli.py @@ -25,50 +25,50 @@ class TestYouTubeTranscriptCli(TestCase): YouTubeTranscriptApi.list_transcripts = MagicMock(return_value=self.transcript_list_mock) def test_argument_parsing(self): - parsed_args = YouTubeTranscriptCli('v1 v2 --json --languages de en'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('v1 v2 --format json --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, '') self.assertEqual(parsed_args.https_proxy, '') - parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en --json'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en --format json'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, '') self.assertEqual(parsed_args.https_proxy, '') - parsed_args = YouTubeTranscriptCli(' --json v1 v2 --languages de en'.split())._parse_args() + parsed_args = YouTubeTranscriptCli(' --format json v1 v2 --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, '') self.assertEqual(parsed_args.https_proxy, '') parsed_args = YouTubeTranscriptCli( - 'v1 v2 --languages de en --json --http-proxy http://user:pass@domain:port --https-proxy https://user:pass@domain:port'.split() + 'v1 v2 --languages de en --format json --http-proxy http://user:pass@domain:port --https-proxy https://user:pass@domain:port'.split() )._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, 'http://user:pass@domain:port') self.assertEqual(parsed_args.https_proxy, 'https://user:pass@domain:port') parsed_args = YouTubeTranscriptCli( - 'v1 v2 --languages de en --json --http-proxy http://user:pass@domain:port'.split() + 'v1 v2 --languages de en --format json --http-proxy http://user:pass@domain:port'.split() )._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, 'http://user:pass@domain:port') self.assertEqual(parsed_args.https_proxy, '') parsed_args = YouTubeTranscriptCli( - 'v1 v2 --languages de en --json --https-proxy https://user:pass@domain:port'.split() + 'v1 v2 --languages de en --format json --https-proxy https://user:pass@domain:port'.split() )._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.https_proxy, 'https://user:pass@domain:port') self.assertEqual(parsed_args.http_proxy, '') @@ -76,28 +76,28 @@ class TestYouTubeTranscriptCli(TestCase): def test_argument_parsing__only_video_ids(self): parsed_args = YouTubeTranscriptCli('v1 v2'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, False) + self.assertEqual(parsed_args.format, None) self.assertEqual(parsed_args.languages, ['en']) def test_argument_parsing__fail_without_video_ids(self): with self.assertRaises(SystemExit): - YouTubeTranscriptCli('--json'.split())._parse_args() + YouTubeTranscriptCli('--format json'.split())._parse_args() def test_argument_parsing__json(self): - parsed_args = YouTubeTranscriptCli('v1 v2 --json'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('v1 v2 --format json'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['en']) - parsed_args = YouTubeTranscriptCli('--json v1 v2'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('--format json v1 v2'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['en']) def test_argument_parsing__languages(self): parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, False) + self.assertEqual(parsed_args.format, None) self.assertEqual(parsed_args.languages, ['de', 'en']) def test_argument_parsing__proxies(self): @@ -135,13 +135,13 @@ class TestYouTubeTranscriptCli(TestCase): def test_argument_parsing__translate(self): parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en --translate cz'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, False) + self.assertEqual(parsed_args.format, None) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.translate, 'cz') parsed_args = YouTubeTranscriptCli('v1 v2 --translate cz --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, False) + self.assertEqual(parsed_args.format, None) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.translate, 'cz') @@ -204,8 +204,7 @@ class TestYouTubeTranscriptCli(TestCase): YouTubeTranscriptApi.list_transcripts.assert_any_call('v2', proxies=None, cookies=None) def test_run__json_output(self): - output = YouTubeTranscriptCli('v1 v2 --languages de en --json'.split()).run() - + output = YouTubeTranscriptCli('v1 v2 --languages de en --format json'.split()).run() # will fail if output is not valid json json.loads(output) From b4592043dc68b115effa180fd2e3755de371a887 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Wed, 8 Jul 2020 15:20:47 -0700 Subject: [PATCH 04/22] Add format kwarg to method calls to pass around Add format kwarg that defaults to None which still gets interpreted to use JSON when its not passed. This kwarg was given to `.get_transcripts()` and `.get_transcript()` since one relies on the other therefore can forward the kwarg. --- youtube_transcript_api/_api.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/youtube_transcript_api/_api.py b/youtube_transcript_api/_api.py index 389cf31..10766fa 100644 --- a/youtube_transcript_api/_api.py +++ b/youtube_transcript_api/_api.py @@ -12,6 +12,8 @@ from ._errors import ( CookiePathInvalid, CookiesInvalid ) +from .formatters import formats + class YouTubeTranscriptApi(): @classmethod @@ -70,7 +72,8 @@ class YouTubeTranscriptApi(): return TranscriptListFetcher(http_client).fetch(video_id) @classmethod - def get_transcripts(cls, video_ids, languages=('en',), continue_after_error=False, proxies=None, cookies=None): + def get_transcripts(cls, video_ids, languages=('en',), + continue_after_error=False, proxies=None, cookies=None, format=None): """ Retrieves the transcripts for a list of videos. @@ -96,7 +99,8 @@ class YouTubeTranscriptApi(): for video_id in video_ids: try: - data[video_id] = cls.get_transcript(video_id, languages, proxies, cookies) + data[video_id] = cls.get_transcript(video_id, languages, + proxies, cookies, format=format) except Exception as exception: if not continue_after_error: raise exception @@ -106,7 +110,8 @@ class YouTubeTranscriptApi(): return data, unretrievable_videos @classmethod - def get_transcript(cls, video_id, languages=('en',), proxies=None, cookies=None): + def get_transcript(cls, video_id, languages=('en',), proxies=None, + cookies=None, format=None): """ Retrieves the transcript for a single video. This is just a shortcut for calling:: @@ -125,7 +130,10 @@ class YouTubeTranscriptApi(): :return: a list of dictionaries containing the 'text', 'start' and 'duration' keys :rtype [{'text': str, 'start': float, 'end': float}]: """ - return cls.list_transcripts(video_id, proxies, cookies).find_transcript(languages).fetch() + Formatter = formats.get_formatter(format) + transcript = cls.list_transcripts(video_id, proxies, cookies, + format=format).find_transcript(languages).fetch() + return ''.join(Formatter.format(transcript)) @classmethod def _load_cookies(cls, cookies, video_id): From c78a37b115e4292419f826401acdd811f2350fa1 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Wed, 8 Jul 2020 15:23:10 -0700 Subject: [PATCH 05/22] Update _cli.py Add formats factory instance that uses the `parsed_args.format` arg to retrieve the formatter class, defaults to JSON if not passed or if given a bad/mistyped name. Might consider error in the case of a bad name given. Shouldn't be too difficult to add that ability if its wanted. --- youtube_transcript_api/_cli.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/youtube_transcript_api/_cli.py b/youtube_transcript_api/_cli.py index 405d6e1..0fdb8e5 100644 --- a/youtube_transcript_api/_cli.py +++ b/youtube_transcript_api/_cli.py @@ -1,10 +1,9 @@ import json -import pprint - import argparse from ._api import YouTubeTranscriptApi +from .formatters import formats class YouTubeTranscriptCli(): @@ -32,9 +31,11 @@ class YouTubeTranscriptCli(): except Exception as exception: exceptions.append(exception) - return '\n\n'.join( - [str(exception) for exception in exceptions] - + ([json.dumps(transcripts) if parsed_args.json else pprint.pformat(transcripts)] if transcripts else []) + Formatter = formats.get_formatter(parsed_args.format) + results = Formatter.format(transcripts) + + return ''.join( + [str(exception) for exception in exceptions] + results ) def _fetch_transcript(self, parsed_args, proxies, cookies, video_id): @@ -98,11 +99,9 @@ class YouTubeTranscriptCli(): help='If this flag is set transcripts which have been manually created will not be retrieved.', ) parser.add_argument( - '--json', - action='store_const', - const=True, - default=False, - help='If this flag is set the output will be JSON formatted.', + '--format', + default=None, + help="Use this flag to set which parser format to use, default is 'json'", ) parser.add_argument( '--translate', From 2c79bd563c2af4bd9e3dd7333172071cc3b4ace8 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Wed, 8 Jul 2020 15:25:32 -0700 Subject: [PATCH 06/22] Fix _api.py old kwarg --- youtube_transcript_api/_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_transcript_api/_api.py b/youtube_transcript_api/_api.py index 10766fa..3cd602a 100644 --- a/youtube_transcript_api/_api.py +++ b/youtube_transcript_api/_api.py @@ -131,8 +131,8 @@ class YouTubeTranscriptApi(): :rtype [{'text': str, 'start': float, 'end': float}]: """ Formatter = formats.get_formatter(format) - transcript = cls.list_transcripts(video_id, proxies, cookies, - format=format).find_transcript(languages).fetch() + transcript = cls.list_transcripts( + video_id,proxies, cookies).find_transcript(languages).fetch() return ''.join(Formatter.format(transcript)) @classmethod From 1c0d5849594183bae21f2f3dd003ee87db1d685d Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Thu, 9 Jul 2020 00:04:08 -0700 Subject: [PATCH 07/22] Add more functionality to TranscriptFormatter base class Due to the behavior of the CLI and API, needed more flexibility for combining 1 or many transcripts for a given formatter. - Now can specify a DELIMITER to separate multiple transcripts on. - Can also specify how those items are combine overriding the combine class method. Remove unused imports Fix adjust some lines to meet PEP --- youtube_transcript_api/formatters.py | 95 ++++++++++++++++------------ 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/youtube_transcript_api/formatters.py b/youtube_transcript_api/formatters.py index a5f30cd..54f7815 100644 --- a/youtube_transcript_api/formatters.py +++ b/youtube_transcript_api/formatters.py @@ -1,13 +1,9 @@ -from abc import ABCMeta +from abc import ABC from abc import abstractclassmethod from collections import defaultdict import json import re -from xml.etree import ElementTree - -from ._html_unescaping import unescape - def parse_timecode(time): """Converts a `time` into a formatted transcript timecode. @@ -31,23 +27,37 @@ def parse_timecode(time): return f"{hours}:{mins}:{secs},{ms}" -class TranscriptFormatter(metaclass=ABCMeta): - """ - Abstract Base TranscriptFormatter class +class TranscriptFormatter(ABC): + """Abstract Base TranscriptFormatter class This class should be inherited from to create additional custom transcript formatters. - """ HTML_TAG_REGEX = re.compile(r'<[^>]*>', re.IGNORECASE) - + DELIMITER = '' + + @classmethod + def combine(cls, transcripts): + """Subclass may override this class method. + + Default behavior of this method will ''.join() the str() + of each transcript in transcripts. + + :param transcripts: a list of many transcripts + :type transcript_data: list[, ...] + :return: A string joined on the `cls.DELIMITER` to combine transcripts + :rtype: str + """ + return cls.DELIMITER.join( + str(transcript) for transcript in transcripts) + @abstractclassmethod def format(cls, transcript_data): """Any subclass must implement this format class method. :param transcript_data: a list of transcripts, 1 or more. :type transcript_data: list[list[dict], list[dict]] - :return: A list where each item is an individual transcript + :return: A list where each item is an individual transcript as a string. :rtype: list[str] """ @@ -56,9 +66,15 @@ class TranscriptFormatter(metaclass=ABCMeta): class JSONTranscriptFormatter(TranscriptFormatter): """Formatter for outputting JSON data""" + DELIMITER = ',' + + @classmethod + def combine(cls, transcripts): + return json.dumps(transcripts) + @classmethod def format(cls, transcript_data): - return [json.dumps(transcript_data)] if transcript_data else [] + return transcript_data class TextTranscriptFormatter(TranscriptFormatter): @@ -66,55 +82,56 @@ class TextTranscriptFormatter(TranscriptFormatter): Converts the fetched transcript data into separated lines of plain text separated by newline breaks (\n) with no timecodes. - """ + DELIMITER = '\n\n' + @classmethod def format(cls, transcript_data): - return ['\n'.join(line['text'] for transcript in transcript_data - for line in transcript)] + return '{}\n'.format('\n'.join( + line['text']for line in transcript_data)) class SRTTranscriptFormatter(TranscriptFormatter): """Formatter for outputting the SRT Format - - Converts the fetched transcript data into a simple .srt file format. + Converts the fetched transcript data into a simple .srt file format. """ + DELIMITER = '\n\n' + @classmethod def format(cls, transcript_data): - contents = [] - for transcript in transcript_data: - content = [] - for frame, item in enumerate(transcript, start=1): - start_time = float(item.get('start')) - duration = float(item.get('dur', '0.0')) + output = [] + for frame, item in enumerate(transcript_data, start=1): + start_time = float(item.get('start')) + duration = float(item.get('dur', '0.0')) - end_time = parse_timecode(start_time + duration) - start_time = parse_timecode(start_time) + end_time = parse_timecode(start_time + duration) + start_time = parse_timecode(start_time) - content.append("{frame}\n".format(frame=frame)) - content.append("{start_time} --> {end_time}\n".format( - start_time=start_time, end_time=end_time)) - content.append("{text}\n\n".format(text=item.get('text'))) - - contents.append(''.join(content)) - return ['\n\n'.join(contents)] + output.append("{frame}\n".format(frame=frame)) + output.append("{start_time} --> {end_time}\n".format( + start_time=start_time, end_time=end_time)) + output.append("{text}".format(text=item.get('text'))) + if frame < len(transcript_data): + output.append('\n\n') + + return '{}\n'.format(''.join(output)) class TranscriptFormatterFactory: """A Transcript Class Factory - + Allows for adding additional custom Transcript classes for the API - to use. Custom Transcript classes must inherit from the + to use. Custom Transcript classes must inherit from the TranscriptFormatter abstract base class. """ def __init__(self): self._formatters = defaultdict(JSONTranscriptFormatter) - + def add_formatter(self, name, formatter_class): """Allows for creating additional transcript formatters. - + :param name: a name given to the `formatter_class` :type name: str :param formatter_class: a subclass of TranscriptFormatter @@ -124,8 +141,8 @@ class TranscriptFormatterFactory: if not issubclass(formatter_class, TranscriptFormatter): raise TypeError( f'{formatter_class} must be a subclass of TranscriptFormatter') - self._formatters.update({name:formatter_class}) - + self._formatters.update({name: formatter_class}) + def add_formatters(self, formatters_dict): """Allow creation of multiple transcript formatters at a time. @@ -137,7 +154,7 @@ class TranscriptFormatterFactory: """ for name, formatter_class in formatters_dict.items(): self.add_formatter(name, formatter_class) - + def get_formatter(self, name): """Retrieve a formatter class by its assigned name. From 5861bdb10471e00e92941199dfaef07e2d49269a Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Thu, 9 Jul 2020 00:06:14 -0700 Subject: [PATCH 08/22] Remove json.dumps from test_api.py Remove unnecessary changes to tests, revert assertions back to how they were. Remove unnecessary join from formatter in `_api.py` --- youtube_transcript_api/_api.py | 4 ++-- youtube_transcript_api/test/test_api.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/youtube_transcript_api/_api.py b/youtube_transcript_api/_api.py index 3cd602a..ef33e01 100644 --- a/youtube_transcript_api/_api.py +++ b/youtube_transcript_api/_api.py @@ -133,8 +133,8 @@ class YouTubeTranscriptApi(): Formatter = formats.get_formatter(format) transcript = cls.list_transcripts( video_id,proxies, cookies).find_transcript(languages).fetch() - return ''.join(Formatter.format(transcript)) - + return Formatter.format(transcript) + @classmethod def _load_cookies(cls, cookies, video_id): cookie_jar = {} diff --git a/youtube_transcript_api/test/test_api.py b/youtube_transcript_api/test/test_api.py index dc552d4..3a09a63 100644 --- a/youtube_transcript_api/test/test_api.py +++ b/youtube_transcript_api/test/test_api.py @@ -50,11 +50,11 @@ class TestYouTubeTranscriptApi(TestCase): self.assertEqual( transcript, - json.dumps([ + [ {'text': 'Hey, this is just a test', 'start': 0.0, 'duration': 1.54}, {'text': 'this is not the original transcript', 'start': 1.54, 'duration': 4.16}, {'text': 'just something shorter, I made up for testing', 'start': 5.7, 'duration': 3.239} - ]) + ] ) def test_list_transcripts(self): @@ -165,11 +165,11 @@ class TestYouTubeTranscriptApi(TestCase): ) self.assertEqual( transcript, - json.dumps([ + [ {'text': 'Hey, this is just a test', 'start': 0.0, 'duration': 1.54}, {'text': 'this is not the original transcript', 'start': 1.54, 'duration': 4.16}, {'text': 'just something shorter, I made up for testing', 'start': 5.7, 'duration': 3.239} - ]) + ] ) def test_get_transcript__with_cookies(self): @@ -179,11 +179,11 @@ class TestYouTubeTranscriptApi(TestCase): self.assertEqual( transcript, - json.dumps([ + [ {'text': 'Hey, this is just a test', 'start': 0.0, 'duration': 1.54}, {'text': 'this is not the original transcript', 'start': 1.54, 'duration': 4.16}, {'text': 'just something shorter, I made up for testing', 'start': 5.7, 'duration': 3.239} - ]) + ] ) @patch('youtube_transcript_api.YouTubeTranscriptApi.get_transcript') From 74d36a821e3d7e9cc907bf0ce0fff38e4a627f44 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Thu, 9 Jul 2020 00:06:36 -0700 Subject: [PATCH 09/22] Update CLI to make use of new DELIMITER and combine method --- youtube_transcript_api/_cli.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/youtube_transcript_api/_cli.py b/youtube_transcript_api/_cli.py index 0fdb8e5..8526b1f 100644 --- a/youtube_transcript_api/_cli.py +++ b/youtube_transcript_api/_cli.py @@ -25,21 +25,24 @@ class YouTubeTranscriptCli(): transcripts = [] exceptions = [] + Formatter = formats.get_formatter(parsed_args.format) + for video_id in parsed_args.video_ids: try: - transcripts.append(self._fetch_transcript(parsed_args, proxies, cookies, video_id)) + transcript = self._fetch_transcript( + parsed_args, proxies, cookies, video_id) + transcripts.append(Formatter.format(transcript)) except Exception as exception: exceptions.append(exception) - Formatter = formats.get_formatter(parsed_args.format) - results = Formatter.format(transcripts) - return ''.join( - [str(exception) for exception in exceptions] + results + [str(exception) for exception in exceptions] + + ([Formatter.combine(transcripts)] if transcripts else []) ) def _fetch_transcript(self, parsed_args, proxies, cookies, video_id): - transcript_list = YouTubeTranscriptApi.list_transcripts(video_id, proxies=proxies, cookies=cookies) + transcript_list = YouTubeTranscriptApi.list_transcripts( + video_id, proxies=proxies, cookies=cookies) if parsed_args.list_transcripts: return str(transcript_list) From c4b8b5b18d6dc473350d67323728687c86bb6dd1 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Thu, 9 Jul 2020 20:13:58 -0700 Subject: [PATCH 10/22] formatters.py python 2 compliant and misc. Remove use of ABC, not part of Python 2 and implementing ABCMeta changes between Python 2 and 3 so left it out entirely. Base class now raises NotImplementedError manually. Fix parse_timecode issue with start and end times being identical Replaced uses of F-strings with .format() also for compatibility. --- youtube_transcript_api/formatters.py | 43 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/youtube_transcript_api/formatters.py b/youtube_transcript_api/formatters.py index 54f7815..53daa55 100644 --- a/youtube_transcript_api/formatters.py +++ b/youtube_transcript_api/formatters.py @@ -1,5 +1,3 @@ -from abc import ABC -from abc import abstractclassmethod from collections import defaultdict import json import re @@ -17,17 +15,18 @@ def parse_timecode(time): >>> parse_timecode(6.93) '00:00:06,930' """ + time = float(time) - hours, mins, secs = ( - str(int(time)//3600).rjust(2, '0'), - str(int(time)//60).rjust(2, '0'), - str(int(time)%60).rjust(2, '0'), - ) - ms = str(int(round((time - int(time))*1000, 2))).rjust(3, '0') - return f"{hours}:{mins}:{secs},{ms}" + times = { + 'hours': str(int(time) // 3600).rjust(2, '0'), + 'mins': str(int(time) // 60).rjust(2, '0'), + 'secs': str(int(time) % 60).rjust(2, '0'), + 'ms': str(int(round((time - int(time))*1000, 2))).rjust(3, '0') + } + return "{hours}:{mins}:{secs},{ms}".format(**times) -class TranscriptFormatter(ABC): +class TranscriptFormatter(object): """Abstract Base TranscriptFormatter class This class should be inherited from to create additional @@ -51,7 +50,7 @@ class TranscriptFormatter(ABC): return cls.DELIMITER.join( str(transcript) for transcript in transcripts) - @abstractclassmethod + @classmethod def format(cls, transcript_data): """Any subclass must implement this format class method. @@ -61,7 +60,9 @@ class TranscriptFormatter(ABC): as a string. :rtype: list[str] """ - pass + raise NotImplementedError( + cls.__name__ + '.format' + ) class JSONTranscriptFormatter(TranscriptFormatter): @@ -103,22 +104,20 @@ class SRTTranscriptFormatter(TranscriptFormatter): output = [] for frame, item in enumerate(transcript_data, start=1): start_time = float(item.get('start')) - duration = float(item.get('dur', '0.0')) - - end_time = parse_timecode(start_time + duration) - start_time = parse_timecode(start_time) + duration = float(item.get('duration', '0.0')) output.append("{frame}\n".format(frame=frame)) output.append("{start_time} --> {end_time}\n".format( - start_time=start_time, end_time=end_time)) + start_time=parse_timecode(start_time), + end_time=parse_timecode(start_time + duration) + )) output.append("{text}".format(text=item.get('text'))) if frame < len(transcript_data): output.append('\n\n') - return '{}\n'.format(''.join(output)) -class TranscriptFormatterFactory: +class TranscriptFormatterFactory(object): """A Transcript Class Factory Allows for adding additional custom Transcript classes for the API @@ -139,8 +138,10 @@ class TranscriptFormatterFactory: :rtype None """ if not issubclass(formatter_class, TranscriptFormatter): - raise TypeError( - f'{formatter_class} must be a subclass of TranscriptFormatter') + raise TypeError(( + '{0} must be a subclass of TranscriptFormatter' + ).format(formatter_class) + ) self._formatters.update({name: formatter_class}) def add_formatters(self, formatters_dict): From 216abbe0233faef79f6b4eaee5e15d902529de4c Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Thu, 9 Jul 2020 20:14:36 -0700 Subject: [PATCH 11/22] Add unittests for formatters.py --- .../test/test_formatters.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 youtube_transcript_api/test/test_formatters.py diff --git a/youtube_transcript_api/test/test_formatters.py b/youtube_transcript_api/test/test_formatters.py new file mode 100644 index 0000000..df56a6d --- /dev/null +++ b/youtube_transcript_api/test/test_formatters.py @@ -0,0 +1,99 @@ +from unittest import TestCase +from mock import MagicMock +import json + +from youtube_transcript_api.formatters import ( + JSONTranscriptFormatter, + parse_timecode, + SRTTranscriptFormatter, + TextTranscriptFormatter, + TranscriptFormatter, + TranscriptFormatterFactory +) + + +class TestTranscriptFormatters(TestCase): + @classmethod + def setUpClass(cls): + cls.transcript = [ + { + 'text': 'Hey, this is just a test', + 'start': 0.0, + 'duration': 1.54 + }, + { + 'text': 'this is not the original transcript', + 'start': 1.54, + 'duration': 4.16 + }, + { + 'text': 'just something shorter, I made up for testing', + 'start': 5.7, + 'duration': 3.239 + } + ] + + def test_base_formatter_combine(self): + expecting = ''.join([str(line) for line in self.transcript]) + + self.assertEqual( + TranscriptFormatter.combine(self.transcript), + expecting + ) + + def test_base_format_not_implemented(self): + with self.assertRaises(NotImplementedError): + TranscriptFormatter.format(self.transcript) + + def test_text_formatter_format(self): + text = '\n'.join([line.get('text') for line in self.transcript]) + text_fmt = TextTranscriptFormatter.format(self.transcript) + self.assertIn(text + '\n', text_fmt) + + def test_srt_formatter_format(self): + start = self.transcript[0].get('start') + duration = self.transcript[0].get('duration') + srt_fmt = SRTTranscriptFormatter.format(self.transcript) + self.assertIn('{start} --> {end}'.format( + start=parse_timecode(start), + end=parse_timecode(start+duration) + ), srt_fmt) + + def test_json_formatter_format(self): + json_fmt = JSONTranscriptFormatter.format(self.transcript) + self.assertIsInstance(json.dumps(json_fmt), str) + + def test_invalid_parse_timecode(self): + start_time = 'not_float' + + with self.assertRaises(ValueError): + parse_timecode(start_time) + + def test_valid_parse_timecode(self): + start_time = 0.0 + end_time = 5.20 + + self.assertEqual( + parse_timecode(start_time), + '00:00:00,000' + ) + + self.assertEqual( + parse_timecode(end_time), + '00:00:05,200' + ) + + def test_formatter_factory_valid_single_add(self): + factory = TranscriptFormatterFactory() + factory.add_formatter('json', JSONTranscriptFormatter) + + self.assertDictEqual( + getattr(factory, '_formatters'), + {'json': JSONTranscriptFormatter} + ) + + def test_formatter_factory_invalid_single_add(self): + factory = TranscriptFormatterFactory() + + with self.assertRaises(TypeError): + factory.add_formatter('magic', MagicMock) From d75ad8c40245d210711b9545a78fb39e8b86568b Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Fri, 21 Aug 2020 11:46:07 -0700 Subject: [PATCH 12/22] Revert "v0.3.1" This reverts commit dec09fa48ae9468203ca79887b10692fcbdc9194. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1490707..7263342 100644 --- a/setup.py +++ b/setup.py @@ -24,10 +24,10 @@ def get_test_suite(): setuptools.setup( name="youtube_transcript_api", - version="0.3.1", + version="0.3.0", author="Jonas Depoix", author_email="jonas.depoix@web.de", - description="This is an python API which allows you to get the transcripts/subtitles for a given YouTube video. It also works for automatically generated subtitles, supports translating subtitles and it does not require a headless browser, like other selenium based solutions do!", + description="This is an python API which allows you to get the transcripts/subtitles for a given YouTube video. It also works for automatically generated subtitles and it does not require a headless browser, like other selenium based solutions do!", long_description=get_long_description(), long_description_content_type="text/markdown", keywords="youtube-api subtitles youtube transcripts transcript subtitle youtube-subtitles youtube-transcripts cli", From 1e9b2c7727875502024a0c200c3a6f63d38b9282 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Fri, 21 Aug 2020 12:21:12 -0700 Subject: [PATCH 13/22] Revert "Add formatters module" Reverting again, apparently I had a misunderstanding of how to revert. Trying again. This reverts commit dca4021dd74f90d57dd3659ff84a5381feb4c9f2. --- setup.py | 4 +- youtube_transcript_api/_api.py | 18 +- youtube_transcript_api/_cli.py | 24 ++- youtube_transcript_api/formatters.py | 174 ------------------ youtube_transcript_api/test/test_api.py | 21 +-- youtube_transcript_api/test/test_cli.py | 45 ++--- .../test/test_formatters.py | 99 ---------- 7 files changed, 50 insertions(+), 335 deletions(-) delete mode 100644 youtube_transcript_api/formatters.py delete mode 100644 youtube_transcript_api/test/test_formatters.py diff --git a/setup.py b/setup.py index 7263342..1490707 100644 --- a/setup.py +++ b/setup.py @@ -24,10 +24,10 @@ def get_test_suite(): setuptools.setup( name="youtube_transcript_api", - version="0.3.0", + version="0.3.1", author="Jonas Depoix", author_email="jonas.depoix@web.de", - description="This is an python API which allows you to get the transcripts/subtitles for a given YouTube video. It also works for automatically generated subtitles and it does not require a headless browser, like other selenium based solutions do!", + description="This is an python API which allows you to get the transcripts/subtitles for a given YouTube video. It also works for automatically generated subtitles, supports translating subtitles and it does not require a headless browser, like other selenium based solutions do!", long_description=get_long_description(), long_description_content_type="text/markdown", keywords="youtube-api subtitles youtube transcripts transcript subtitle youtube-subtitles youtube-transcripts cli", diff --git a/youtube_transcript_api/_api.py b/youtube_transcript_api/_api.py index ef33e01..389cf31 100644 --- a/youtube_transcript_api/_api.py +++ b/youtube_transcript_api/_api.py @@ -12,8 +12,6 @@ from ._errors import ( CookiePathInvalid, CookiesInvalid ) -from .formatters import formats - class YouTubeTranscriptApi(): @classmethod @@ -72,8 +70,7 @@ class YouTubeTranscriptApi(): return TranscriptListFetcher(http_client).fetch(video_id) @classmethod - def get_transcripts(cls, video_ids, languages=('en',), - continue_after_error=False, proxies=None, cookies=None, format=None): + def get_transcripts(cls, video_ids, languages=('en',), continue_after_error=False, proxies=None, cookies=None): """ Retrieves the transcripts for a list of videos. @@ -99,8 +96,7 @@ class YouTubeTranscriptApi(): for video_id in video_ids: try: - data[video_id] = cls.get_transcript(video_id, languages, - proxies, cookies, format=format) + data[video_id] = cls.get_transcript(video_id, languages, proxies, cookies) except Exception as exception: if not continue_after_error: raise exception @@ -110,8 +106,7 @@ class YouTubeTranscriptApi(): return data, unretrievable_videos @classmethod - def get_transcript(cls, video_id, languages=('en',), proxies=None, - cookies=None, format=None): + def get_transcript(cls, video_id, languages=('en',), proxies=None, cookies=None): """ Retrieves the transcript for a single video. This is just a shortcut for calling:: @@ -130,11 +125,8 @@ class YouTubeTranscriptApi(): :return: a list of dictionaries containing the 'text', 'start' and 'duration' keys :rtype [{'text': str, 'start': float, 'end': float}]: """ - Formatter = formats.get_formatter(format) - transcript = cls.list_transcripts( - video_id,proxies, cookies).find_transcript(languages).fetch() - return Formatter.format(transcript) - + return cls.list_transcripts(video_id, proxies, cookies).find_transcript(languages).fetch() + @classmethod def _load_cookies(cls, cookies, video_id): cookie_jar = {} diff --git a/youtube_transcript_api/_cli.py b/youtube_transcript_api/_cli.py index 8526b1f..405d6e1 100644 --- a/youtube_transcript_api/_cli.py +++ b/youtube_transcript_api/_cli.py @@ -1,9 +1,10 @@ import json +import pprint + import argparse from ._api import YouTubeTranscriptApi -from .formatters import formats class YouTubeTranscriptCli(): @@ -25,24 +26,19 @@ class YouTubeTranscriptCli(): transcripts = [] exceptions = [] - Formatter = formats.get_formatter(parsed_args.format) - for video_id in parsed_args.video_ids: try: - transcript = self._fetch_transcript( - parsed_args, proxies, cookies, video_id) - transcripts.append(Formatter.format(transcript)) + transcripts.append(self._fetch_transcript(parsed_args, proxies, cookies, video_id)) except Exception as exception: exceptions.append(exception) - return ''.join( + return '\n\n'.join( [str(exception) for exception in exceptions] - + ([Formatter.combine(transcripts)] if transcripts else []) + + ([json.dumps(transcripts) if parsed_args.json else pprint.pformat(transcripts)] if transcripts else []) ) def _fetch_transcript(self, parsed_args, proxies, cookies, video_id): - transcript_list = YouTubeTranscriptApi.list_transcripts( - video_id, proxies=proxies, cookies=cookies) + transcript_list = YouTubeTranscriptApi.list_transcripts(video_id, proxies=proxies, cookies=cookies) if parsed_args.list_transcripts: return str(transcript_list) @@ -102,9 +98,11 @@ class YouTubeTranscriptCli(): help='If this flag is set transcripts which have been manually created will not be retrieved.', ) parser.add_argument( - '--format', - default=None, - help="Use this flag to set which parser format to use, default is 'json'", + '--json', + action='store_const', + const=True, + default=False, + help='If this flag is set the output will be JSON formatted.', ) parser.add_argument( '--translate', diff --git a/youtube_transcript_api/formatters.py b/youtube_transcript_api/formatters.py deleted file mode 100644 index 53daa55..0000000 --- a/youtube_transcript_api/formatters.py +++ /dev/null @@ -1,174 +0,0 @@ -from collections import defaultdict -import json -import re - - -def parse_timecode(time): - """Converts a `time` into a formatted transcript timecode. - - :param time: a float representing time in seconds. - :type time: float - :return: a string formatted as a timecode, 'HH:MM:SS,MS' - :rtype str - - :example: - >>> parse_timecode(6.93) - '00:00:06,930' - """ - - time = float(time) - times = { - 'hours': str(int(time) // 3600).rjust(2, '0'), - 'mins': str(int(time) // 60).rjust(2, '0'), - 'secs': str(int(time) % 60).rjust(2, '0'), - 'ms': str(int(round((time - int(time))*1000, 2))).rjust(3, '0') - } - return "{hours}:{mins}:{secs},{ms}".format(**times) - - -class TranscriptFormatter(object): - """Abstract Base TranscriptFormatter class - - This class should be inherited from to create additional - custom transcript formatters. - """ - HTML_TAG_REGEX = re.compile(r'<[^>]*>', re.IGNORECASE) - DELIMITER = '' - - @classmethod - def combine(cls, transcripts): - """Subclass may override this class method. - - Default behavior of this method will ''.join() the str() - of each transcript in transcripts. - - :param transcripts: a list of many transcripts - :type transcript_data: list[, ...] - :return: A string joined on the `cls.DELIMITER` to combine transcripts - :rtype: str - """ - return cls.DELIMITER.join( - str(transcript) for transcript in transcripts) - - @classmethod - def format(cls, transcript_data): - """Any subclass must implement this format class method. - - :param transcript_data: a list of transcripts, 1 or more. - :type transcript_data: list[list[dict], list[dict]] - :return: A list where each item is an individual transcript - as a string. - :rtype: list[str] - """ - raise NotImplementedError( - cls.__name__ + '.format' - ) - - -class JSONTranscriptFormatter(TranscriptFormatter): - """Formatter for outputting JSON data""" - DELIMITER = ',' - - @classmethod - def combine(cls, transcripts): - return json.dumps(transcripts) - - @classmethod - def format(cls, transcript_data): - return transcript_data - - -class TextTranscriptFormatter(TranscriptFormatter): - """Formatter for outputting a Plain Text Format - - Converts the fetched transcript data into separated lines of - plain text separated by newline breaks (\n) with no timecodes. - """ - DELIMITER = '\n\n' - - @classmethod - def format(cls, transcript_data): - return '{}\n'.format('\n'.join( - line['text']for line in transcript_data)) - - -class SRTTranscriptFormatter(TranscriptFormatter): - """Formatter for outputting the SRT Format - - Converts the fetched transcript data into a simple .srt file format. - """ - DELIMITER = '\n\n' - - @classmethod - def format(cls, transcript_data): - output = [] - for frame, item in enumerate(transcript_data, start=1): - start_time = float(item.get('start')) - duration = float(item.get('duration', '0.0')) - - output.append("{frame}\n".format(frame=frame)) - output.append("{start_time} --> {end_time}\n".format( - start_time=parse_timecode(start_time), - end_time=parse_timecode(start_time + duration) - )) - output.append("{text}".format(text=item.get('text'))) - if frame < len(transcript_data): - output.append('\n\n') - return '{}\n'.format(''.join(output)) - - -class TranscriptFormatterFactory(object): - """A Transcript Class Factory - - Allows for adding additional custom Transcript classes for the API - to use. Custom Transcript classes must inherit from the - TranscriptFormatter abstract base class. - """ - def __init__(self): - self._formatters = defaultdict(JSONTranscriptFormatter) - - def add_formatter(self, name, formatter_class): - """Allows for creating additional transcript formatters. - - - :param name: a name given to the `formatter_class` - :type name: str - :param formatter_class: a subclass of TranscriptFormatter - :type formatter_class: class - :rtype None - """ - if not issubclass(formatter_class, TranscriptFormatter): - raise TypeError(( - '{0} must be a subclass of TranscriptFormatter' - ).format(formatter_class) - ) - self._formatters.update({name: formatter_class}) - - def add_formatters(self, formatters_dict): - """Allow creation of multiple transcript formatters at a time. - - :param formatters_dict: key(s) are the string name to be given - to the formatter class, value for each key should be a subclass - of TranscriptFormatter. - :type formatters_dict: dict - :rtype None - """ - for name, formatter_class in formatters_dict.items(): - self.add_formatter(name, formatter_class) - - def get_formatter(self, name): - """Retrieve a formatter class by its assigned name. - - :param name: the string name given to the formatter class. - :type name: str - :return: a subclass of `TranscriptFormatter` - """ - return self._formatters[name] - - -formats = TranscriptFormatterFactory() -formats.add_formatters({ - 'json': JSONTranscriptFormatter, - 'srt': SRTTranscriptFormatter, - 'text': TextTranscriptFormatter -}) diff --git a/youtube_transcript_api/test/test_api.py b/youtube_transcript_api/test/test_api.py index 3a09a63..a081711 100644 --- a/youtube_transcript_api/test/test_api.py +++ b/youtube_transcript_api/test/test_api.py @@ -1,6 +1,6 @@ from unittest import TestCase from mock import patch -import json + import os import requests @@ -21,10 +21,7 @@ from youtube_transcript_api import ( def load_asset(filename): - filepath = '{dirname}/assets/{filename}'.format( - dirname=os.path.dirname(__file__), filename=filename) - - with open(filepath, 'r', encoding='utf-8') as file: + with open('{dirname}/assets/{filename}'.format(dirname=os.path.dirname(__file__), filename=filename)) as file: return file.read() @@ -161,7 +158,7 @@ class TestYouTubeTranscriptApi(TestCase): def test_get_transcript__with_proxy(self): proxies = {'http': '', 'https:': ''} transcript = YouTubeTranscriptApi.get_transcript( - 'GJLlxj_dtq8', proxies=proxies, format=None + 'GJLlxj_dtq8', proxies=proxies ) self.assertEqual( transcript, @@ -194,8 +191,8 @@ class TestYouTubeTranscriptApi(TestCase): YouTubeTranscriptApi.get_transcripts([video_id_1, video_id_2], languages=languages) - mock_get_transcript.assert_any_call(video_id_1, languages, None, None, format=None) - mock_get_transcript.assert_any_call(video_id_2, languages, None, None, format=None) + mock_get_transcript.assert_any_call(video_id_1, languages, None, None) + mock_get_transcript.assert_any_call(video_id_2, languages, None, None) self.assertEqual(mock_get_transcript.call_count, 2) @patch('youtube_transcript_api.YouTubeTranscriptApi.get_transcript', side_effect=Exception('Error')) @@ -210,20 +207,20 @@ class TestYouTubeTranscriptApi(TestCase): YouTubeTranscriptApi.get_transcripts(['video_id_1', 'video_id_2'], continue_after_error=True) - mock_get_transcript.assert_any_call(video_id_1, ('en',), None, None, format=None) - mock_get_transcript.assert_any_call(video_id_2, ('en',), None, None, format=None) + mock_get_transcript.assert_any_call(video_id_1, ('en',), None, None) + mock_get_transcript.assert_any_call(video_id_2, ('en',), None, None) @patch('youtube_transcript_api.YouTubeTranscriptApi.get_transcript') def test_get_transcripts__with_cookies(self, mock_get_transcript): cookies = '/example_cookies.txt' YouTubeTranscriptApi.get_transcripts(['GJLlxj_dtq8'], cookies=cookies) - mock_get_transcript.assert_any_call('GJLlxj_dtq8', ('en',), None, cookies, format=None) + mock_get_transcript.assert_any_call('GJLlxj_dtq8', ('en',), None, cookies) @patch('youtube_transcript_api.YouTubeTranscriptApi.get_transcript') def test_get_transcripts__with_proxies(self, mock_get_transcript): proxies = {'http': '', 'https:': ''} YouTubeTranscriptApi.get_transcripts(['GJLlxj_dtq8'], proxies=proxies) - mock_get_transcript.assert_any_call('GJLlxj_dtq8', ('en',), proxies, None, format=None) + mock_get_transcript.assert_any_call('GJLlxj_dtq8', ('en',), proxies, None) def test_load_cookies(self): dirname, filename = os.path.split(os.path.abspath(__file__)) diff --git a/youtube_transcript_api/test/test_cli.py b/youtube_transcript_api/test/test_cli.py index ba74c77..158cd35 100644 --- a/youtube_transcript_api/test/test_cli.py +++ b/youtube_transcript_api/test/test_cli.py @@ -25,50 +25,50 @@ class TestYouTubeTranscriptCli(TestCase): YouTubeTranscriptApi.list_transcripts = MagicMock(return_value=self.transcript_list_mock) def test_argument_parsing(self): - parsed_args = YouTubeTranscriptCli('v1 v2 --format json --languages de en'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('v1 v2 --json --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, 'json') + self.assertEqual(parsed_args.json, True) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, '') self.assertEqual(parsed_args.https_proxy, '') - parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en --format json'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en --json'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, 'json') + self.assertEqual(parsed_args.json, True) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, '') self.assertEqual(parsed_args.https_proxy, '') - parsed_args = YouTubeTranscriptCli(' --format json v1 v2 --languages de en'.split())._parse_args() + parsed_args = YouTubeTranscriptCli(' --json v1 v2 --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, 'json') + self.assertEqual(parsed_args.json, True) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, '') self.assertEqual(parsed_args.https_proxy, '') parsed_args = YouTubeTranscriptCli( - 'v1 v2 --languages de en --format json --http-proxy http://user:pass@domain:port --https-proxy https://user:pass@domain:port'.split() + 'v1 v2 --languages de en --json --http-proxy http://user:pass@domain:port --https-proxy https://user:pass@domain:port'.split() )._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, 'json') + self.assertEqual(parsed_args.json, True) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, 'http://user:pass@domain:port') self.assertEqual(parsed_args.https_proxy, 'https://user:pass@domain:port') parsed_args = YouTubeTranscriptCli( - 'v1 v2 --languages de en --format json --http-proxy http://user:pass@domain:port'.split() + 'v1 v2 --languages de en --json --http-proxy http://user:pass@domain:port'.split() )._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, 'json') + self.assertEqual(parsed_args.json, True) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, 'http://user:pass@domain:port') self.assertEqual(parsed_args.https_proxy, '') parsed_args = YouTubeTranscriptCli( - 'v1 v2 --languages de en --format json --https-proxy https://user:pass@domain:port'.split() + 'v1 v2 --languages de en --json --https-proxy https://user:pass@domain:port'.split() )._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, 'json') + self.assertEqual(parsed_args.json, True) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.https_proxy, 'https://user:pass@domain:port') self.assertEqual(parsed_args.http_proxy, '') @@ -76,28 +76,28 @@ class TestYouTubeTranscriptCli(TestCase): def test_argument_parsing__only_video_ids(self): parsed_args = YouTubeTranscriptCli('v1 v2'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, None) + self.assertEqual(parsed_args.json, False) self.assertEqual(parsed_args.languages, ['en']) def test_argument_parsing__fail_without_video_ids(self): with self.assertRaises(SystemExit): - YouTubeTranscriptCli('--format json'.split())._parse_args() + YouTubeTranscriptCli('--json'.split())._parse_args() def test_argument_parsing__json(self): - parsed_args = YouTubeTranscriptCli('v1 v2 --format json'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('v1 v2 --json'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, 'json') + self.assertEqual(parsed_args.json, True) self.assertEqual(parsed_args.languages, ['en']) - parsed_args = YouTubeTranscriptCli('--format json v1 v2'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('--json v1 v2'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, 'json') + self.assertEqual(parsed_args.json, True) self.assertEqual(parsed_args.languages, ['en']) def test_argument_parsing__languages(self): parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, None) + self.assertEqual(parsed_args.json, False) self.assertEqual(parsed_args.languages, ['de', 'en']) def test_argument_parsing__proxies(self): @@ -135,13 +135,13 @@ class TestYouTubeTranscriptCli(TestCase): def test_argument_parsing__translate(self): parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en --translate cz'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, None) + self.assertEqual(parsed_args.json, False) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.translate, 'cz') parsed_args = YouTubeTranscriptCli('v1 v2 --translate cz --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.format, None) + self.assertEqual(parsed_args.json, False) self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.translate, 'cz') @@ -204,7 +204,8 @@ class TestYouTubeTranscriptCli(TestCase): YouTubeTranscriptApi.list_transcripts.assert_any_call('v2', proxies=None, cookies=None) def test_run__json_output(self): - output = YouTubeTranscriptCli('v1 v2 --languages de en --format json'.split()).run() + output = YouTubeTranscriptCli('v1 v2 --languages de en --json'.split()).run() + # will fail if output is not valid json json.loads(output) diff --git a/youtube_transcript_api/test/test_formatters.py b/youtube_transcript_api/test/test_formatters.py deleted file mode 100644 index df56a6d..0000000 --- a/youtube_transcript_api/test/test_formatters.py +++ /dev/null @@ -1,99 +0,0 @@ -from unittest import TestCase -from mock import MagicMock -import json - -from youtube_transcript_api.formatters import ( - JSONTranscriptFormatter, - parse_timecode, - SRTTranscriptFormatter, - TextTranscriptFormatter, - TranscriptFormatter, - TranscriptFormatterFactory -) - - -class TestTranscriptFormatters(TestCase): - @classmethod - def setUpClass(cls): - cls.transcript = [ - { - 'text': 'Hey, this is just a test', - 'start': 0.0, - 'duration': 1.54 - }, - { - 'text': 'this is not the original transcript', - 'start': 1.54, - 'duration': 4.16 - }, - { - 'text': 'just something shorter, I made up for testing', - 'start': 5.7, - 'duration': 3.239 - } - ] - - def test_base_formatter_combine(self): - expecting = ''.join([str(line) for line in self.transcript]) - - self.assertEqual( - TranscriptFormatter.combine(self.transcript), - expecting - ) - - def test_base_format_not_implemented(self): - with self.assertRaises(NotImplementedError): - TranscriptFormatter.format(self.transcript) - - def test_text_formatter_format(self): - text = '\n'.join([line.get('text') for line in self.transcript]) - text_fmt = TextTranscriptFormatter.format(self.transcript) - self.assertIn(text + '\n', text_fmt) - - def test_srt_formatter_format(self): - start = self.transcript[0].get('start') - duration = self.transcript[0].get('duration') - srt_fmt = SRTTranscriptFormatter.format(self.transcript) - self.assertIn('{start} --> {end}'.format( - start=parse_timecode(start), - end=parse_timecode(start+duration) - ), srt_fmt) - - def test_json_formatter_format(self): - json_fmt = JSONTranscriptFormatter.format(self.transcript) - self.assertIsInstance(json.dumps(json_fmt), str) - - def test_invalid_parse_timecode(self): - start_time = 'not_float' - - with self.assertRaises(ValueError): - parse_timecode(start_time) - - def test_valid_parse_timecode(self): - start_time = 0.0 - end_time = 5.20 - - self.assertEqual( - parse_timecode(start_time), - '00:00:00,000' - ) - - self.assertEqual( - parse_timecode(end_time), - '00:00:05,200' - ) - - def test_formatter_factory_valid_single_add(self): - factory = TranscriptFormatterFactory() - factory.add_formatter('json', JSONTranscriptFormatter) - - self.assertDictEqual( - getattr(factory, '_formatters'), - {'json': JSONTranscriptFormatter} - ) - - def test_formatter_factory_invalid_single_add(self): - factory = TranscriptFormatterFactory() - - with self.assertRaises(TypeError): - factory.add_formatter('magic', MagicMock) From f3dc6f508f6ff3753798f2d8d66307fbb7641830 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Tue, 1 Sep 2020 15:21:47 -0700 Subject: [PATCH 14/22] Add new formatters.py module --- youtube_transcript_api/formatters.py | 91 ++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 youtube_transcript_api/formatters.py diff --git a/youtube_transcript_api/formatters.py b/youtube_transcript_api/formatters.py new file mode 100644 index 0000000..6696215 --- /dev/null +++ b/youtube_transcript_api/formatters.py @@ -0,0 +1,91 @@ +import json + + +class Formatter(object): + """Formatter should be used as an abstract base class. + + Formatter classes should inherit from this class and implement + their own .format() method which should return a string. A + transcript is represented by a List of Dictionary items. + + :param transcript: list representing 1 or more transcripts + :type transcript: list + """ + def __init__(self, transcript): + if not isinstance(transcript, list): + raise TypeError("'transcript' must be of type: List") + + self._transcript = transcript + + def format(self, **kwargs): + raise NotImplementedError('A subclass of Formatter must implement ' \ + 'their own .format() method.') + + +class JSONFormatter(Formatter): + def format(self, **kwargs): + """Converts a transcript into a JSON string. + + :return: A JSON string representation of the transcript.' + :rtype str + """ + return json.dumps(self._transcript, **kwargs) + + +class TextFormatter(Formatter): + def format(self, **kwargs): + """Converts a transcript into plain text with no timestamps. + + :return: all transcript text lines separated by newline breaks.' + :rtype str + """ + return "\n".join(line['text'] for line in self._transcript) + + +class WebVTTFormatter(Formatter): + def _seconds_to_timestamp(self, time): + """Helper that converts `time` into a transcript cue timestamp. + + :reference: https://www.w3.org/TR/webvtt1/#webvtt-timestamp + + :param time: a float representing time in seconds. + :type time: float + :return: a string formatted as a cue timestamp, 'HH:MM:SS.MS' + :rtype str + :example: + >>> self._seconds_to_timestamp(6.93) + '00:00:06.930' + """ + time = float(time) + hours, mins, secs = ( + int(time) // 3600, + int(time) // 60, + int(time) % 60, + ) + ms = int(round((time - int(time))*1000, 2)) + return "{:02d}:{:02d}:{:02d}.{:03d}".format(hours, mins, secs, ms) + + def format(self, **kwargs): + """A basic implementation of WEBVTT formatting. + + :reference: https://www.w3.org/TR/webvtt1/#introduction-caption + """ + lines = [] + for i, line in enumerate(self._transcript): + if i < len(self._transcript)-1: + # Looks ahead, use next start time since duration value + # would create an overlap between start times. + time_text = "{} --> {}".format( + self._seconds_to_timestamp(line['start']), + self._seconds_to_timestamp(self._transcript[i+1]['start']) + ) + else: + # Reached the end, cannot look ahead, use duration now. + duration = line['start'] + line['duration'] + time_text = "{} --> {}".format( + self._seconds_to_timestamp(line['start']), + self._seconds_to_timestamp(duration) + ) + lines.append("{}\n{}".format(time_text, line['text'])) + + return "WEBVTT\n\n" + "\n\n".join(lines) + "\n" From 36f29c30586dedc037eb597654b250d5897f79e3 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Tue, 1 Sep 2020 15:23:04 -0700 Subject: [PATCH 15/22] Add a few initial test cases, could use more --- .../test/test_formatters.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 youtube_transcript_api/test/test_formatters.py diff --git a/youtube_transcript_api/test/test_formatters.py b/youtube_transcript_api/test/test_formatters.py new file mode 100644 index 0000000..9550845 --- /dev/null +++ b/youtube_transcript_api/test/test_formatters.py @@ -0,0 +1,57 @@ +import json +from mock import MagicMock +from unittest import TestCase + +from youtube_transcript_api.formatters import ( + Formatter, + JSONFormatter, + TextFormatter, + WebVTTFormatter +) + + +class TestFormatters(TestCase): + def setUp(self): + self.transcript = [ + {'text': 'Test line 1', 'start': 0.0, 'duration': 1.50}, + {'text': 'line between', 'start': 1.5, 'duration': 2.0}, + {'text': 'testing the end line', 'start': 2.5, 'duration': 3.25} + ] + + def test_base_formatter_valid_type(self): + with self.assertRaises(TypeError) as err: + Formatter({"test": []}) + expected_err = "'transcript' must be of type: List" + self.assertEqual(expected_err, str(err.exception)) + + def test_base_formatter_format_call(self): + with self.assertRaises(NotImplementedError) as err: + Formatter(self.transcript).format() + + expected_err = "A subclass of Formatter must implement their own " \ + ".format() method." + self.assertEqual(expected_err, str(err.exception)) + + def test_webvtt_formatter_starting(self): + content = WebVTTFormatter(self.transcript).format() + lines = content.split('\n') + # test starting lines + self.assertEqual(lines[0], "WEBVTT") + self.assertEqual(lines[1], "") + + def test_webvtt_formatter_ending(self): + content = WebVTTFormatter(self.transcript).format() + lines = content.split('\n') + # test ending lines + self.assertEqual(lines[-2], self.transcript[-1]['text']) + self.assertEqual(lines[-1], "") + + def test_json_formatter(self): + content = JSONFormatter(self.transcript).format() + self.assertEqual(json.loads(content), self.transcript) + + def test_text_formatter(self): + content = TextFormatter(self.transcript).format() + lines = content.split('\n') + self.assertEqual(lines[0], self.transcript[0]["text"]) + self.assertEqual(lines[-1], self.transcript[-1]["text"]) From d3cd5d1254d768f0d8512ea9caafe4e895e6d2f3 Mon Sep 17 00:00:00 2001 From: Chris Howell Date: Tue, 1 Sep 2020 15:23:48 -0700 Subject: [PATCH 16/22] Add docs for formatters --- README.md | 559 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 312 insertions(+), 247 deletions(-) diff --git a/README.md b/README.md index 13a0cda..564e44a 100644 --- a/README.md +++ b/README.md @@ -1,247 +1,312 @@ - -# YouTube Transcript/Subtitle API (including automatically generated subtitles and subtitle translations) - -[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) [![Build Status](https://travis-ci.org/jdepoix/youtube-transcript-api.svg)](https://travis-ci.org/jdepoix/youtube-transcript-api) [![Coverage Status](https://coveralls.io/repos/github/jdepoix/youtube-transcript-api/badge.svg?branch=master)](https://coveralls.io/github/jdepoix/youtube-transcript-api?branch=master) [![MIT license](http://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](http://opensource.org/licenses/MIT) [![image](https://img.shields.io/pypi/v/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) [![image](https://img.shields.io/pypi/pyversions/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) - -This is an python API which allows you to get the transcripts/subtitles for a given YouTube video. It also works for automatically generated subtitles, supports translating subtitles and it does not require a headless browser, like other selenium based solutions do! - -## Install - -It is recommended to [install this module by using pip](https://pypi.org/project/youtube-transcript-api/): - -``` -pip install youtube_transcript_api -``` - -If you want to use it from source, you'll have to install the dependencies manually: - -``` -pip install -r requirements.txt -``` - -You can either integrate this module [into an existing application](#api), or just use it via an [CLI](#cli). - -## API - -The easiest way to get a transcript for a given video is to execute: - -```python -from youtube_transcript_api import YouTubeTranscriptApi - -YouTubeTranscriptApi.get_transcript(video_id) -``` - -This will return a list of dictionaries looking somewhat like this: - -```python -[ - { - 'text': 'Hey there', - 'start': 7.58, - 'duration': 6.13 - }, - { - 'text': 'how are you', - 'start': 14.08, - 'duration': 7.58 - }, - # ... -] -``` - -You can also add the `languages` param if you want to make sure the transcripts are retrieved in your desired language (it defaults to english). - -```python -YouTubeTranscriptApi.get_transcripts(video_ids, languages=['de', 'en']) -``` - -It's a list of language codes in a descending priority. In this example it will first try to fetch the german transcript (`'de'`) and then fetch the english transcript (`'en'`) if it fails to do so. If you want to find out which languages are available first, [have a look at `list_transcripts()`](#list-available-transcripts) - -To get transcripts for a list of video ids you can call: - -```python -YouTubeTranscriptApi.get_transcripts(video_ids, languages=['de', 'en']) -``` - -`languages` also is optional here. - -### List available transcripts - -If you want to list all transcripts which are available for a given video you can call: - -```python -transcript_list = YouTubeTranscriptApi.list_transcripts(video_id) -``` - -This will return a `TranscriptList` object which is iterable and provides methods to filter the list of transcripts for specific languages and types, like: - -```python -transcript = transcript_list.find_transcript(['de', 'en']) -``` - -By default this module always picks manually created transcripts over automatically created ones, if a transcript in the requested language is available both manually created and generated. The `TranscriptList` allows you to bypass this default behaviour by searching for specific transcript types: - -```python -# filter for manually created transcripts -transcript = transcript_list.find_manually_created_transcript(['de', 'en']) - -# or automatically generated ones -transcript = transcript_list.find_generated_transcript(['de', 'en']) -``` - -The methods `find_generated_transcript`, `find_manually_created_transcript`, `find_generated_transcript` return `Transcript` objects. They contain metadata regarding the transcript: - -```python -print( - transcript.video_id, - transcript.language, - transcript.language_code, - # whether it has been manually created or generated by YouTube - transcript.is_generated, - # whether this transcript can be translated or not - transcript.is_translatable, - # a list of languages the transcript can be translated to - transcript.translation_languages, -) -``` - -and provide the method, which allows you to fetch the actual transcript data: - -```python -transcript.fetch() -``` - -### Translate transcript - -YouTube has a feature which allows you to automatically translate subtitles. This module also makes it possible to access this feature. To do so `Transcript` objects provide a `translate()` method, which returns a new translated `Transcript` object: - -```python -transcript = transcript_list.find_transcript(['en']) -translated_transcript = transcript.translate('de') -print(translated_transcript.fetch()) -``` - -### By example -```python -# retrieve the available transcripts -transcript_list = YouTubeTranscriptApi.list_transcripts('video_id') - -# iterate over all available transcripts -for transcript in transcript_list: - - # the Transcript object provides metadata properties - print( - transcript.video_id, - transcript.language, - transcript.language_code, - # whether it has been manually created or generated by YouTube - transcript.is_generated, - # whether this transcript can be translated or not - transcript.is_translatable, - # a list of languages the transcript can be translated to - transcript.translation_languages, - ) - - # fetch the actual transcript data - print(transcript.fetch()) - - # translating the transcript will return another transcript object - print(transcript.translate('en').fetch()) - -# you can also directly filter for the language you are looking for, using the transcript list -transcript = transcript_list.find_transcript(['de', 'en']) - -# or just filter for manually created transcripts -transcript = transcript_list.find_manually_created_transcript(['de', 'en']) - -# or automatically generated ones -transcript = transcript_list.find_generated_transcript(['de', 'en']) -``` - -## CLI - -Execute the CLI script using the video ids as parameters and the results will be printed out to the command line: - -``` -youtube_transcript_api ... -``` - -The CLI also gives you the option to provide a list of preferred languages: - -``` -youtube_transcript_api ... --languages de en -``` - -You can also specify if you want to exclude automatically generated or manually created subtitles: - -``` -youtube_transcript_api ... --languages de en --exclude-generated -youtube_transcript_api ... --languages de en --exclude-manually-created -``` - -If you would prefer to write it into a file or pipe it into another application, you can also output the results as json using the following line: - -``` -youtube_transcript_api ... --languages de en --json > transcripts.json -``` - -Translating transcripts using the CLI is also possible: - -``` -youtube_transcript_api ... --languages en --translate de -``` - -If you are not sure which languages are available for a given video you can call, to list all available transcripts: - -``` -youtube_transcript_api --list-transcripts -``` - -## Proxy - -You can specify a https/http proxy, which will be used during the requests to YouTube: - -```python -from youtube_transcript_api import YouTubeTranscriptApi - -YouTubeTranscriptApi.get_transcript(video_id, proxies={"http": "http://user:pass@domain:port", "https": "https://user:pass@domain:port"}) -``` - -As the `proxies` dict is passed on to the `requests.get(...)` call, it follows the [format used by the requests library](http://docs.python-requests.org/en/master/user/advanced/#proxies). - -Using the CLI: - -``` -youtube_transcript_api --http-proxy http://user:pass@domain:port --https-proxy https://user:pass@domain:port -``` - -## Cookies - -Some videos are age restricted, so this module won't be able to access those videos without some sort of authentication. To do this, you will need to have access to the desired video in a browser. Then, you will need to download that pages cookies into a text file. You can use the Chrome extension [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg?hl=en) or the Firefox extension [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/). - -Once you have that, you can use it with the module to access age-restricted videos' captions like so. - -```python -from youtube_transcript_api import YouTubeTranscriptApi - -YouTubeTranscriptApi.get_transcript(video_id, cookies='/path/to/your/cookies.txt') - -YouTubeTranscriptApi.get_transcripts([video_id], cookies='/path/to/your/cookies.txt') -``` - -Using the CLI: - -``` -youtube_transcript_api --cookies /path/to/your/cookies.txt -``` - - -## Warning - - This code uses an undocumented part of the YouTube API, which is called by the YouTube web-client. So there is no guarantee that it won't stop working tomorrow, if they change how things work. I will however do my best to make things working again as soon as possible if that happens. So if it stops working, let me know! - -## Donation - -If this project makes you happy by reducing your development time, you can make me happy by treating me to a cup of coffee :) - -[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) + +# YouTube Transcript/Subtitle API (including automatically generated subtitles and subtitle translations) + +[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) [![Build Status](https://travis-ci.org/jdepoix/youtube-transcript-api.svg)](https://travis-ci.org/jdepoix/youtube-transcript-api) [![Coverage Status](https://coveralls.io/repos/github/jdepoix/youtube-transcript-api/badge.svg?branch=master)](https://coveralls.io/github/jdepoix/youtube-transcript-api?branch=master) [![MIT license](http://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](http://opensource.org/licenses/MIT) [![image](https://img.shields.io/pypi/v/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) [![image](https://img.shields.io/pypi/pyversions/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) + +This is an python API which allows you to get the transcripts/subtitles for a given YouTube video. It also works for automatically generated subtitles, supports translating subtitles and it does not require a headless browser, like other selenium based solutions do! + +## Install + +It is recommended to [install this module by using pip](https://pypi.org/project/youtube-transcript-api/): + +``` +pip install youtube_transcript_api +``` + +If you want to use it from source, you'll have to install the dependencies manually: + +``` +pip install -r requirements.txt +``` + +You can either integrate this module [into an existing application](#api), or just use it via an [CLI](#cli). + +## API + +The easiest way to get a transcript for a given video is to execute: + +```python +from youtube_transcript_api import YouTubeTranscriptApi + +YouTubeTranscriptApi.get_transcript(video_id) +``` + +This will return a list of dictionaries looking somewhat like this: + +```python +[ + { + 'text': 'Hey there', + 'start': 7.58, + 'duration': 6.13 + }, + { + 'text': 'how are you', + 'start': 14.08, + 'duration': 7.58 + }, + # ... +] +``` + +You can also add the `languages` param if you want to make sure the transcripts are retrieved in your desired language (it defaults to english). + +```python +YouTubeTranscriptApi.get_transcripts(video_ids, languages=['de', 'en']) +``` + +It's a list of language codes in a descending priority. In this example it will first try to fetch the german transcript (`'de'`) and then fetch the english transcript (`'en'`) if it fails to do so. If you want to find out which languages are available first, [have a look at `list_transcripts()`](#list-available-transcripts) + +To get transcripts for a list of video ids you can call: + +```python +YouTubeTranscriptApi.get_transcripts(video_ids, languages=['de', 'en']) +``` + +`languages` also is optional here. + +### List available transcripts + +If you want to list all transcripts which are available for a given video you can call: + +```python +transcript_list = YouTubeTranscriptApi.list_transcripts(video_id) +``` + +This will return a `TranscriptList` object which is iterable and provides methods to filter the list of transcripts for specific languages and types, like: + +```python +transcript = transcript_list.find_transcript(['de', 'en']) +``` + +By default this module always picks manually created transcripts over automatically created ones, if a transcript in the requested language is available both manually created and generated. The `TranscriptList` allows you to bypass this default behaviour by searching for specific transcript types: + +```python +# filter for manually created transcripts +transcript = transcript_list.find_manually_created_transcript(['de', 'en']) + +# or automatically generated ones +transcript = transcript_list.find_generated_transcript(['de', 'en']) +``` + +The methods `find_generated_transcript`, `find_manually_created_transcript`, `find_generated_transcript` return `Transcript` objects. They contain metadata regarding the transcript: + +```python +print( + transcript.video_id, + transcript.language, + transcript.language_code, + # whether it has been manually created or generated by YouTube + transcript.is_generated, + # whether this transcript can be translated or not + transcript.is_translatable, + # a list of languages the transcript can be translated to + transcript.translation_languages, +) +``` + +and provide the method, which allows you to fetch the actual transcript data: + +```python +transcript.fetch() +``` + +### Translate transcript + +YouTube has a feature which allows you to automatically translate subtitles. This module also makes it possible to access this feature. To do so `Transcript` objects provide a `translate()` method, which returns a new translated `Transcript` object: + +```python +transcript = transcript_list.find_transcript(['en']) +translated_transcript = transcript.translate('de') +print(translated_transcript.fetch()) +``` + +### By example +```python +# retrieve the available transcripts +transcript_list = YouTubeTranscriptApi.list_transcripts('video_id') + +# iterate over all available transcripts +for transcript in transcript_list: + + # the Transcript object provides metadata properties + print( + transcript.video_id, + transcript.language, + transcript.language_code, + # whether it has been manually created or generated by YouTube + transcript.is_generated, + # whether this transcript can be translated or not + transcript.is_translatable, + # a list of languages the transcript can be translated to + transcript.translation_languages, + ) + + # fetch the actual transcript data + print(transcript.fetch()) + + # translating the transcript will return another transcript object + print(transcript.translate('en').fetch()) + +# you can also directly filter for the language you are looking for, using the transcript list +transcript = transcript_list.find_transcript(['de', 'en']) + +# or just filter for manually created transcripts +transcript = transcript_list.find_manually_created_transcript(['de', 'en']) + +# or automatically generated ones +transcript = transcript_list.find_generated_transcript(['de', 'en']) +``` + +### Using Formatters +Formatters are meant to be an additional layer of processing of the transcript you pass it. The goal is to convert the transcript from its Python data type into a consistent string of a given "format". Such as a basic text (`.txt`) or even formats that have a defined specification such as JSON (`.json`), WebVTT format (`.vtt`), Comma-separated format (`.csv`), etc... + +The `formatters` submodule provides a few basic formatters to wrap around you transcript data in cases where you might want to do something such as output a specific format then write that format to a file. Maybe to backup/store and run another script against at a later time. + +We provided a few subclasses of formatters to use: + +- JSONFormatter +- TextFormatter +- WebVTTFormatter (a basic implementation) + +Here is how to import from the `formatters` module. + +```python +# the base class to inherit from when creating your own formatter. +from youtube_transcript_api.formatters import Formatter + +# some provided subclasses, each outputs a different string format. +from youtube_transcript_api.formatters import JSONFormatter +from youtube_transcript_api.formatters import TextFormatter +from youtube_transcript_api.formatters import WebVTTFormatter +``` + +### Provided Formatter Example +Lets say we wanted to retrieve a transcript and write that transcript as a JSON file in the same format as the API returned it as. That would look something like this: + +```python +# your_custom_script.py + +from youtube_transcript_api import YouTubeTranscriptApi +from youtube_transcript_api.formatters import JSONFormatter + +# Must be a single transcript. +transcript = YouTubeTranscriptApi.get_transcript(video_id) + +# .format() turns the transcript into a JSON string. +json_formatted = JSONFormatter(transcript).format() + + +# Now we can write it out to a file. +with open('your_filename.json', 'w', encoding='utf-8') as json_file: + json_file.write(json_formatted) + +# Now should have a new JSON file that you can easily read back into Python. +``` + +**Passing extra keyword arguments** + +Since JSONFormatter leverages `json.dumps()` you can also forward keyword arguments into `.format()` such as making your file output prettier by forwarding the `indent=2` keyword argument. + +```python +json_formatted = JSONFormatter(transcript).format(indent=2) +``` + +### Custom Formatter Example +You can implement your own formatter class. Just inherit from the `Formatter` base class and ensure you implement the `def format(self, **kwargs):` method which should ultimately return a string when called on your formatter instance. + +```python + +class MyCustomFormatter(Formatter): + def format(self, **kwargs): + # Do your custom work in here, but return a string. + return 'your processed output data as a string.' +``` + +## CLI + +Execute the CLI script using the video ids as parameters and the results will be printed out to the command line: + +``` +youtube_transcript_api ... +``` + +The CLI also gives you the option to provide a list of preferred languages: + +``` +youtube_transcript_api ... --languages de en +``` + +You can also specify if you want to exclude automatically generated or manually created subtitles: + +``` +youtube_transcript_api ... --languages de en --exclude-generated +youtube_transcript_api ... --languages de en --exclude-manually-created +``` + +If you would prefer to write it into a file or pipe it into another application, you can also output the results as json using the following line: + +``` +youtube_transcript_api ... --languages de en --json > transcripts.json +``` + +Translating transcripts using the CLI is also possible: + +``` +youtube_transcript_api ... --languages en --translate de +``` + +If you are not sure which languages are available for a given video you can call, to list all available transcripts: + +``` +youtube_transcript_api --list-transcripts +``` + +## Proxy + +You can specify a https/http proxy, which will be used during the requests to YouTube: + +```python +from youtube_transcript_api import YouTubeTranscriptApi + +YouTubeTranscriptApi.get_transcript(video_id, proxies={"http": "http://user:pass@domain:port", "https": "https://user:pass@domain:port"}) +``` + +As the `proxies` dict is passed on to the `requests.get(...)` call, it follows the [format used by the requests library](http://docs.python-requests.org/en/master/user/advanced/#proxies). + +Using the CLI: + +``` +youtube_transcript_api --http-proxy http://user:pass@domain:port --https-proxy https://user:pass@domain:port +``` + +## Cookies + +Some videos are age restricted, so this module won't be able to access those videos without some sort of authentication. To do this, you will need to have access to the desired video in a browser. Then, you will need to download that pages cookies into a text file. You can use the Chrome extension [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg?hl=en) or the Firefox extension [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/). + +Once you have that, you can use it with the module to access age-restricted videos' captions like so. + +```python +from youtube_transcript_api import YouTubeTranscriptApi + +YouTubeTranscriptApi.get_transcript(video_id, cookies='/path/to/your/cookies.txt') + +YouTubeTranscriptApi.get_transcripts([video_id], cookies='/path/to/your/cookies.txt') +``` + +Using the CLI: + +``` +youtube_transcript_api --cookies /path/to/your/cookies.txt +``` + + +## Warning + + This code uses an undocumented part of the YouTube API, which is called by the YouTube web-client. So there is no guarantee that it won't stop working tomorrow, if they change how things work. I will however do my best to make things working again as soon as possible if that happens. So if it stops working, let me know! + +## Donation + +If this project makes you happy by reducing your development time, you can make me happy by treating me to a cup of coffee :) + +[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) From d31413932932af597a230b5b9fa30ea003c40aa1 Mon Sep 17 00:00:00 2001 From: Jonas Depoix Date: Mon, 15 Mar 2021 17:16:15 +0100 Subject: [PATCH 17/22] added FormatterLoader --- youtube_transcript_api/_errors.py | 4 +- youtube_transcript_api/formatters.py | 40 ++++++++++++++++++- .../test/test_formatters.py | 36 +++++++++++++++-- 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/youtube_transcript_api/_errors.py b/youtube_transcript_api/_errors.py index 1b8360a..c3afb32 100644 --- a/youtube_transcript_api/_errors.py +++ b/youtube_transcript_api/_errors.py @@ -37,13 +37,15 @@ class CouldNotRetrieveTranscript(Exception): class VideoUnavailable(CouldNotRetrieveTranscript): CAUSE_MESSAGE = 'The video is no longer available' - + + class TooManyRequests(CouldNotRetrieveTranscript): CAUSE_MESSAGE = ("YouTube is receiving too many requests from this IP and now requires solving a captcha to continue. One of the following things can be done to work around this:\n\ - Manually solve the captcha in a browser and export the cookie. Read here how to use that cookie with youtube-transcript-api: https://github.com/jdepoix/youtube-transcript-api#cookies\n\ - Use a different IP address\n\ - Wait until the ban on your IP has been lifted") + class TranscriptsDisabled(CouldNotRetrieveTranscript): CAUSE_MESSAGE = 'Subtitles are disabled for this video' diff --git a/youtube_transcript_api/formatters.py b/youtube_transcript_api/formatters.py index 6696215..1cc6e9d 100644 --- a/youtube_transcript_api/formatters.py +++ b/youtube_transcript_api/formatters.py @@ -1,5 +1,7 @@ import json +import pprint + class Formatter(object): """Formatter should be used as an abstract base class. @@ -22,6 +24,16 @@ class Formatter(object): 'their own .format() method.') +class PrettyPrintFormatter(Formatter): + def format(self, **kwargs): + """Pretty prints a transcript. + + :return: A pretty printed string representation of the transcript dict.' + :rtype str + """ + return pprint.pformat(self._transcript, **kwargs) + + class JSONFormatter(Formatter): def format(self, **kwargs): """Converts a transcript into a JSON string. @@ -72,12 +84,12 @@ class WebVTTFormatter(Formatter): """ lines = [] for i, line in enumerate(self._transcript): - if i < len(self._transcript)-1: + if i < len(self._transcript) - 1: # Looks ahead, use next start time since duration value # would create an overlap between start times. time_text = "{} --> {}".format( self._seconds_to_timestamp(line['start']), - self._seconds_to_timestamp(self._transcript[i+1]['start']) + self._seconds_to_timestamp(self._transcript[i + 1]['start']) ) else: # Reached the end, cannot look ahead, use duration now. @@ -89,3 +101,27 @@ class WebVTTFormatter(Formatter): lines.append("{}\n{}".format(time_text, line['text'])) return "WEBVTT\n\n" + "\n\n".join(lines) + "\n" + + +class FormatterLoader(object): + TYPES = { + 'json': JSONFormatter, + 'pretty': PrettyPrintFormatter, + 'text': TextFormatter, + 'webvvt': WebVTTFormatter, + } + + class UnknownFormatterType(Exception): + def __init__(self, formatter_type): + super(FormatterLoader.UnknownFormatterType, self).__init__( + f'The format \'{formatter_type}\' is not supported. ' + f'Choose one of the following formats: {", ".join(FormatterLoader.TYPES.keys())}' + ) + + def __init__(self, formatter_type='pretty'): + if formatter_type not in FormatterLoader.TYPES.keys(): + raise FormatterLoader.UnknownFormatterType(formatter_type) + self._formatter = FormatterLoader.TYPES[formatter_type] + + def load(self, transcript): + return self._formatter(transcript) diff --git a/youtube_transcript_api/test/test_formatters.py b/youtube_transcript_api/test/test_formatters.py index 9550845..bb0b274 100644 --- a/youtube_transcript_api/test/test_formatters.py +++ b/youtube_transcript_api/test/test_formatters.py @@ -1,12 +1,15 @@ -import json -from mock import MagicMock from unittest import TestCase +import json + +import pprint + from youtube_transcript_api.formatters import ( Formatter, JSONFormatter, TextFormatter, - WebVTTFormatter + WebVTTFormatter, + PrettyPrintFormatter, FormatterLoader ) @@ -35,6 +38,7 @@ class TestFormatters(TestCase): def test_webvtt_formatter_starting(self): content = WebVTTFormatter(self.transcript).format() lines = content.split('\n') + # test starting lines self.assertEqual(lines[0], "WEBVTT") self.assertEqual(lines[1], "") @@ -42,16 +46,40 @@ class TestFormatters(TestCase): def test_webvtt_formatter_ending(self): content = WebVTTFormatter(self.transcript).format() lines = content.split('\n') + # test ending lines self.assertEqual(lines[-2], self.transcript[-1]['text']) self.assertEqual(lines[-1], "") - + + def test_pretty_print_formatter(self): + content = PrettyPrintFormatter(self.transcript).format() + + self.assertEqual(content, pprint.pformat(self.transcript)) + def test_json_formatter(self): content = JSONFormatter(self.transcript).format() + self.assertEqual(json.loads(content), self.transcript) def test_text_formatter(self): content = TextFormatter(self.transcript).format() lines = content.split('\n') + self.assertEqual(lines[0], self.transcript[0]["text"]) self.assertEqual(lines[-1], self.transcript[-1]["text"]) + + def test_formatter_loader(self): + loader = FormatterLoader('json') + formatter = loader.load(self.transcript) + + self.assertTrue(isinstance(formatter, JSONFormatter)) + + def test_formatter_loader__default_formatter(self): + loader = FormatterLoader() + formatter = loader.load(self.transcript) + + self.assertTrue(isinstance(formatter, PrettyPrintFormatter)) + + def test_formatter_loader__unknown_format(self): + with self.assertRaises(FormatterLoader.UnknownFormatterType): + FormatterLoader('png') From cf585e21759dbd9109cafebec2716a8d417f5281 Mon Sep 17 00:00:00 2001 From: Jonas Depoix Date: Mon, 15 Mar 2021 17:57:44 +0100 Subject: [PATCH 18/22] refactored formatters to use format_transcript and format_transcripts --- README.md | 1 + youtube_transcript_api/_cli.py | 17 ++-- youtube_transcript_api/formatters.py | 90 +++++++++++++------ youtube_transcript_api/test/test_cli.py | 50 ++++++----- .../test/test_formatters.py | 61 ++++++++----- 5 files changed, 138 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index 591e8b5..59f6bb0 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ The `formatters` submodule provides a few basic formatters to wrap around you tr We provided a few subclasses of formatters to use: - JSONFormatter +- PrettyPrintFormatter - TextFormatter - WebVTTFormatter (a basic implementation) diff --git a/youtube_transcript_api/_cli.py b/youtube_transcript_api/_cli.py index bf83331..870be3a 100644 --- a/youtube_transcript_api/_cli.py +++ b/youtube_transcript_api/_cli.py @@ -1,11 +1,9 @@ -import json - -import pprint - import argparse from ._api import YouTubeTranscriptApi +from .formatters import FormatterLoader + class YouTubeTranscriptCli(object): def __init__(self, args): @@ -34,7 +32,7 @@ class YouTubeTranscriptCli(object): return '\n\n'.join( [str(exception) for exception in exceptions] - + ([json.dumps(transcripts) if parsed_args.json else pprint.pformat(transcripts)] if transcripts else []) + + ([FormatterLoader().load(parsed_args.format).format_transcripts(transcripts)] if transcripts else []) ) def _fetch_transcript(self, parsed_args, proxies, cookies, video_id): @@ -98,11 +96,10 @@ class YouTubeTranscriptCli(object): help='If this flag is set transcripts which have been manually created will not be retrieved.', ) parser.add_argument( - '--json', - action='store_const', - const=True, - default=False, - help='If this flag is set the output will be JSON formatted.', + '--format', + type=str, + default='pretty', + choices=tuple(FormatterLoader.TYPES.keys()), ) parser.add_argument( '--translate', diff --git a/youtube_transcript_api/formatters.py b/youtube_transcript_api/formatters.py index 1cc6e9d..d957a41 100644 --- a/youtube_transcript_api/formatters.py +++ b/youtube_transcript_api/formatters.py @@ -9,49 +9,75 @@ class Formatter(object): Formatter classes should inherit from this class and implement their own .format() method which should return a string. A transcript is represented by a List of Dictionary items. - - :param transcript: list representing 1 or more transcripts - :type transcript: list """ - def __init__(self, transcript): - if not isinstance(transcript, list): - raise TypeError("'transcript' must be of type: List") - self._transcript = transcript - - def format(self, **kwargs): + def format_transcript(self, transcript, **kwargs): raise NotImplementedError('A subclass of Formatter must implement ' \ - 'their own .format() method.') + 'their own .format_transcript() method.') + + def format_transcripts(self, transcripts, **kwargs): + raise NotImplementedError('A subclass of Formatter must implement ' \ + 'their own .format_transcripts() method.') class PrettyPrintFormatter(Formatter): - def format(self, **kwargs): + def format_transcript(self, transcript, **kwargs): """Pretty prints a transcript. - :return: A pretty printed string representation of the transcript dict.' + :param transcript: + :return: A pretty printed string representation of the transcript.' :rtype str """ - return pprint.pformat(self._transcript, **kwargs) + return pprint.pformat(transcript, **kwargs) + + def format_transcripts(self, transcripts, **kwargs): + """Pretty prints a list of transcripts. + + :param transcripts: + :return: A pretty printed string representation of the transcripts.' + :rtype str + """ + return self.format_transcript(transcripts, **kwargs) class JSONFormatter(Formatter): - def format(self, **kwargs): + def format_transcript(self, transcript, **kwargs): """Converts a transcript into a JSON string. + :param transcript: :return: A JSON string representation of the transcript.' :rtype str """ - return json.dumps(self._transcript, **kwargs) + return json.dumps(transcript, **kwargs) + + def format_transcripts(self, transcripts, **kwargs): + """Converts a list of transcripts into a JSON string. + + :param transcripts: + :return: A JSON string representation of the transcript.' + :rtype str + """ + return self.format_transcript(transcripts, **kwargs) class TextFormatter(Formatter): - def format(self, **kwargs): + def format_transcript(self, transcript, **kwargs): """Converts a transcript into plain text with no timestamps. + :param transcript: :return: all transcript text lines separated by newline breaks.' :rtype str """ - return "\n".join(line['text'] for line in self._transcript) + return '\n'.join(line['text'] for line in transcript) + + def format_transcripts(self, transcripts, **kwargs): + """Converts a list of transcripts into plain text with no timestamps. + + :param transcripts: + :return: all transcript text lines separated by newline breaks.' + :rtype str + """ + return '\n\n\n'.join([self.format_transcript(transcript, **kwargs) for transcript in transcripts]) class WebVTTFormatter(Formatter): @@ -77,19 +103,20 @@ class WebVTTFormatter(Formatter): ms = int(round((time - int(time))*1000, 2)) return "{:02d}:{:02d}:{:02d}.{:03d}".format(hours, mins, secs, ms) - def format(self, **kwargs): + def format_transcript(self, transcript, **kwargs): """A basic implementation of WEBVTT formatting. + :param transcript: :reference: https://www.w3.org/TR/webvtt1/#introduction-caption """ lines = [] - for i, line in enumerate(self._transcript): - if i < len(self._transcript) - 1: + for i, line in enumerate(transcript): + if i < len(transcript) - 1: # Looks ahead, use next start time since duration value # would create an overlap between start times. time_text = "{} --> {}".format( self._seconds_to_timestamp(line['start']), - self._seconds_to_timestamp(self._transcript[i + 1]['start']) + self._seconds_to_timestamp(transcript[i + 1]['start']) ) else: # Reached the end, cannot look ahead, use duration now. @@ -102,6 +129,14 @@ class WebVTTFormatter(Formatter): return "WEBVTT\n\n" + "\n\n".join(lines) + "\n" + def format_transcripts(self, transcripts, **kwargs): + """A basic implementation of WEBVTT formatting for a list of transcripts. + + :param transcripts: + :reference: https://www.w3.org/TR/webvtt1/#introduction-caption + """ + return '\n\n\n'.join([self.format_transcript(transcript, **kwargs) for transcript in transcripts]) + class FormatterLoader(object): TYPES = { @@ -118,10 +153,13 @@ class FormatterLoader(object): f'Choose one of the following formats: {", ".join(FormatterLoader.TYPES.keys())}' ) - def __init__(self, formatter_type='pretty'): + def load(self, formatter_type='pretty'): + """ + Loads the Formatter for the given formatter type. + + :param formatter_type: + :return: Formatter object + """ if formatter_type not in FormatterLoader.TYPES.keys(): raise FormatterLoader.UnknownFormatterType(formatter_type) - self._formatter = FormatterLoader.TYPES[formatter_type] - - def load(self, transcript): - return self._formatter(transcript) + return FormatterLoader.TYPES[formatter_type]() diff --git a/youtube_transcript_api/test/test_cli.py b/youtube_transcript_api/test/test_cli.py index 158cd35..1cb3eff 100644 --- a/youtube_transcript_api/test/test_cli.py +++ b/youtube_transcript_api/test/test_cli.py @@ -25,50 +25,52 @@ class TestYouTubeTranscriptCli(TestCase): YouTubeTranscriptApi.list_transcripts = MagicMock(return_value=self.transcript_list_mock) def test_argument_parsing(self): - parsed_args = YouTubeTranscriptCli('v1 v2 --json --languages de en'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('v1 v2 --format json --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, '') self.assertEqual(parsed_args.https_proxy, '') - parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en --json'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en --format json'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, '') self.assertEqual(parsed_args.https_proxy, '') - parsed_args = YouTubeTranscriptCli(' --json v1 v2 --languages de en'.split())._parse_args() + parsed_args = YouTubeTranscriptCli(' --format json v1 v2 --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, '') self.assertEqual(parsed_args.https_proxy, '') parsed_args = YouTubeTranscriptCli( - 'v1 v2 --languages de en --json --http-proxy http://user:pass@domain:port --https-proxy https://user:pass@domain:port'.split() + 'v1 v2 --languages de en --format json ' + '--http-proxy http://user:pass@domain:port ' + '--https-proxy https://user:pass@domain:port'.split() )._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, 'http://user:pass@domain:port') self.assertEqual(parsed_args.https_proxy, 'https://user:pass@domain:port') parsed_args = YouTubeTranscriptCli( - 'v1 v2 --languages de en --json --http-proxy http://user:pass@domain:port'.split() + 'v1 v2 --languages de en --format json --http-proxy http://user:pass@domain:port'.split() )._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.http_proxy, 'http://user:pass@domain:port') self.assertEqual(parsed_args.https_proxy, '') parsed_args = YouTubeTranscriptCli( - 'v1 v2 --languages de en --json --https-proxy https://user:pass@domain:port'.split() + 'v1 v2 --languages de en --format json --https-proxy https://user:pass@domain:port'.split() )._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.https_proxy, 'https://user:pass@domain:port') self.assertEqual(parsed_args.http_proxy, '') @@ -76,28 +78,28 @@ class TestYouTubeTranscriptCli(TestCase): def test_argument_parsing__only_video_ids(self): parsed_args = YouTubeTranscriptCli('v1 v2'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, False) + self.assertEqual(parsed_args.format, 'pretty') self.assertEqual(parsed_args.languages, ['en']) def test_argument_parsing__fail_without_video_ids(self): with self.assertRaises(SystemExit): - YouTubeTranscriptCli('--json'.split())._parse_args() + YouTubeTranscriptCli('--format json'.split())._parse_args() def test_argument_parsing__json(self): - parsed_args = YouTubeTranscriptCli('v1 v2 --json'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('v1 v2 --format json'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['en']) - parsed_args = YouTubeTranscriptCli('--json v1 v2'.split())._parse_args() + parsed_args = YouTubeTranscriptCli('--format json v1 v2'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, True) + self.assertEqual(parsed_args.format, 'json') self.assertEqual(parsed_args.languages, ['en']) def test_argument_parsing__languages(self): parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, False) + self.assertEqual(parsed_args.format, 'pretty') self.assertEqual(parsed_args.languages, ['de', 'en']) def test_argument_parsing__proxies(self): @@ -135,13 +137,13 @@ class TestYouTubeTranscriptCli(TestCase): def test_argument_parsing__translate(self): parsed_args = YouTubeTranscriptCli('v1 v2 --languages de en --translate cz'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, False) + self.assertEqual(parsed_args.format, 'pretty') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.translate, 'cz') parsed_args = YouTubeTranscriptCli('v1 v2 --translate cz --languages de en'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['v1', 'v2']) - self.assertEqual(parsed_args.json, False) + self.assertEqual(parsed_args.format, 'pretty') self.assertEqual(parsed_args.languages, ['de', 'en']) self.assertEqual(parsed_args.translate, 'cz') @@ -188,7 +190,9 @@ class TestYouTubeTranscriptCli(TestCase): def test_run__exclude_manually_created_and_generated(self): self.assertEqual( - YouTubeTranscriptCli('v1 v2 --languages de en --exclude-manually-created --exclude-generated'.split()).run(), + YouTubeTranscriptCli( + 'v1 v2 --languages de en --exclude-manually-created --exclude-generated'.split() + ).run(), '' ) @@ -204,7 +208,7 @@ class TestYouTubeTranscriptCli(TestCase): YouTubeTranscriptApi.list_transcripts.assert_any_call('v2', proxies=None, cookies=None) def test_run__json_output(self): - output = YouTubeTranscriptCli('v1 v2 --languages de en --json'.split()).run() + output = YouTubeTranscriptCli('v1 v2 --languages de en --format json'.split()).run() # will fail if output is not valid json json.loads(output) diff --git a/youtube_transcript_api/test/test_formatters.py b/youtube_transcript_api/test/test_formatters.py index bb0b274..748ed02 100644 --- a/youtube_transcript_api/test/test_formatters.py +++ b/youtube_transcript_api/test/test_formatters.py @@ -20,23 +20,16 @@ class TestFormatters(TestCase): {'text': 'line between', 'start': 1.5, 'duration': 2.0}, {'text': 'testing the end line', 'start': 2.5, 'duration': 3.25} ] - - def test_base_formatter_valid_type(self): - with self.assertRaises(TypeError) as err: - Formatter({"test": []}) - expected_err = "'transcript' must be of type: List" - self.assertEqual(expected_err, str(err.exception)) - + self.transcripts = [self.transcript, self.transcript] + def test_base_formatter_format_call(self): - with self.assertRaises(NotImplementedError) as err: - Formatter(self.transcript).format() - - expected_err = "A subclass of Formatter must implement their own " \ - ".format() method." - self.assertEqual(expected_err, str(err.exception)) + with self.assertRaises(NotImplementedError): + Formatter().format_transcript(self.transcript) + with self.assertRaises(NotImplementedError): + Formatter().format_transcripts([self.transcript]) def test_webvtt_formatter_starting(self): - content = WebVTTFormatter(self.transcript).format() + content = WebVTTFormatter().format_transcript(self.transcript) lines = content.split('\n') # test starting lines @@ -44,42 +37,66 @@ class TestFormatters(TestCase): self.assertEqual(lines[1], "") def test_webvtt_formatter_ending(self): - content = WebVTTFormatter(self.transcript).format() + content = WebVTTFormatter().format_transcript(self.transcript) lines = content.split('\n') # test ending lines self.assertEqual(lines[-2], self.transcript[-1]['text']) self.assertEqual(lines[-1], "") + def test_webvtt_formatter_many(self): + formatter = WebVTTFormatter() + content = formatter.format_transcripts(self.transcripts) + formatted_single_transcript = formatter.format_transcript(self.transcript) + + self.assertEqual(content, formatted_single_transcript + '\n\n\n' + formatted_single_transcript) + def test_pretty_print_formatter(self): - content = PrettyPrintFormatter(self.transcript).format() + content = PrettyPrintFormatter().format_transcript(self.transcript) self.assertEqual(content, pprint.pformat(self.transcript)) + def test_pretty_print_formatter_many(self): + content = PrettyPrintFormatter().format_transcripts(self.transcripts) + + self.assertEqual(content, pprint.pformat(self.transcripts)) + def test_json_formatter(self): - content = JSONFormatter(self.transcript).format() + content = JSONFormatter().format_transcript(self.transcript) self.assertEqual(json.loads(content), self.transcript) + def test_json_formatter_many(self): + content = JSONFormatter().format_transcripts(self.transcripts) + + self.assertEqual(json.loads(content), self.transcripts) + def test_text_formatter(self): - content = TextFormatter(self.transcript).format() + content = TextFormatter().format_transcript(self.transcript) lines = content.split('\n') self.assertEqual(lines[0], self.transcript[0]["text"]) self.assertEqual(lines[-1], self.transcript[-1]["text"]) + def test_text_formatter_many(self): + formatter = TextFormatter() + content = formatter.format_transcripts(self.transcripts) + formatted_single_transcript = formatter.format_transcript(self.transcript) + + self.assertEqual(content, formatted_single_transcript + '\n\n\n' + formatted_single_transcript) + def test_formatter_loader(self): - loader = FormatterLoader('json') - formatter = loader.load(self.transcript) + loader = FormatterLoader() + formatter = loader.load('json') self.assertTrue(isinstance(formatter, JSONFormatter)) def test_formatter_loader__default_formatter(self): loader = FormatterLoader() - formatter = loader.load(self.transcript) + formatter = loader.load() self.assertTrue(isinstance(formatter, PrettyPrintFormatter)) def test_formatter_loader__unknown_format(self): with self.assertRaises(FormatterLoader.UnknownFormatterType): - FormatterLoader('png') + FormatterLoader().load('png') From a2221950941337516408f14d75b75280c73df829 Mon Sep 17 00:00:00 2001 From: Jonas Depoix Date: Mon, 15 Mar 2021 18:06:08 +0100 Subject: [PATCH 19/22] updated README --- README.md | 271 ++++-------------------------------------------------- 1 file changed, 16 insertions(+), 255 deletions(-) diff --git a/README.md b/README.md index 59f6bb0..8d06e2b 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,8 @@ print(translated_transcript.fetch()) ### By example ```python +from youtube_transcript_api import YouTubeTranscriptApi + # retrieve the available transcripts transcript_list = YouTubeTranscriptApi.list_transcripts('video_id') @@ -194,8 +196,10 @@ from youtube_transcript_api.formatters import JSONFormatter # Must be a single transcript. transcript = YouTubeTranscriptApi.get_transcript(video_id) -# .format() turns the transcript into a JSON string. -json_formatted = JSONFormatter(transcript).format() +formatter = JSONFormatter() + +# .format_transcript(transcript) turns the transcript into a JSON string. +json_formatted = formatter.format_transcript(transcript) # Now we can write it out to a file. @@ -207,21 +211,25 @@ with open('your_filename.json', 'w', encoding='utf-8') as json_file: **Passing extra keyword arguments** -Since JSONFormatter leverages `json.dumps()` you can also forward keyword arguments into `.format()` such as making your file output prettier by forwarding the `indent=2` keyword argument. +Since JSONFormatter leverages `json.dumps()` you can also forward keyword arguments into `.format_transcript(transcript)` such as making your file output prettier by forwarding the `indent=2` keyword argument. ```python -json_formatted = JSONFormatter(transcript).format(indent=2) +json_formatted = JSONFormatter().format_transcript(transcript, indent=2) ``` ### Custom Formatter Example -You can implement your own formatter class. Just inherit from the `Formatter` base class and ensure you implement the `def format(self, **kwargs):` method which should ultimately return a string when called on your formatter instance. +You can implement your own formatter class. Just inherit from the `Formatter` base class and ensure you implement the `format_transcript(self, transcript, **kwargs)` and `format_transcripts(self, transcripts, **kwargs)` methods which should ultimately return a string when called on your formatter instance. ```python class MyCustomFormatter(Formatter): - def format(self, **kwargs): + def format_transcript(self, transcript, **kwargs): # Do your custom work in here, but return a string. return 'your processed output data as a string.' + + def format_transcripts(self, transcripts, **kwargs): + # Do your custom work in here to format a list of transcripts, but return a string. + return 'your processed output data as a string.' ``` ## CLI @@ -248,7 +256,7 @@ youtube_transcript_api ... --languages de en If you would prefer to write it into a file or pipe it into another application, you can also output the results as json using the following line: ``` -youtube_transcript_api ... --languages de en --json > transcripts.json +youtube_transcript_api ... --languages de en --format json > transcripts.json ``` Translating transcripts using the CLI is also possible: @@ -310,251 +318,4 @@ youtube_transcript_api --cookies /path/to/you If this project makes you happy by reducing your development time, you can make me happy by treating me to a cup of coffee :) -[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) - -# YouTube Transcript/Subtitle API (including automatically generated subtitles and subtitle translations) - -[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) [![Build Status](https://travis-ci.com/jdepoix/youtube-transcript-api.svg)](https://travis-ci.com/jdepoix/youtube-transcript-api) [![Coverage Status](https://coveralls.io/repos/github/jdepoix/youtube-transcript-api/badge.svg?branch=master)](https://coveralls.io/github/jdepoix/youtube-transcript-api?branch=master) [![MIT license](http://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](http://opensource.org/licenses/MIT) [![image](https://img.shields.io/pypi/v/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) [![image](https://img.shields.io/pypi/pyversions/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) - -This is a python API which allows you to get the transcript/subtitles for a given YouTube video. It also works for automatically generated subtitles, supports translating subtitles and it does not require a headless browser, like other selenium based solutions do! - -## Install - -It is recommended to [install this module by using pip](https://pypi.org/project/youtube-transcript-api/): - -``` -pip install youtube_transcript_api -``` - -If you want to use it from source, you'll have to install the dependencies manually: - -``` -pip install -r requirements.txt -``` - -You can either integrate this module [into an existing application](#api), or just use it via an [CLI](#cli). - -## API - -The easiest way to get a transcript for a given video is to execute: - -```python -from youtube_transcript_api import YouTubeTranscriptApi - -YouTubeTranscriptApi.get_transcript(video_id) -``` - -This will return a list of dictionaries looking somewhat like this: - -```python -[ - { - 'text': 'Hey there', - 'start': 7.58, - 'duration': 6.13 - }, - { - 'text': 'how are you', - 'start': 14.08, - 'duration': 7.58 - }, - # ... -] -``` - -You can also add the `languages` param if you want to make sure the transcripts are retrieved in your desired language (it defaults to english). - -```python -YouTubeTranscriptApi.get_transcripts(video_ids, languages=['de', 'en']) -``` - -It's a list of language codes in a descending priority. In this example it will first try to fetch the german transcript (`'de'`) and then fetch the english transcript (`'en'`) if it fails to do so. If you want to find out which languages are available first, [have a look at `list_transcripts()`](#list-available-transcripts) - -To get transcripts for a list of video ids you can call: - -```python -YouTubeTranscriptApi.get_transcripts(video_ids, languages=['de', 'en']) -``` - -`languages` also is optional here. - -### List available transcripts - -If you want to list all transcripts which are available for a given video you can call: - -```python -transcript_list = YouTubeTranscriptApi.list_transcripts(video_id) -``` - -This will return a `TranscriptList` object which is iterable and provides methods to filter the list of transcripts for specific languages and types, like: - -```python -transcript = transcript_list.find_transcript(['de', 'en']) -``` - -By default this module always picks manually created transcripts over automatically created ones, if a transcript in the requested language is available both manually created and generated. The `TranscriptList` allows you to bypass this default behaviour by searching for specific transcript types: - -```python -# filter for manually created transcripts -transcript = transcript_list.find_manually_created_transcript(['de', 'en']) - -# or automatically generated ones -transcript = transcript_list.find_generated_transcript(['de', 'en']) -``` - -The methods `find_generated_transcript`, `find_manually_created_transcript`, `find_generated_transcript` return `Transcript` objects. They contain metadata regarding the transcript: - -```python -print( - transcript.video_id, - transcript.language, - transcript.language_code, - # whether it has been manually created or generated by YouTube - transcript.is_generated, - # whether this transcript can be translated or not - transcript.is_translatable, - # a list of languages the transcript can be translated to - transcript.translation_languages, -) -``` - -and provide the method, which allows you to fetch the actual transcript data: - -```python -transcript.fetch() -``` - -### Translate transcript - -YouTube has a feature which allows you to automatically translate subtitles. This module also makes it possible to access this feature. To do so `Transcript` objects provide a `translate()` method, which returns a new translated `Transcript` object: - -```python -transcript = transcript_list.find_transcript(['en']) -translated_transcript = transcript.translate('de') -print(translated_transcript.fetch()) -``` - -### By example -```python -# retrieve the available transcripts -transcript_list = YouTubeTranscriptApi.list_transcripts('video_id') - -# iterate over all available transcripts -for transcript in transcript_list: - - # the Transcript object provides metadata properties - print( - transcript.video_id, - transcript.language, - transcript.language_code, - # whether it has been manually created or generated by YouTube - transcript.is_generated, - # whether this transcript can be translated or not - transcript.is_translatable, - # a list of languages the transcript can be translated to - transcript.translation_languages, - ) - - # fetch the actual transcript data - print(transcript.fetch()) - - # translating the transcript will return another transcript object - print(transcript.translate('en').fetch()) - -# you can also directly filter for the language you are looking for, using the transcript list -transcript = transcript_list.find_transcript(['de', 'en']) - -# or just filter for manually created transcripts -transcript = transcript_list.find_manually_created_transcript(['de', 'en']) - -# or automatically generated ones -transcript = transcript_list.find_generated_transcript(['de', 'en']) -``` - -## CLI - -Execute the CLI script using the video ids as parameters and the results will be printed out to the command line: - -``` -youtube_transcript_api ... -``` - -The CLI also gives you the option to provide a list of preferred languages: - -``` -youtube_transcript_api ... --languages de en -``` - -You can also specify if you want to exclude automatically generated or manually created subtitles: - -``` -youtube_transcript_api ... --languages de en --exclude-generated -youtube_transcript_api ... --languages de en --exclude-manually-created -``` - -If you would prefer to write it into a file or pipe it into another application, you can also output the results as json using the following line: - -``` -youtube_transcript_api ... --languages de en --json > transcripts.json -``` - -Translating transcripts using the CLI is also possible: - -``` -youtube_transcript_api ... --languages en --translate de -``` - -If you are not sure which languages are available for a given video you can call, to list all available transcripts: - -``` -youtube_transcript_api --list-transcripts -``` - -## Proxy - -You can specify a https/http proxy, which will be used during the requests to YouTube: - -```python -from youtube_transcript_api import YouTubeTranscriptApi - -YouTubeTranscriptApi.get_transcript(video_id, proxies={"http": "http://user:pass@domain:port", "https": "https://user:pass@domain:port"}) -``` - -As the `proxies` dict is passed on to the `requests.get(...)` call, it follows the [format used by the requests library](http://docs.python-requests.org/en/master/user/advanced/#proxies). - -Using the CLI: - -``` -youtube_transcript_api --http-proxy http://user:pass@domain:port --https-proxy https://user:pass@domain:port -``` - -## Cookies - -Some videos are age restricted, so this module won't be able to access those videos without some sort of authentication. To do this, you will need to have access to the desired video in a browser. Then, you will need to download that pages cookies into a text file. You can use the Chrome extension [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg?hl=en) or the Firefox extension [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/). - -Once you have that, you can use it with the module to access age-restricted videos' captions like so. - -```python -from youtube_transcript_api import YouTubeTranscriptApi - -YouTubeTranscriptApi.get_transcript(video_id, cookies='/path/to/your/cookies.txt') - -YouTubeTranscriptApi.get_transcripts([video_id], cookies='/path/to/your/cookies.txt') -``` - -Using the CLI: - -``` -youtube_transcript_api --cookies /path/to/your/cookies.txt -``` - - -## Warning - - This code uses an undocumented part of the YouTube API, which is called by the YouTube web-client. So there is no guarantee that it won't stop working tomorrow, if they change how things work. I will however do my best to make things working again as soon as possible if that happens. So if it stops working, let me know! - -## Donation - -If this project makes you happy by reducing your development time, you can make me happy by treating me to a cup of coffee :) - -[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) +[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) \ No newline at end of file From 2f592a37a225c815c9810196970f1088be562aa5 Mon Sep 17 00:00:00 2001 From: Jonas Depoix Date: Mon, 15 Mar 2021 18:13:34 +0100 Subject: [PATCH 20/22] removed string formatting which is not supported in older python versions --- youtube_transcript_api/formatters.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/youtube_transcript_api/formatters.py b/youtube_transcript_api/formatters.py index d957a41..ed9f5ae 100644 --- a/youtube_transcript_api/formatters.py +++ b/youtube_transcript_api/formatters.py @@ -149,8 +149,11 @@ class FormatterLoader(object): class UnknownFormatterType(Exception): def __init__(self, formatter_type): super(FormatterLoader.UnknownFormatterType, self).__init__( - f'The format \'{formatter_type}\' is not supported. ' - f'Choose one of the following formats: {", ".join(FormatterLoader.TYPES.keys())}' + 'The format \'{formatter_type}\' is not supported. ' + 'Choose one of the following formats: {supported_formatter_types}'.format( + formatter_type=formatter_type, + supported_formatter_types=', '.join(FormatterLoader.TYPES.keys()), + ) ) def load(self, formatter_type='pretty'): From e69759f6d6c796531c0ddb2f85273a27aad700fc Mon Sep 17 00:00:00 2001 From: Jonas Depoix Date: Mon, 22 Mar 2021 19:04:10 +0100 Subject: [PATCH 21/22] fixed test --- youtube_transcript_api/test/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_transcript_api/test/test_cli.py b/youtube_transcript_api/test/test_cli.py index 9c5894f..d14f331 100644 --- a/youtube_transcript_api/test/test_cli.py +++ b/youtube_transcript_api/test/test_cli.py @@ -84,7 +84,7 @@ class TestYouTubeTranscriptCli(TestCase): def test_argument_parsing__video_ids_starting_with_dash(self): parsed_args = YouTubeTranscriptCli('\-v1 \-\-v2 \--v3'.split())._parse_args() self.assertEqual(parsed_args.video_ids, ['-v1', '--v2', '--v3']) - self.assertEqual(parsed_args.json, False) + self.assertEqual(parsed_args.format, 'pretty') self.assertEqual(parsed_args.languages, ['en']) def test_argument_parsing__fail_without_video_ids(self): From 943be911ec436127ae87793e68f2425946114268 Mon Sep 17 00:00:00 2001 From: Jonas Depoix Date: Mon, 22 Mar 2021 19:06:46 +0100 Subject: [PATCH 22/22] fixed travis ci url in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 75e80f6..3099cd1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # YouTube Transcript/Subtitle API (including automatically generated subtitles and subtitle translations) -[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) [![Build Status](https://travis-ci.com/jdepoix/youtube-transcript-api.svg)](https://travis-ci.com/jdepoix/youtube-transcript-api) [![Coverage Status](https://coveralls.io/repos/github/jdepoix/youtube-transcript-api/badge.svg?branch=master)](https://coveralls.io/github/jdepoix/youtube-transcript-api?branch=master) [![MIT license](http://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](http://opensource.org/licenses/MIT) [![image](https://img.shields.io/pypi/v/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) [![image](https://img.shields.io/pypi/pyversions/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) +[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BAENLEW8VUJ6G&source=url) [![Build Status](https://travis-ci.com/jdepoix/youtube-transcript-api.svg?branch=master)](https://travis-ci.com/jdepoix/youtube-transcript-api) [![Coverage Status](https://coveralls.io/repos/github/jdepoix/youtube-transcript-api/badge.svg?branch=master)](https://coveralls.io/github/jdepoix/youtube-transcript-api?branch=master) [![MIT license](http://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](http://opensource.org/licenses/MIT) [![image](https://img.shields.io/pypi/v/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) [![image](https://img.shields.io/pypi/pyversions/youtube-transcript-api.svg)](https://pypi.org/project/youtube-transcript-api/) This is a python API which allows you to get the transcript/subtitles for a given YouTube video. It also works for automatically generated subtitles, supports translating subtitles and it does not require a headless browser, like other selenium based solutions do!