transcript-fetcher/fetch-transcripts.py at main · douglaskayama/transcript-fetcher · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
import json
import os
from googleapiclient.discovery import build
from pytube import Playlist
from random import uniform
from time import sleep
from youtube_transcript_api import YouTubeTranscriptApi
from youtube_transcript_api.formatters import TextFormatter
from youtube_transcript_api._errors import TranscriptsDisabled
from youtube_transcript_api._errors import CouldNotRetrieveTranscript

# --- Configuration ---
API_SERVICE_NAME = "youtube"
API_VERSION = "v3"
YOUTUBE_API_KEY = os.environ.get('YOUTUBE_API_KEY')
ATTEMPT_LIMIT = 5

playlist_urls = [
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHcUdIGv9aaZqkt-z0fepFa8', # Algoritmos e Programação de Computadores I - COM110
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfzI1njXYuMrtKcN5aLlCzw', # Algoritmos e Programação de Computadores II - COM120
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHex0IsMeE_tzBP0WMYASaQD', # Estruturas de Dados - COM160
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfMN2osQotoGDpPhW2XAWqZ', # Fundamentos Matemáticos para Computação - COM150

    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfegQ0xwO_qBrLjugk_8Njz',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHeP4gGYftJ5zJjpxFIteSeC',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHeFlcr9wqa9TcR5TON8fiN8',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHftoO8fFQ_w8Yu94u_gYvwC',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfOpI8wLFZ5nCmTWLxrP8y1',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHddkwLSoD2DGYwDIYiAPK5n',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHft5MrDCS6wN7T04Y6aqGP7',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHd3USNLAx6HK4zyrcPM_7KI',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHdAqXqb5l62TFRJU2I6FB0t',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHcUxOCRh-Y1zMP9Gxzm6jh1',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHd09mzdcmr0ONmiYstEYqAt',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHdR2nC-q3-8dSsR0Q6qvU9e',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfC_xPOFuEWKeSJiN3fMgqc',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHdyqnml0ZTlqzhBRQTyx_bx',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHcfiRY7l3sT7nnpAWQ3UAYy',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHe7nfTSOc6YKTy2uOuylI5_',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfAaUEBFk5vSRPajl8VqG9I',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHccZ7PEneZbrR4hgMC8h3oL', # Breaking on 17?
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfOCHNwfOYzClRTCUBQTa0t',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHcnZoyl__lrxCYdpM0DPOLa',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHdOy3BxqufYBPfiKe_vkOZq',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfg1gUQ9f_49JRTeOjsJ4d8',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHc7EZRFbYLiz5L_XHOqyuoB',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHdFbi5c8RXQZe_tORTLHp7z',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHd_ixwm7eBUc3sLThO0WzkF',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHeY9PqSG5dRy9Pf8eo7PNet',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHdhJSCnNZbucqY-gNJfg6zg',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfu9uc4UsHATXBjOcV0nq9c',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHdJMPnZ0YSdllBJisdGAS6W',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHdHQb0mFyRsMucW3yGPlIfI',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHd98frA3q6UHsjZPhGS-Ier',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHf5CzlhZkRewco1TbzPqAbF',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHd74tFP2EXySW5oU4XCaEX4',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHcAIG4Pcy_acp97LsGg6TIU',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHejSF4-QdMYe1qU6DGBc9Pi',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHe1pre_5IZIPnfD-n_VOxGb',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHcRsvr_wMOuIlOh_Xdsl0yL',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHejp35SWvPNdjS8rQPvVD4j',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHcrLVvW55zrwRycEo0VKEY9',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHdNMHdD4vKxA8H97hdhP8VU',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHfqJ7-FrRBR83rtvQh7CI38',
    'https://www.youtube.com/playlist?list=PLxI8Can9yAHc6SJQ_R2Sspivu5sEuSBPq',
]

youtube = build(API_SERVICE_NAME, API_VERSION, developerKey=YOUTUBE_API_KEY)
youtube_transcript_api = YouTubeTranscriptApi()

for current_playlist in playlist_urls:
    playlist = Playlist(current_playlist)
    print(f'Fetching transcript from: {playlist.title} ID: {playlist.playlist_id}')
    print(f'Length of playlist: {len(playlist.videos)}')
    os.makedirs(f'save/{playlist.playlist_id}', exist_ok=True)

    error_counter = 0
    i = 0
    while i < len(playlist.videos):
        video = playlist.videos[i]

        video_id = video.video_id
        print(f'Index: {i} Video ID: {video_id}')
        try:
            response = youtube.videos().list(id=video_id, part='snippet').execute()
            snippet = response['items'][0]['snippet']
            title = snippet['title']
            description = snippet['description']
            print(f'Video: {title} ID: {video_id}')
            print(f'Description: {description}')

            print('--')
            playlist_data = {'playlist_id': playlist.playlist_id, 'playlist_title': playlist.title}
            video_data = {'video_id': video_id, 'title': title, 'description': description, 'in_playlist': playlist_data}
            with open(f'save/{playlist.playlist_id}/{i+1:02}-{video_id}.json', 'w', encoding='utf-8') as f:
                json.dump(video_data, f, ensure_ascii=False, indent=4)

            fetched_transcript = youtube_transcript_api.fetch(video_id, languages=['pt'])
            if fetched_transcript:
                print('Fetched transcript successfully.')
                text_formatter = TextFormatter()
                text_transcript = text_formatter.format_transcript(fetched_transcript)
                with open(f'save/{playlist.playlist_id}/{i+1:02}-{video_id}-transcript.txt', 'w', encoding='utf-8') as f:
                    f.write(text_transcript)
            else:
                print('No transcript found.')
            print('--#--')
            sleep_time = uniform(1, 3)
            print(f'Sleeping for {sleep_time:.2f} seconds to avoid API rate limits.')
            sleep(sleep_time)
            i += 1
        except TranscriptsDisabled:
            print(f'Transcript is disabled for video {i} ({video_id}).')
            i += 1
        except CouldNotRetrieveTranscript:
            print(f'Could not retrieve transcript for video {i} ({video_id}).')
            i += 1
        except Exception as e:
            if error_counter < ATTEMPT_LIMIT:
                error_counter += 1
                print(f'Error fetching video {i} ({video_id}): {e}')
                sleep_time = 5 * uniform(1, 3)
                if 'quota' in str(e).lower():
                    print('Quota exceeded. Sleeping for 60 seconds before retrying.')
                    sleep_time = 60
                else:
                    print('An error occurred. Retrying after a short delay.')
                print(f'Sleeping for {sleep_time} seconds before retrying.')
                sleep(sleep_time)
                continue
            else:
                print(f'Error fetching video {i} ({video_id}): {e}')
                print(f'Skipping this video after {ATTEMPT_LIMIT} attempts.')
                i += 1
                error_counter = 0
                continue
        finally:
            print(f'Finished processing video {i} ({video_id}).\n')
            error_counter = 0
            print(f'Current index: {i}, Total videos processed: {len(playlist.videos)}')