Ritual/main.py at master · sky-cake/Ritual · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
import os
import traceback

import configs
from archive import Archive
from catalog import Catalog
from db.ritual import RitualDb, create_ritual_db
from scanner.scanner import ScannerDb
from fetcher import Fetcher
from filter import Filter
from loop import Loop
from media_fp import AsagiMediaFP, SutraMediaFP, MediaFP
from posts import Posts
from state import State
from utils import (
    fetch_and_save_boards_json,
    load_boards_with_archive,
    make_path,
    read_json,
    sleep,
    assert_thumbnail_deps
)


class Init:
    def __init__(self):
        if configs.make_thumbnails:
            assert_thumbnail_deps(configs.logger)

        boards_json_path = make_path('cache', 'boards.json')
        if os.path.isfile(boards_json_path):
            boards_json = read_json(boards_json_path)
            configs.logger.info(f'Loaded boards.json from {boards_json_path}')
        else:
            boards_json = fetch_and_save_boards_json(boards_json_path, configs.url_boards, configs.logger)

        configs.boards_with_archive = load_boards_with_archive(boards_json)

        configs.logger.info(f'{len(configs.boards_with_archive)} boards have archive support')


def process_board(board: str, db: RitualDb, fetcher: Fetcher, loop: Loop, state: State, media_fp: MediaFP):
    loop.set_start_time()

    catalog = Catalog(fetcher, board)
    if not catalog.fetch_catalog():
        return

    state.update_thread_meta(board, catalog.tid_2_page, catalog.tid_2_thread)

    # results in a max of one archive.json endpoint fetch per loop
    archive = Archive(fetcher, board)

    filter = Filter(fetcher, db, board, state)
    filter.filter_catalog(catalog)

    posts = Posts(db, fetcher, board, filter.tid_2_thread, state, catalog)
    posts.fetch_posts(archive)

    if configs.boards[board].get('thread_text') != False:
        posts.save_posts()

    filter.set_tid_2_posts(posts.tid_2_posts)
    filter.get_pids_for_download()

    media_fp.download_media_for_ids(board, posts.pid_2_post, filter.full_pids, filter.thumb_pids)
    media_fp.flush(board)

    loop.set_board_duration_minutes(board)


def save_on_error(state: State, ritual_db: RitualDb, scanner_db: ScannerDb | None, media_fp: MediaFP, board: str):
    configs.logger.info('Saving State...')
    state.save()
    configs.logger.info('  Done')

    configs.logger.info('Shutting down MediaFP...')
    media_fp.shutdown(board)
    configs.logger.info('  Done')

    configs.logger.info('Shutting down RitualDb...')
    ritual_db.save_and_close()
    configs.logger.info('  Done')

    if scanner_db and scanner_db.conn:
        configs.logger.info('Shutting down ScannerDb...')
        scanner_db.conn.commit()
        scanner_db.conn.close()
        configs.logger.info('  Done')


def get_media_fp(fetcher: Fetcher, ritual_db: RitualDb, scanner_db: ScannerDb | None) -> MediaFP:
    if configs.media_fp == 'sutra':
        return SutraMediaFP(fetcher, configs.media_save_path, ritual_db, scanner_db)

    if configs.media_fp == 'asagi':
        return AsagiMediaFP(fetcher, configs.media_save_path, ritual_db, scanner_db)

    raise ValueError(configs.media_fp)


def main():
    Init()
    ritual_db = create_ritual_db()
    loop = Loop()
    state = State(loop)
    fetcher = Fetcher(state)

    scanner_db = ScannerDb(configs.scanner_db_path) if configs.scanner_db_enabled else None

    media_fp = get_media_fp(fetcher, ritual_db, scanner_db)

    critical_error_count = 0
    board = ''
    while True:
        try:
            for board in configs.boards:
                process_board(board, ritual_db, fetcher, loop, state, media_fp)

            fetcher.sleep()
            state.save()

            loop.increment_loop()
            loop.log_board_durations()
            loop.sleep()

        except KeyboardInterrupt:
            configs.logger.info('Received interrupt signal')
            save_on_error(state, ritual_db, scanner_db, media_fp, board)
            break

        except Exception as e:
            configs.logger.error(f'Critical error in main loop: {e}')
            configs.logger.error(traceback.format_exc())
            save_on_error(state, ritual_db, scanner_db, media_fp, board)
            critical_error_count += 1
            n_critical_errors = 5
            if critical_error_count >= n_critical_errors:
                configs.logger.error(f'Critical error count reached {n_critical_errors}, exiting...')
                break

            sleep_for = critical_error_count * 60
            configs.logger.info(f'Sleeping for {sleep_for}s, maybe the issue will resolve itself by then...')
            sleep(sleep_for)

    configs.logger.info('Exited while loop, ending program.')


if __name__=='__main__':
    main()