BBC-Dari-Python-Crawler-for-Making-Dataset/config.yaml at main · MiladRahimi93/BBC-Dari-Python-Crawler-for-Making-Dataset · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
sources:
  bbc:
    base_url: https://www.bbc.com/persian
    categories:
    - https://www.bbc.com/persian/topics/c06gq913yyrt
    - https://www.bbc.com/persian/topics/c06gq918x2vt
    - https://www.bbc.com/persian/topics/c06gq9zkpzwt
    - https://www.bbc.com/persian/topics/c1d8ye58xl8t
    - https://www.bbc.com/persian/topics/c1gdq94v96rt
    - https://www.bbc.com/persian/topics/c1gdq94ygkpt
    - https://www.bbc.com/persian/topics/c1gdq968vz9t
    - https://www.bbc.com/persian/topics/c1gdq968z44t
    - https://www.bbc.com/persian/topics/c1gdq9w51wnt
    - https://www.bbc.com/persian/topics/c2dwq91r0jnt
    - https://www.bbc.com/persian/topics/c2dwq91rng2t
    - https://www.bbc.com/persian/topics/c2dwq96ry15t
    - https://www.bbc.com/persian/topics/c2dwq96rzp6t
    - https://www.bbc.com/persian/topics/c2jnd4q66jnt
    - https://www.bbc.com/persian/topics/c2jnd4qvkyvt
    - https://www.bbc.com/persian/topics/c340qy612nvt
    - https://www.bbc.com/persian/topics/c340qy638jrt
    - https://www.bbc.com/persian/topics/c340qy6n04pt
    - https://www.bbc.com/persian/topics/c340qy6n337t
    - https://www.bbc.com/persian/topics/c340qyz2vgvt
    - https://www.bbc.com/persian/topics/c340qyzny7vt
    - https://www.bbc.com/persian/topics/c4gpl07vl4et
    - https://www.bbc.com/persian/topics/c51vkr309zpt
    - https://www.bbc.com/persian/topics/c5qvp89k1wkt
    - https://www.bbc.com/persian/topics/c5qvp89yg7qt
    - https://www.bbc.com/persian/topics/c5qvp8gjv45t
    - https://www.bbc.com/persian/topics/c5qvp8w59n1t
    - https://www.bbc.com/persian/topics/c5qvp8wr36dt
    - https://www.bbc.com/persian/topics/c6vzy34ny76t
    - https://www.bbc.com/persian/topics/c6vzy3qgk2kt
    - https://www.bbc.com/persian/topics/c6z7mnr559gt
    - https://www.bbc.com/persian/topics/c95y3rzg41zt
    - https://www.bbc.com/persian/topics/c9wpm0epm45t
    - https://www.bbc.com/persian/topics/c9yemn6ee5et
    - https://www.bbc.com/persian/topics/cdg4e7pwg6kt
    - https://www.bbc.com/persian/topics/cdr5610d0ngt
    - https://www.bbc.com/persian/topics/cdr561xgwnxt
    - https://www.bbc.com/persian/topics/cdr561y98d0t
    - https://www.bbc.com/persian/topics/cg7261g501pt
    - https://www.bbc.com/persian/topics/cg7261gn4wxt
    - https://www.bbc.com/persian/topics/cg7261gz350t
    - https://www.bbc.com/persian/topics/cgrlgzz87nkt
    - https://www.bbc.com/persian/topics/cjgn70v6n12t
    - https://www.bbc.com/persian/topics/cjgn70v8pzpt
    - https://www.bbc.com/persian/topics/ckd08q69x8yt
    - https://www.bbc.com/persian/topics/ckdxnwr4r1yt
    - https://www.bbc.com/persian/topics/ckdxnwv3w4gt
    - https://www.bbc.com/persian/topics/ckdxnwvnpdjt
    - https://www.bbc.com/persian/topics/ckdxnwvwwjnt
    - https://www.bbc.com/persian/topics/cl8l9mvlllqt
    - https://www.bbc.com/persian/topics/clm0z4jgj3xt
    - https://www.bbc.com/persian/topics/cly21g435gpt
    - https://www.bbc.com/persian/topics/cnm27r5xrlnt
    - https://www.bbc.com/persian/topics/cnq68798yw0t
    - https://www.bbc.com/persian/topics/cnq68798z9nt
    - https://www.bbc.com/persian/topics/cnq6879k7yjt
    - https://www.bbc.com/persian/topics/cnq6879zdx9t
    - https://www.bbc.com/persian/topics/cnq687vp35xt
    - https://www.bbc.com/persian/topics/cp5e8kz2jz4t
    - https://www.bbc.com/persian/topics/cpqer7ewwdlt
    - https://www.bbc.com/persian/topics/cpzd49npn6gt
    - https://www.bbc.com/persian/topics/cpzd49wj36nt
    - https://www.bbc.com/persian/topics/cqywjvnvpg4t
    - https://www.bbc.com/persian/topics/cqywjvzg00zt
    - https://www.bbc.com/persian/topics/cr50y7rdkn0t
    - https://www.bbc.com/persian/topics/cr50y7rg1vyt
    - https://www.bbc.com/persian/topics/cr50y7rjq5kt
    - https://www.bbc.com/persian/topics/cr50y7rz64pt
    - https://www.bbc.com/persian/topics/cr50y7wg60vt
    - https://www.bbc.com/persian/topics/crd9pewkvlqt
    - https://www.bbc.com/persian/topics/cv2g1k78005t
    - https://www.bbc.com/persian/topics/cv2r628ddpyt
    - https://www.bbc.com/persian/topics/cvjp23gwxnxt
    - https://www.bbc.com/persian/topics/cvjp23vd49gt
    - https://www.bbc.com/persian/topics/cw9q48x6558t
    - https://www.bbc.com/persian/topics/cw9qgeqd1zqt
    - https://www.bbc.com/persian/topics/cwr9j2d7kw7t
    - https://www.bbc.com/persian/topics/cwr9j2dn3nwt
    - https://www.bbc.com/persian/topics/cwr9j2dyvppt
    - https://www.bbc.com/persian/topics/cwy6874zgpqt
    - https://www.bbc.com/persian/topics/cwygkxqek59t
    - https://www.bbc.com/persian/topics/cx3zg3el7l7t
    - https://www.bbc.com/persian/topics/cxnykjr8480t
    - https://www.bbc.com/persian/topics/cxnykjzqr1yt
    - https://www.bbc.com/persian/topics/cxr3ex12k6et
    - https://www.bbc.com/persian/topics/cyx5kr1kq9kt
    - https://www.bbc.com/persian/topics/cz7k02839xet
    -
crawler:
  delay_seconds: 2
  retries: 3
  user_agents:
  - Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)
    Chrome/124.0.0.0 Safari/537.36
  - Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5_2) AppleWebKit/605.1.15 (KHTML, like
    Gecko) Version/16.6 Safari/605.1.15
output:
  directory: data
  base_filename: dari_dataset
language:
  target_langs:
  - fa
  min_chars: 50