11import logging
22import time
33import unittest
4- from datetime import timedelta , date , datetime
5-
6- from Ccdb import Ccdb , ObjectVersion
7- from rules import last_only
8- import os
9- import sys
10- import importlib
11-
12- def import_path (path ): # needed because o2-qc-repo-cleaner has no suffix
13- module_name = os .path .basename (path ).replace ('-' , '_' )
14- spec = importlib .util .spec_from_loader (
15- module_name ,
16- importlib .machinery .SourceFileLoader (module_name , path )
17- )
18- module = importlib .util .module_from_spec (spec )
19- spec .loader .exec_module (module )
20- sys .modules [module_name ] = module
21- return module
22-
23- one_per_hour = import_path ("../qcrepocleaner/rules/1_per_hour.py" )
4+ from importlib import import_module
5+ from qcrepocleaner .Ccdb import Ccdb
6+ from tests import test_utils
7+ from tests .test_utils import CCDB_TEST_URL
8+
9+ one_per_hour = import_module (".1_per_hour" , "qcrepocleaner.rules" ) # file names should not start with a number...
2410
2511class Test1PerHour (unittest .TestCase ):
2612 """
@@ -35,86 +21,64 @@ class Test1PerHour(unittest.TestCase):
3521 one_minute = 60000
3622
3723 def setUp (self ):
38- self .ccdb = Ccdb ('http:// ccdb-test.cern.ch:8080' )
24+ self .ccdb = Ccdb (CCDB_TEST_URL ) # ccdb-test but please use IP to avoid DNS alerts
3925 self .path = "qc/TST/MO/repo/test"
4026 self .run = 124321
4127 self .extra = {}
4228
4329
4430 def test_1_per_hour (self ):
4531 """
46- 60 versions, 2 minutes apart
32+ 120 versions
4733 grace period of 15 minutes
48- First version is preserved (always). 7 are preserved during the grace period at the end.
49- One more is preserved after 1 hour. --> 9 preserved
34+ First version is preserved (always). 14 are preserved during the grace period at the end.
35+ One more is preserved after 1 hour. --> 16 preserved
5036 """
5137 logging .basicConfig (level = logging .INFO , format = '%(asctime)s - %(levelname)s - %(message)s' ,
5238 datefmt = '%d-%b-%y %H:%M:%S' )
5339 logging .getLogger ().setLevel (int (10 ))
5440
5541 # Prepare data
5642 test_path = self .path + "/test_1_per_hour"
57- self .prepare_data (test_path , 60 , 2 )
43+ test_utils .clean_data (self .ccdb , test_path )
44+ test_utils .prepare_data (self .ccdb , test_path , [120 ], [0 ], 123 )
5845
5946 stats = one_per_hour .process (self .ccdb , test_path , 15 , 1 , self .in_ten_years , self .extra )
60- self .assertEqual (stats ["deleted" ], 51 )
61- self .assertEqual (stats ["preserved" ], 9 )
47+ logging .info (stats )
48+ self .assertEqual (stats ["deleted" ], 104 )
49+ self .assertEqual (stats ["preserved" ], 16 )
6250
6351 objects_versions = self .ccdb .getVersionsList (test_path )
64- self .assertEqual (len (objects_versions ), 9 )
52+ self .assertEqual (len (objects_versions ), 16 )
6553
6654
6755 def test_1_per_hour_period (self ):
6856 """
69- 60 versions, 2 minutes apart
57+ 120 versions
7058 no grace period
7159 period of acceptance: 1 hour in the middle
72- We have therefore 30 versions in the acceptance period.
60+ We have therefore 60 versions in the acceptance period.
7361 Only 1 of them, the one 1 hour after the first version in the set, will be preserved, the others are deleted.
74- Thus we have 29 deletion. Everything outside the acceptance period is kept.
62+ Thus we have 59 deletion. Everything outside the acceptance period is kept.
7563 """
7664 logging .basicConfig (level = logging .INFO , format = '%(asctime)s - %(levelname)s - %(message)s' ,
7765 datefmt = '%d-%b-%y %H:%M:%S' )
7866 logging .getLogger ().setLevel (int (10 ))
7967
8068 # Prepare data
8169 test_path = self .path + "/test_1_per_hour_period"
82- self .prepare_data (test_path , 60 , 2 )
70+ test_utils .clean_data (self .ccdb , test_path )
71+ test_utils .prepare_data (self .ccdb , test_path , [120 ], [0 ], 123 )
8372 current_timestamp = int (time .time () * 1000 )
84- logging .debug (f"{ current_timestamp } { datetime .today ()} )
85-
86- objects_versions = self .ccdb .getVersionsList (test_path )
87- created = len (objects_versions )
8873
8974 stats = one_per_hour .process (self .ccdb , test_path , 15 , current_timestamp - 90 * 60 * 1000 ,
9075 current_timestamp - 30 * 60 * 1000 , self .extra )
91- self .assertEqual (stats ["deleted" ], 29 )
92- self .assertEqual (stats ["preserved" ], 31 )
76+ logging .info (stats )
77+ self .assertEqual (stats ["deleted" ], 59 )
78+ self .assertEqual (stats ["preserved" ], 61 )
9379
9480 objects_versions = self .ccdb .getVersionsList (test_path )
95- self .assertEqual (len (objects_versions ), 31 )
96-
97-
98- def prepare_data (self , path , number_versions , minutes_between ):
99- """
100- Prepare a data set starting `since_minutes` in the past.
101- 1 version per minute
102- """
103-
104- current_timestamp = int (time .time () * 1000 )
105- data = {'part' : 'part' }
106- run = 1234
107- counter = 0
108-
109- for x in range (number_versions + 1 ):
110- counter = counter + 1
111- from_ts = current_timestamp - minutes_between * x * 60 * 1000
112- to_ts = current_timestamp
113- metadata = {'RunNumber' : str (run )}
114- version_info = ObjectVersion (path = path , validFrom = from_ts , validTo = to_ts , metadata = metadata )
115- self .ccdb .putVersion (version = version_info , data = data )
116-
117- logging .debug (f"counter : { counter } )
81+ self .assertEqual (len (objects_versions ), 61 )
11882
11983
12084if __name__ == '__main__' :
0 commit comments