Skip to content

Commit cdaad25

Browse files
resolve sanity check last id file acces problems (#6825)
Co-authored-by: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>
1 parent bf5cacb commit cdaad25

File tree

6 files changed

+136
-45
lines changed

6 files changed

+136
-45
lines changed

debian/cloudstack-usage.postinst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,13 @@ case "$1" in
4141
ln -s /etc/cloudstack/management/key /etc/cloudstack/usage/key
4242
fi
4343

44+
# creating the last sanity checked id pointer
45+
mkdir -p /usr/local/libexec
46+
if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then
47+
echo 1 > /usr/local/libexec/sanity-check-last-id
48+
fi
49+
chown cloud:cloud /usr/local/libexec/sanity-check-last-id
50+
4451
# Print help message
4552
if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then
4653
acs_version=$(dpkg -l |grep cloudstack-usage |head -n1 |awk '{print $3}')

packaging/centos7/cloud.spec

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -552,6 +552,12 @@ if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then
552552
ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key
553553
fi
554554

555+
mkdir -p /usr/local/libexec
556+
if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then
557+
echo 1 > /usr/local/libexec/sanity-check-last-id
558+
fi
559+
chown cloud:cloud /usr/local/libexec/sanity-check-last-id
560+
555561
%posttrans usage
556562
# Print help message
557563
if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then
@@ -689,6 +695,9 @@ pip3 install --upgrade urllib3
689695
%attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal
690696

691697
%changelog
698+
* Fri Oct 14 2022 Daan Hoogland <daan.hoogland@gmail.com> 4.18.0
699+
- initialising sanity check pointer file
700+
692701
* Thu Apr 30 2015 Rohit Yadav <bhaisaab@apache.org> 4.6.0
693702
- Remove awsapi package
694703

packaging/centos8/cloud.spec

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -543,6 +543,12 @@ if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then
543543
ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key
544544
fi
545545

546+
mkdir -p /usr/local/libexec
547+
if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then
548+
echo 1 > /usr/local/libexec/sanity-check-last-id
549+
fi
550+
chown cloud:cloud /usr/local/libexec/sanity-check-last-id
551+
546552
%posttrans usage
547553
# Print help message
548554
if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then
@@ -677,6 +683,9 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
677683
%attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal
678684

679685
%changelog
686+
* Fri Oct 14 2022 Daan Hoogland <daan.hoogland@gmail.com> 4.18.0
687+
- initialising sanity check pointer file
688+
680689
* Thu Apr 30 2015 Rohit Yadav <bhaisaab@apache.org> 4.6.0
681690
- Remove awsapi package
682691

packaging/suse15/cloud.spec

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,12 @@ if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then
537537
ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key
538538
fi
539539

540+
mkdir -p /usr/local/libexec
541+
if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then
542+
echo 1 > /usr/local/libexec/sanity-check-last-id
543+
fi
544+
chown cloud:cloud /usr/local/libexec/sanity-check-last-id
545+
540546
%posttrans usage
541547
# Print help message
542548
if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then
@@ -671,6 +677,9 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
671677
%attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal
672678

673679
%changelog
680+
* Fri Oct 14 2022 Daan Hoogland <daan.hoogland@gmail.com> 4.18.0
681+
- initialising sanity check pointer file
682+
674683
* Tue Jun 29 2021 David Jumani <dj.davidjumani1994@gmail.com> 4.16.0
675684
- Adding SUSE 15 support
676685

usage/src/main/java/com/cloud/usage/UsageManagerImpl.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2156,6 +2156,7 @@ protected void deleteOneOffJobs(String hostname, int pid) {
21562156
private class SanityCheck extends ManagedContextRunnable {
21572157
@Override
21582158
protected void runInContext() {
2159+
s_logger.info("running sanity check");
21592160
UsageSanityChecker usc = new UsageSanityChecker();
21602161
try {
21612162
String errors = usc.runSanityCheck();

usage/src/main/java/com/cloud/usage/UsageSanityChecker.java

Lines changed: 101 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public class UsageSanityChecker {
5151

5252
protected void reset() {
5353
errors = new StringBuilder();
54-
checkCases = new ArrayList<CheckCase>();
54+
checkCases = new ArrayList<>();
5555
}
5656

5757
protected boolean checkItemCountByPstmt() throws SQLException {
@@ -69,48 +69,51 @@ protected boolean checkItemCountByPstmt(CheckCase checkCase) throws SQLException
6969
/*
7070
* Check for item usage records which are created after it is removed
7171
*/
72-
try (PreparedStatement pstmt = conn.prepareStatement(checkCase.sqlTemplate)) {
73-
if(checkCase.checkId) {
74-
pstmt.setInt(1, lastId);
75-
pstmt.setInt(2, maxId);
76-
}
77-
try(ResultSet rs = pstmt.executeQuery();) {
78-
if (rs.next() && (rs.getInt(1) > 0)) {
79-
errors.append(String.format("Error: Found %s %s\n", rs.getInt(1), checkCase.itemName));
80-
checkOk = false;
72+
try (PreparedStatement pstmt = conn.prepareStatement(checkCase.getSqlTemplate())) {
73+
if (checkCase.isCheckId()) {
74+
if (lastId > 0) {
75+
pstmt.setInt(1, lastId);
76+
}
77+
if (maxId > lastId) {
78+
pstmt.setInt(2, maxId);
8179
}
82-
}catch (Exception e)
83-
{
84-
s_logger.error("checkItemCountByPstmt:Exception:"+e.getMessage());
85-
throw new CloudRuntimeException("checkItemCountByPstmt:Exception:"+e.getMessage(),e);
8680
}
81+
checkOk = isCheckOkForPstmt(checkCase, checkOk, pstmt);
8782
}
8883
catch (Exception e)
8984
{
90-
s_logger.error("checkItemCountByPstmt:Exception:"+e.getMessage());
91-
throw new CloudRuntimeException("checkItemCountByPstmt:Exception:"+e.getMessage(),e);
85+
throwPreparedStatementExcecutionException("preparing statement", checkCase.getSqlTemplate(), e);
86+
}
87+
return checkOk;
88+
}
89+
90+
private boolean isCheckOkForPstmt(CheckCase checkCase, boolean checkOk, PreparedStatement pstmt) {
91+
try (ResultSet rs = pstmt.executeQuery();) {
92+
if (rs.next() && (rs.getInt(1) > 0)) {
93+
errors.append(String.format("Error: Found %s %s%n", rs.getInt(1), checkCase.getItemName()));
94+
checkOk = false;
95+
}
96+
} catch (Exception e)
97+
{
98+
throwPreparedStatementExcecutionException("check is failing", pstmt.toString(), e);
9299
}
93100
return checkOk;
94101
}
95102

103+
private static void throwPreparedStatementExcecutionException(String msgPrefix, String stmt, Exception e) {
104+
String msg = String.format("%s for prepared statement \"%s\" reason: %s", msgPrefix, stmt, e.getMessage());
105+
s_logger.error(msg);
106+
throw new CloudRuntimeException(msg, e);
107+
}
108+
96109
protected void checkMaxUsage() throws SQLException {
97110
int aggregationRange = DEFAULT_AGGREGATION_RANGE;
98-
try (PreparedStatement pstmt = conn.prepareStatement(
99-
"SELECT value FROM `cloud`.`configuration` where name = 'usage.stats.job.aggregation.range'");)
111+
String sql = "SELECT value FROM `cloud`.`configuration` WHERE name = 'usage.stats.job.aggregation.range'";
112+
try (PreparedStatement pstmt = conn.prepareStatement(sql);)
100113
{
101-
try(ResultSet rs = pstmt.executeQuery();) {
102-
if (rs.next()) {
103-
aggregationRange = rs.getInt(1);
104-
} else {
105-
s_logger.debug("Failed to retrieve aggregation range. Using default : " + aggregationRange);
106-
}
107-
}catch (SQLException e) {
108-
s_logger.error("checkMaxUsage:Exception:"+e.getMessage());
109-
throw new CloudRuntimeException("checkMaxUsage:Exception:"+e.getMessage());
110-
}
114+
aggregationRange = getAggregationRange(aggregationRange, pstmt);
111115
} catch (SQLException e) {
112-
s_logger.error("checkMaxUsage:Exception:"+e.getMessage());
113-
throw new CloudRuntimeException("checkMaxUsage:Exception:"+e.getMessage());
116+
throwPreparedStatementExcecutionException("preparing atatement", sql, e);
114117
}
115118
int aggregationHours = aggregationRange / 60;
116119

@@ -120,6 +123,21 @@ protected void checkMaxUsage() throws SQLException {
120123
lastCheckId);
121124
}
122125

126+
private static int getAggregationRange(int aggregationRange, PreparedStatement pstmt) {
127+
try (ResultSet rs = pstmt.executeQuery();) {
128+
if (rs.next()) {
129+
aggregationRange = rs.getInt(1);
130+
} else {
131+
if (s_logger.isDebugEnabled()) {
132+
s_logger.debug("Failed to retrieve aggregation range. Using default : " + aggregationRange);
133+
}
134+
}
135+
} catch (SQLException e) {
136+
throwPreparedStatementExcecutionException("retrieval aggregate value is failing", pstmt.toString(), e);
137+
}
138+
return aggregationRange;
139+
}
140+
123141
protected void checkVmUsage() {
124142
addCheckCase("select count(*) from cloud_usage.cloud_usage cu inner join cloud.vm_instance vm "
125143
+ "where vm.type = 'User' and cu.usage_type in (1 , 2) "
@@ -170,14 +188,21 @@ protected void checkSnapshotUsage() {
170188
}
171189

172190
protected void readLastCheckId(){
191+
if (s_logger.isDebugEnabled()) {
192+
s_logger.debug("reading last checked id for sanity check");
193+
}
173194
try(BufferedReader reader = new BufferedReader(new FileReader(lastCheckFile));) {
174195
String lastIdText = null;
175196
lastId = -1;
176-
if ((reader != null) && (lastIdText = reader.readLine()) != null) {
197+
if ((lastIdText = reader.readLine()) != null) {
177198
lastId = Integer.parseInt(lastIdText);
178199
}
179200
} catch (Exception e) {
180-
s_logger.error("readLastCheckId:Exception:"+e.getMessage(),e);
201+
String msg = String.format("error reading the LastCheckId reason: %s", e.getMessage());
202+
s_logger.error(msg);
203+
s_logger.debug(msg, e);
204+
} finally {
205+
s_logger.info(String.format("using %d as last checked id to start from in sanity check", lastId));
181206
}
182207
}
183208

@@ -188,20 +213,23 @@ protected void readMaxId() throws SQLException {
188213
maxId = -1;
189214
if (rs.next() && (rs.getInt(1) > 0)) {
190215
maxId = rs.getInt(1);
191-
lastCheckId += " and cu.id <= ?";
216+
if (maxId > lastId) {
217+
lastCheckId += " and cu.id <= ?";
218+
}
192219
}
193220
}catch (Exception e) {
194221
s_logger.error("readMaxId:"+e.getMessage(),e);
195222
}
196223
}
197224

198225
protected void updateNewMaxId() {
226+
s_logger.info(String.format("writing %d as the new last id checked", maxId));
199227
try (FileWriter fstream = new FileWriter(lastCheckFile);
200-
BufferedWriter out = new BufferedWriter(fstream);
228+
BufferedWriter out = new BufferedWriter(fstream);
201229
){
202230
out.write("" + maxId);
203231
} catch (IOException e) {
204-
s_logger.error("updateNewMaxId:Exception:"+e.getMessage());
232+
s_logger.error(String.format("Exception writing the last checked id: %d reason: %s", maxId, e.getMessage()));
205233
// Error while writing last check id
206234
}
207235
}
@@ -238,13 +266,17 @@ protected Connection getConnection() {
238266
return TransactionLegacy.getStandaloneConnection();
239267
}
240268

241-
public static void main(String args[]) {
269+
/**
270+
* usage something like: /usr/bin/java -Xmx2G -cp /usr/share/cloudstack-usage/*:/usr/share/cloudstack-usage/lib/*:/usr/share/cloudstack-mysql-ha/lib/*:/etc/cloudstack/usage:/usr/share/java/mysql-connector-java.jar:/usr/share/cloudstack-common com.cloud.usage.UsageSanityChecker
271+
* @param args none
272+
*/
273+
public static void main(String[] args) {
242274
UsageSanityChecker usc = new UsageSanityChecker();
243275
String sanityErrors;
244276
try {
245277
sanityErrors = usc.runSanityCheck();
246278
if (sanityErrors.length() > 0) {
247-
s_logger.error(sanityErrors.toString());
279+
s_logger.error(sanityErrors);
248280
}
249281
} catch (SQLException e) {
250282
e.printStackTrace();
@@ -266,19 +298,43 @@ protected void addCheckCase(String sqlTemplate, String itemName) {
266298
* encapsulating what change for each specific case
267299
*/
268300
class CheckCase {
269-
public String sqlTemplate;
270-
public String itemName;
271-
public boolean checkId = false;
301+
public String getSqlTemplate() {
302+
return this.sqlTemplate;
303+
}
272304

273-
public CheckCase(String sqlTemplate, String itemName, String lastCheckId) {
274-
checkId = true;
275-
this.sqlTemplate = sqlTemplate + lastCheckId;
305+
public void setSqlTemplate(final String sqlTemplate) {
306+
this.sqlTemplate = sqlTemplate;
307+
}
308+
309+
public String getItemName() {
310+
return this.itemName;
311+
}
312+
313+
public void setItemName(final String itemName) {
276314
this.itemName = itemName;
277315
}
278316

317+
public boolean isCheckId() {
318+
return this.checkId;
319+
}
320+
321+
public void setCheckId(final boolean checkId) {
322+
this.checkId = checkId;
323+
}
324+
325+
private String sqlTemplate;
326+
private String itemName;
327+
private boolean checkId = false;
328+
329+
public CheckCase(String sqlTemplate, String itemName, String lastCheckId) {
330+
setCheckId(true);
331+
setSqlTemplate(sqlTemplate + lastCheckId);
332+
setItemName(itemName);
333+
}
334+
279335
public CheckCase(String sqlTemplate, String itemName) {
280336
checkId = false;
281-
this.sqlTemplate = sqlTemplate;
282-
this.itemName = itemName;
337+
setSqlTemplate(sqlTemplate);
338+
setItemName(itemName);
283339
}
284340
}

0 commit comments

Comments
 (0)