Skip to content
This repository was archived by the owner on Jun 24, 2025. It is now read-only.

Commit 09e66f8

Browse files
vaycorevaycore
committed
优化数据看板的数据加载机制
1 parent 501654b commit 09e66f8

4 files changed

Lines changed: 238 additions & 15 deletions

File tree

‎extender/src/main/java/burp/vaycore/common/helper/DataTableItemLoader.java‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
package burp.vaycore.common.helper;
2+
3+
import burp.vaycore.common.log.Logger;
4+
import burp.vaycore.common.utils.Utils;
5+
6+
import java.util.ArrayList;
7+
import java.util.List;
8+
import java.util.concurrent.ConcurrentLinkedQueue;
9+
import java.util.concurrent.Executors;
10+
import java.util.concurrent.ScheduledExecutorService;
11+
import java.util.concurrent.TimeUnit;
12+
import java.util.concurrent.atomic.AtomicLong;
13+
14+
/**
15+
* 数据表格组件 Item 加载器
16+
* <p>
17+
* Created by vaycore on 2024-04-13.
18+
*/
19+
public class DataTableItemLoader<T> {
20+
21+
private static final int NO_DATA_STOP_MILLIS = 10000;
22+
private static final int MAX_DATA_QUEUE_SIZE = 1000;
23+
24+
private OnDataItemLoadEvent<T> mOnDataItemLoadEvent;
25+
private final long mInitialDelay;
26+
private final long mPeriod;
27+
private final TimeUnit mTimeUnit;
28+
private final long mPeriodMillis;
29+
private ScheduledExecutorService mExecutor;
30+
private final AtomicLong mStopMillis;
31+
private final long mMinBatchSize;
32+
private final long mMaxBatchSize;
33+
private final ConcurrentLinkedQueue<T> mDataQueue;
34+
35+
public DataTableItemLoader(long period) {
36+
this(period, TimeUnit.MILLISECONDS);
37+
}
38+
39+
public DataTableItemLoader(OnDataItemLoadEvent<T> event, long period) {
40+
this(event, period, TimeUnit.MILLISECONDS);
41+
}
42+
43+
public DataTableItemLoader(long period, TimeUnit unit) {
44+
this(null, period, unit);
45+
}
46+
47+
public DataTableItemLoader(OnDataItemLoadEvent<T> event, long period, TimeUnit unit) {
48+
this(event, 0, period, unit);
49+
}
50+
51+
public DataTableItemLoader(long initialDelay, long period) {
52+
this(initialDelay, period, TimeUnit.MILLISECONDS);
53+
}
54+
55+
public DataTableItemLoader(OnDataItemLoadEvent<T> event, long initialDelay, long period) {
56+
this(event, initialDelay, period, TimeUnit.MILLISECONDS);
57+
}
58+
59+
public DataTableItemLoader(long initialDelay, long period, TimeUnit unit) {
60+
this(null, initialDelay, period, unit);
61+
}
62+
63+
public DataTableItemLoader(OnDataItemLoadEvent<T> event, long initialDelay, long period, TimeUnit unit) {
64+
if (period <= 0) {
65+
throw new IllegalArgumentException("Period must be greater than 0");
66+
}
67+
this.mOnDataItemLoadEvent = event;
68+
this.mInitialDelay = initialDelay;
69+
this.mPeriod = period;
70+
this.mTimeUnit = unit;
71+
this.mPeriodMillis = mTimeUnit.toMillis(mPeriod);
72+
this.mStopMillis = new AtomicLong(System.currentTimeMillis());
73+
this.mMinBatchSize = mPeriodMillis;
74+
this.mMaxBatchSize = (long) (mMinBatchSize * 1.6);
75+
this.mDataQueue = new ConcurrentLinkedQueue<>();
76+
}
77+
78+
private void start() {
79+
if (isRunning()) {
80+
return;
81+
}
82+
mExecutor = Executors.newSingleThreadScheduledExecutor();
83+
mExecutor.scheduleAtFixedRate(this::run, mInitialDelay, mPeriod, mTimeUnit);
84+
Logger.debug("DataTableItemLoader started");
85+
}
86+
87+
private void stop() {
88+
if (!isRunning()) {
89+
return;
90+
}
91+
mExecutor.shutdownNow();
92+
Logger.debug("DataTableItemLoader stopped");
93+
}
94+
95+
public void flush() {
96+
// 取出所有队列数据，添加到列表
97+
List<T> temps = new ArrayList<>();
98+
while (!mDataQueue.isEmpty()) {
99+
T item = mDataQueue.poll();
100+
temps.add(item);
101+
}
102+
invokeOnDataItemLoadEvent(temps);
103+
temps.clear();
104+
Logger.debug("DataTableItemLoader flushed");
105+
}
106+
107+
private synchronized void restart() {
108+
stop();
109+
start();
110+
}
111+
112+
private boolean isRunning() {
113+
return mExecutor != null && !mExecutor.isShutdown();
114+
}
115+
116+
private void run() {
117+
if (mDataQueue.isEmpty()) {
118+
long stopMillis = System.currentTimeMillis() - mStopMillis.get();
119+
// 限制时间内无数据，停止计时器
120+
if (stopMillis >= NO_DATA_STOP_MILLIS) {
121+
Logger.debug("DataTableItemLoader no data, stopping...");
122+
stop();
123+
}
124+
return;
125+
}
126+
long batchSize = Utils.nextLong(mMinBatchSize, mMaxBatchSize);
127+
int counter = 0;
128+
// 取出队列的数据
129+
ArrayList<T> temps = new ArrayList<>();
130+
while (counter < batchSize && !mDataQueue.isEmpty()) {
131+
T data = mDataQueue.poll();
132+
temps.add(data);
133+
counter++;
134+
}
135+
invokeOnDataItemLoadEvent(temps);
136+
temps.clear();
137+
}
138+
139+
public void pushItem(T item) {
140+
mDataQueue.offer(item);
141+
while (mDataQueue.size() > MAX_DATA_QUEUE_SIZE) {
142+
// 队列数据达到限制，暂停生产，等待消费
143+
try {
144+
Thread.sleep(mPeriodMillis);
145+
Logger.debug("DataTableItemLoader reached max queue size, sleeping...");
146+
} catch (InterruptedException e) {
147+
// 线程中断了，加载所有数据
148+
flush();
149+
return;
150+
}
151+
}
152+
// 添加数据后，重置计数器
153+
mStopMillis.set(System.currentTimeMillis());
154+
// 检测是否已经停止运行
155+
if (!isRunning()) {
156+
restart();
157+
}
158+
}
159+
160+
public void setOnDataItemLoadEvent(OnDataItemLoadEvent<T> event) {
161+
this.mOnDataItemLoadEvent = event;
162+
}
163+
164+
private void invokeOnDataItemLoadEvent(List<T> items) {
165+
if (mOnDataItemLoadEvent != null) {
166+
mOnDataItemLoadEvent.onDataItemLoaded(items);
167+
}
168+
}
169+
170+
public interface OnDataItemLoadEvent<T> {
171+
172+
void onDataItemLoaded(List<T> items);
173+
}
174+
}

‎extender/src/main/java/burp/vaycore/common/utils/Utils.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,17 @@ public static int randomInt(int maxValue) {
6464
return (int) (minValue + Math.random() * (maxValue - minValue + 1));
6565
}
6666

67+
/**
68+
* 生成 long 类型随机值
69+
*
70+
* @param minValue 最小值（包含）
71+
* @param maxValue 最大值（包含）
72+
* @return 随机值
73+
*/
74+
public static long nextLong(long minValue, long maxValue) {
75+
return (long) (minValue + Math.random() * (maxValue - minValue + 1));
76+
}
77+
6778
/**
6879
* 从列表随机获取一个数据
6980
*

‎extender/src/main/java/burp/vaycore/onescan/ui/tab/DataBoardTab.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,10 @@ private void stopTask() {
231231
mListenProxyMessage.setSelected(false);
232232
// 发送事件消息
233233
sendTabEvent(EVENT_STOP_TASK);
234+
// 通知停止添加任务数据
235+
if (mTaskTable != null) {
236+
mTaskTable.stopAddTaskData();
237+
}
234238
}
235239

236240
/**

‎extender/src/main/java/burp/vaycore/onescan/ui/widget/TaskTable.java‎

Lines changed: 49 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import burp.vaycore.common.filter.FilterRule;
44
import burp.vaycore.common.filter.TableFilter;
5+
import burp.vaycore.common.helper.DataTableItemLoader;
56
import burp.vaycore.common.helper.IconHash;
67
import burp.vaycore.common.helper.UIHelper;
78
import burp.vaycore.common.layout.VLayout;
@@ -23,10 +24,10 @@
2324
import java.awt.event.MouseEvent;
2425
import java.net.MalformedURLException;
2526
import java.net.URL;
26-
import java.util.ArrayList;
27-
import java.util.Comparator;
2827
import java.util.List;
28+
import java.util.*;
2929
import java.util.concurrent.atomic.AtomicInteger;
30+
import java.util.stream.Collectors;
3031

3132
/**
3233
* 任务列表
@@ -571,6 +572,13 @@ public void clearAll() {
571572
mLastSelectedRow = -1;
572573
}
573574

575+
/**
576+
* 停止添加任务数据（任务停止时调用）
577+
*/
578+
public void stopAddTaskData() {
579+
mTaskTableModel.stopAddTaskData();
580+
}
581+
574582
/**
575583
* 获取扫描任务的数量
576584
*/
@@ -634,7 +642,8 @@ public interface OnTaskTableEventListener {
634642
/**
635643
* 列表适配器
636644
*/
637-
public static class TaskTableModel extends AbstractTableModel {
645+
public static class TaskTableModel extends AbstractTableModel
646+
implements DataTableItemLoader.OnDataItemLoadEvent<TaskData> {
638647

639648
public static final String[] COLUMN_NAMES = new String[]{
640649
L.get("task_table_columns.id"),
@@ -650,43 +659,58 @@ public static class TaskTableModel extends AbstractTableModel {
650659
L.get("task_table_columns.comment"),
651660
L.get("task_table_columns.color"),
652661
};
653-
private final ArrayList<TaskData> mData;
662+
private final List<TaskData> mData;
654663
private final AtomicInteger mCounter;
664+
private final DataTableItemLoader<TaskData> mItemLoader;
655665

656666
public TaskTableModel() {
657-
mData = new ArrayList<>();
667+
mData = Collections.synchronizedList(new ArrayList<>());
658668
mCounter = new AtomicInteger();
669+
mItemLoader = new DataTableItemLoader<>(this, 500);
659670
}
660671

661672
public void add(TaskData data) {
662673
if (data == null || data.getReqResp() == null) {
663674
return;
664675
}
665-
synchronized (this.mData) {
666-
int index = mData.size();
667-
int id = mCounter.getAndIncrement();
668-
data.setId(id);
669-
this.mData.add(data);
670-
fireTableRowsInserted(index, index);
676+
int id = mCounter.getAndIncrement();
677+
data.setId(id);
678+
mItemLoader.pushItem(data);
679+
}
680+
681+
public void addAll(List<TaskData> items) {
682+
if (items == null || items.isEmpty()) {
683+
return;
684+
}
685+
// 数据不允许为空
686+
List<TaskData> validItems = items.stream().filter(Objects::nonNull).collect(Collectors.toList());
687+
if (validItems.isEmpty()) {
688+
return;
671689
}
690+
SwingUtilities.invokeLater(() -> {
691+
int firstRow = getRowCount();
692+
this.mData.addAll(validItems);
693+
int lastRow = getRowCount() - 1;
694+
fireTableRowsInserted(firstRow, lastRow);
695+
});
672696
}
673697

674698
public void removeItems(List<TaskData> list) {
675699
if (list == null || list.isEmpty()) {
676700
return;
677701
}
678-
synchronized (this.mData) {
702+
SwingUtilities.invokeLater(() -> {
679703
this.mData.removeAll(list);
680704
fireTableDataChanged();
681-
}
705+
});
682706
}
683707

684708
public void clearAll() {
685-
synchronized (this.mData) {
709+
SwingUtilities.invokeLater(() -> {
686710
mData.clear();
687711
mCounter.set(0);
688712
fireTableDataChanged();
689-
}
713+
});
690714
}
691715

692716
@Override
@@ -714,5 +738,15 @@ public Class<?> getColumnClass(int columnIndex) {
714738
public String getColumnName(int column) {
715739
return COLUMN_NAMES[column];
716740
}
741+
742+
@Override
743+
public void onDataItemLoaded(List<TaskData> items) {
744+
addAll(items);
745+
}
746+
747+
public void stopAddTaskData() {
748+
// 任务停止后。取出所有队列数据，添加到列表
749+
mItemLoader.flush();
750+
}
717751
}
718752
}

0 commit comments

Comments
 (0)