Skip to content

Commit e412d45

Browse files
arfioarfio
committed
traceevent: Handle pytorch traces
Make use of the event path, changes the logic to handle array and object trace. Add a test using a small sample of stable diffusion trace. Signed-off-by: Arnaud Fiorini <fiorini.arnaud@gmail.com>
1 parent af6a511 commit e412d45

4 files changed

Lines changed: 188 additions & 28 deletions

File tree

‎tracetypes/org.eclipse.tracecompass.incubator.traceevent.core.tests/src/org/eclipse/tracecompass/incubator/traceevent/core/tests/TraceEventTraceTest.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,28 @@ public void testChromeosTrace() throws TmfTraceException {
201201
assertEquals(expectedProperties, properties);
202202
}
203203

204+
/**
205+
* Test a chromeOs system trace
206+
*
207+
* @throws TmfTraceException
208+
* should not happen
209+
*/
210+
@Test
211+
public void testPytorchTrace() throws TmfTraceException {
212+
String[] env = { "Type", "Trace-Event"};
213+
Map<String, String> expectedProperties = new LinkedHashMap<>();
214+
for (int i = 0; i < env.length; i += 2) {
215+
expectedProperties.put(env[i], env[i + 1]);
216+
}
217+
String path = "traces/stable-diffusion.json";
218+
int nbEvents = 12;
219+
ITmfTimestamp startTime = TmfTimestamp.fromNanos(5624160250199160L);
220+
ITmfTimestamp endTime = TmfTimestamp.fromNanos(5624160263891443L);
221+
Map<String, String> properties = testTrace(path, nbEvents, startTime, endTime);
222+
assertNotNull(properties);
223+
assertEquals(expectedProperties, properties);
224+
}
225+
204226
private static Map<String, String> testTrace(String path, int nbEvents, ITmfTimestamp startTime, ITmfTimestamp endTime) throws TmfTraceException {
205227
ITmfTrace trace = new TraceEventTrace();
206228
try {

‎tracetypes/org.eclipse.tracecompass.incubator.traceevent.core.tests/traces/stable-diffusion.json‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
{
2+
"schemaVersion": 1,
3+
"deviceProperties": [
4+
{
5+
"id": 0, "name": "AMD Radeon RX 6700 XT", "totalGlobalMem": 12868124672,
6+
"computeMajor": 10, "computeMinor": 3,
7+
"maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
8+
"regsPerBlock": 65536, "warpSize": 32,
9+
"sharedMemPerBlock": 65536, "numSms": 20
10+
, "maxSharedMemoryPerMultiProcessor": 65536
11+
}
12+
],
13+
"record_shapes": 1,
14+
"profile_memory": 1,
15+
"traceEvents": [
16+
{
17+
"ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 301562, "tid": 455947,
18+
"ts": 5624160250199.160, "dur": 25.537,
19+
"args": {
20+
"External id": 1,"Record function id": 0, "Concrete Inputs": ["[1, 77]", "4", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 0
21+
}
22+
},
23+
{
24+
"ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 301562, "tid": 455947,
25+
"ts": 5624160251010.565, "dur": 1.312,
26+
"args": {
27+
"External id": 2,"Record function id": 0, "Concrete Inputs": ["", "", "4", "False", "False", ""], "Input type": ["long int", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[77, 1], [], [], [], [], []], "Input Dims": [[1, 77], [], [], [], [], []], "Ev Idx": 1
28+
}
29+
},
30+
{
31+
"ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 301562, "tid": 455947,
32+
"ts": 5624160251455.876, "dur": 1.172,
33+
"args": {
34+
"External id": 3,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[77, 1]], "Input Dims": [[1, 77]], "Ev Idx": 2
35+
}
36+
},
37+
{
38+
"ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 301562, "tid": 455947,
39+
"ts": 5624160251470.503, "dur": 2330.181,
40+
"args": {
41+
"External id": 4,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[77, 1], [], [], [], [], [], [], []], "Input Dims": [[1, 77], [], [], [], [], [], [], []], "Ev Idx": 3
42+
}
43+
},
44+
{
45+
"ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 301562, "tid": 455947,
46+
"ts": 5624160252062.298, "dur": 1737.624,
47+
"args": {
48+
"External id": 5,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", ""], "Input Strides": [[77, 1], [], [], [], [], [], []], "Input Dims": [[1, 77], [], [], [], [], [], []], "Ev Idx": 4
49+
}
50+
},
51+
{
52+
"ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 301562, "tid": 455947,
53+
"ts": 5624160252071.275, "dur": 17.252,
54+
"args": {
55+
"External id": 6,"Record function id": 0, "Concrete Inputs": ["[1, 77]", "[77, 1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5
56+
}
57+
},
58+
{
59+
"ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 301562, "tid": 455947,
60+
"ts": 5624160252092.114, "dur": 1703.270,
61+
"args": {
62+
"External id": 7,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[77, 1], [77, 1], []], "Input Dims": [[1, 77], [1, 77], []], "Ev Idx": 6
63+
}
64+
},
65+
{
66+
"ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 301562, "tid": 455947,
67+
"ts": 5624160253949.171, "dur": 17.723,
68+
"args": {
69+
"External id": 8,"Record function id": 0, "Concrete Inputs": ["", "[-1, 77]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[77, 1], []], "Input Dims": [[1, 77], []], "Ev Idx": 7
70+
}
71+
},
72+
{
73+
"ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 301562, "tid": 455947,
74+
"ts": 5624160263728.188, "dur": 60.773,
75+
"args": {
76+
"External id": 9,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[77, 1], [], [], [], []], "Input Dims": [[1, 77], [], [], [], []], "Ev Idx": 8
77+
}
78+
},
79+
{
80+
"ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 301562, "tid": 455947,
81+
"ts": 5624160263767.241, "dur": 5.159,
82+
"args": {
83+
"External id": 10,"Record function id": 0, "Concrete Inputs": ["", "[1, 77]", "[77, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[77, 1], [], [], []], "Input Dims": [[1, 77], [], [], []], "Ev Idx": 9
84+
}
85+
},
86+
{
87+
"ph": "X", "cat": "cpu_op", "name": "aten::embedding", "pid": 301562, "tid": 455947,
88+
"ts": 5624160263879.590, "dur": 140.292,
89+
"args": {
90+
"External id": 11,"Record function id": 0, "Concrete Inputs": ["", "", "-1", "False", "False"], "Input type": ["c10::Half", "long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[768, 1], [77, 1], [], [], []], "Input Dims": [[49408, 768], [1, 77], [], [], []], "Ev Idx": 10
91+
}
92+
},
93+
{
94+
"ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 301562, "tid": 455947,
95+
"ts": 5624160263891.443, "dur": 7.193,
96+
"args": {
97+
"External id": 12,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[77, 1], []], "Input Dims": [[1, 77], []], "Ev Idx": 11
98+
}
99+
}
100+
]
101+
}
102+

‎tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/trace/TraceEventSortingJob.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
package org.eclipse.tracecompass.incubator.internal.traceevent.core.trace;
1313

1414
import java.io.IOException;
15+
import java.util.List;
1516

1617
import org.eclipse.tracecompass.internal.jsontrace.core.job.SortingJob;
1718
import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
@@ -31,9 +32,11 @@ public class TraceEventSortingJob extends SortingJob {
3132
* the trace to be sort
3233
* @param path
3334
* the path to the trace file
35+
* @param pathToEvents
36+
* the json key to the events array
3437
*/
35-
public TraceEventSortingJob(ITmfTrace trace, String path) {
36-
super(trace, path, "\"ts\":", 1); //$NON-NLS-1$
38+
public TraceEventSortingJob(ITmfTrace trace, String path, List<String> pathToEvents) {
39+
super(trace, path, "\"ts\":", pathToEvents); //$NON-NLS-1$
3740
}
3841

3942
@Override

‎tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/trace/TraceEventTrace.java‎

Lines changed: 59 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import java.io.File;
1515
import java.io.IOException;
1616
import java.io.RandomAccessFile;
17+
import java.util.Collections;
1718
import java.util.HashMap;
1819
import java.util.HashSet;
1920
import java.util.List;
@@ -101,6 +102,8 @@ public class TraceEventTrace extends JsonTrace {
101102
*/
102103
private static final String THREAD_SORT_INDEX = "thread_sort_index"; //$NON-NLS-1$
103104

105+
private static final String EVENTS_ARRAY_KEY = "traceEvents"; //$NON-NLS-1$
106+
104107
private final @NonNull Map<Object, String> fPidNames = new HashMap<>();
105108
private final @NonNull NavigableMap<Integer, String> fTidNames = new TreeMap<>();
106109
private final @NonNull Iterable<@NonNull ITmfEventAspect<?>> fEventAspects;
@@ -168,22 +171,30 @@ public void initTrace(IResource resource, String path, Class<? extends ITmfEvent
168171
fProperties.put("Type", "Trace-Event"); //$NON-NLS-1$ //$NON-NLS-2$
169172
String dir = TmfTraceManager.getSupplementaryFileDir(this);
170173
fFile = new File(dir + new File(path).getName());
171-
if (!fFile.exists()) {
172-
Job sortJob = new TraceEventSortingJob(this, path);
173-
sortJob.schedule();
174-
while (sortJob.getResult() == null) {
175-
try {
176-
sortJob.join();
177-
} catch (InterruptedException e) {
178-
throw new TmfTraceException(e.getMessage(), e);
174+
175+
try {
176+
if (!fFile.exists()) {
177+
Job sortJob;
178+
try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
179+
if (isArrayTrace(rafile)) {
180+
sortJob = new TraceEventSortingJob(this, path, Collections.emptyList());
181+
} else {
182+
sortJob = new TraceEventSortingJob(this, path, List.of(EVENTS_ARRAY_KEY));
183+
}
184+
}
185+
sortJob.schedule();
186+
while (sortJob.getResult() == null) {
187+
try {
188+
sortJob.join();
189+
} catch (InterruptedException e) {
190+
throw new TmfTraceException(e.getMessage(), e);
191+
}
192+
}
193+
IStatus result = sortJob.getResult();
194+
if (!result.isOK()) {
195+
throw new TmfTraceException("Job failed " + result.getMessage()); //$NON-NLS-1$
179196
}
180197
}
181-
IStatus result = sortJob.getResult();
182-
if (!result.isOK()) {
183-
throw new TmfTraceException("Job failed " + result.getMessage()); //$NON-NLS-1$
184-
}
185-
}
186-
try {
187198
fFileInput = new BufferedRandomAccessFile(fFile, "r"); //$NON-NLS-1$
188199
goToCorrectStart(fFileInput);
189200
/* Set the start and (current) end times for this trace */
@@ -203,6 +214,15 @@ public void initTrace(IResource resource, String path, Class<? extends ITmfEvent
203214
}
204215
}
205216

217+
private static boolean isArrayTrace(RandomAccessFile rafile) throws IOException {
218+
int val = ' ';
219+
while(val == ' ') {
220+
val = rafile.read();
221+
}
222+
rafile.seek(0);
223+
return val == '[';
224+
}
225+
206226
/**
207227
* Update the file position to be at the actual start of events, after the
208228
* trace event header
@@ -213,31 +233,44 @@ public void initTrace(IResource resource, String path, Class<? extends ITmfEvent
213233
* Exceptions reading the file
214234
*/
215235
protected static void goToCorrectStart(RandomAccessFile rafile) throws IOException {
216-
// skip start if it's {"traceEvents":
217-
String traceEventsKey = "\"traceEvents\""; //$NON-NLS-1$
218-
StringBuilder sb = new StringBuilder();
219-
int val = rafile.read();
236+
// The file either starts with [ and then all the subsequent objects are
237+
// events
238+
if (isArrayTrace(rafile)) {
239+
return;
240+
}
241+
// or we have to skip to the key EVENTS_ARRAY_KEY to get the array of
242+
// events
243+
StringBuilder stringBuffer = new StringBuilder();
220244
/*
221245
* Skip list contains all the odd control characters
222246
*/
223247
Set<Integer> skipList = new HashSet<>();
224-
skipList.add((int) ':');
225248
skipList.add((int) '\t');
226249
skipList.add((int) '\n');
227250
skipList.add((int) '\r');
228251
skipList.add((int) ' ');
229252
skipList.add((int) '\b');
230253
skipList.add((int) '\f');
231-
while (val != -1 && val != ':' && sb.length() < 14) {
254+
int maxBufferSize = 200;
255+
int maxOffset = 50000;
256+
int offset = 1;
257+
int val = 0;
258+
while (val != -1 && offset < maxOffset) {
232259
if (!skipList.contains(val)) {
233-
sb.append((char) val);
260+
stringBuffer.append((char) val);
261+
if (stringBuffer.length() > maxBufferSize) {
262+
String trailingEnd = stringBuffer.substring(maxBufferSize - EVENTS_ARRAY_KEY.length(), stringBuffer.length());
263+
stringBuffer = new StringBuilder(trailingEnd);
264+
}
265+
}
266+
if (stringBuffer.toString().endsWith(EVENTS_ARRAY_KEY)) {
267+
while (val != ':') {
268+
val = rafile.read();
269+
}
270+
return;
234271
}
235272
val = rafile.read();
236-
}
237-
if (!(sb.toString().startsWith('{' + traceEventsKey) && rafile.length() > 14)) {
238-
// Trace does not start with {"TraceEvents", maybe it's the events
239-
// directly, go back to start of trace
240-
rafile.seek(0);
273+
offset += 1;
241274
}
242275
}
243276

0 commit comments

Comments
 (0)