forked from kumar-shridhar/Know-Your-Intent
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtrain_test_split.py
More file actions
30 lines (24 loc) · 761 Bytes
/
Copy pathtrain_test_split.py
File metadata and controls
30 lines (24 loc) · 761 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
import csv
import json
with open("/path/to/dataset/file.json", "r") as f:
dataset = json.load(f)
all = [[sample["text"], sample["intent"]] for sample in dataset["sentences"]]
with open('/path/to/test/file.csv') as csvfile:
readCSV = csv.reader(csvfile, delimiter=',')
all_rows = list(readCSV)
test, train = [], []
for t in all:
found = False
for tes in all_rows:
if tes[0] == t[0]:
test.append(t)
found = True
break
if not found:
train.append(t)
print(len(train))
print(len(test))
with open("train.csv", "w") as f:
f.write("\n".join(["\t".join(sample) for sample in train]))
with open("test.csv", "w") as f:
f.write("\n".join(["\t".join(sample) for sample in test]))