Skip to content

Commit c8053bd

Browse files
MorganCaronMorganCaron
committed
Language/CSV/Parsing
1 parent 159fe09 commit c8053bd

5 files changed

Lines changed: 143 additions & 73 deletions

File tree

‎modules/Language/CSV/Mapping.mpp‎

Lines changed: 17 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,30 @@ import std;
44

55
import CppUtils.String;
66
import CppUtils.Type.Mapping;
7-
import CppUtils.FileSystem;
87

98
export namespace CppUtils::Language::CSV
109
{
10+
template<class Mapping>
11+
struct GetFunctionMapping final
12+
{
13+
using Type = Type::Mapping<>;
14+
};
15+
16+
template<class Mapping>
17+
requires requires { typename Mapping::FunctionMapping; }
18+
struct GetFunctionMapping<Mapping> final
19+
{
20+
using Type = typename Mapping::FunctionMapping;
21+
};
22+
1123
using Line = std::vector<std::string_view>;
1224

1325
template<class Object, class Mapping>
1426
[[nodiscard]] inline auto toStruct(const auto& csvLine) -> Object
1527
{
1628
using CSVMapping = typename Mapping::CSVMapping;
1729
using StructMapping = typename Mapping::StructMapping;
18-
using FunctionMapping = typename Mapping::FunctionMapping;
30+
using FunctionMapping = typename GetFunctionMapping<Mapping>::Type;
1931
static_assert(std::is_default_constructible_v<Object>);
2032
auto object = Object{};
2133
std::apply([&](auto&&... csvPairs) -> void {
@@ -43,36 +55,15 @@ export namespace CppUtils::Language::CSV
4355
return object;
4456
}
4557

46-
template<class Object, class Mapping>
47-
[[nodiscard]] inline auto toStruct(std::string_view csvLine, std::string_view separator = ";") -> Object
48-
{
49-
const auto tokens = csvLine
50-
| std::views::split(separator)
51-
| std::views::transform([](auto&& range) { return std::string_view{range}; })
52-
| std::ranges::to<std::vector<std::string_view>>();
53-
return toStruct<Object, Mapping>(tokens);
54-
}
55-
5658
template<class Object, class Mapping>
5759
[[nodiscard]] inline auto toStructs(
58-
std::string_view csv,
59-
std::string_view columnSeparator = ";",
60-
std::string_view lineSeparator = "\n",
60+
const auto& lines,
6161
std::size_t dropNFirstLines = 0,
6262
std::vector<Object>&& objects = {}) -> std::vector<Object>
6363
{
64-
const auto lines = csv
65-
| std::views::split(lineSeparator)
66-
| std::views::transform([&](auto&& range) {
67-
auto line = std::string_view{range};
68-
if (lineSeparator == "\n" and not std::empty(line) and line.back() == '\r')
69-
line.remove_suffix(1);
70-
return line;
71-
})
72-
| std::ranges::to<std::vector<std::string_view>>();
7364
for (auto i = dropNFirstLines; i < std::size(lines); ++i)
74-
if (const auto trimmedLine = String::trimString(lines[i]); not std::empty(trimmedLine))
75-
objects.push_back(toStruct<Object, Mapping>(trimmedLine, columnSeparator));
65+
if (const auto& line = lines[i]; not std::empty(line))
66+
objects.push_back(toStruct<Object, Mapping>(line));
7667
return std::move(objects);
7768
}
7869

@@ -116,29 +107,4 @@ export namespace CppUtils::Language::CSV
116107
}
117108
return csv;
118109
}
119-
120-
template<class Object, class Mapping>
121-
[[nodiscard]] inline auto loadFile(
122-
const std::filesystem::path& filePath,
123-
std::string_view columnSeparator = ";",
124-
std::string_view lineSeparator = "\n",
125-
std::size_t dropNFirstLines = 0,
126-
std::vector<Object>&& elements = {}) -> std::vector<Object>
127-
{
128-
auto fileContent = FileSystem::String::read(filePath);
129-
return toStructs<Object, Mapping>(fileContent, columnSeparator, lineSeparator, dropNFirstLines, std::move(elements));
130-
}
131-
132-
template<class Object, class Mapping>
133-
[[nodiscard]] inline auto loadDirectory(const std::filesystem::path& directory,
134-
std::string_view columnSeparator = ";",
135-
std::string_view lineSeparator = "\n",
136-
std::size_t dropNFirstLines = 0,
137-
std::vector<Object>&& elements = {}) -> std::vector<Object>
138-
{
139-
FileSystem::forFilesWithExtension(directory, ".csv", [&](const auto& filePath) {
140-
elements = loadFile<Object, Mapping>(filePath, columnSeparator, lineSeparator, dropNFirstLines, std::move(elements));
141-
});
142-
return std::move(elements);
143-
}
144110
}

‎modules/Language/CSV/Parsing.mpp‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
export module CppUtils.Language.CSV.Parsing;
2+
3+
import std;
4+
import CppUtils.String;
5+
import CppUtils.FileSystem;
6+
import CppUtils.Language.CSV.Mapping;
7+
8+
export namespace CppUtils::Language::CSV
9+
{
10+
[[nodiscard]] inline auto splitLine(std::string_view line, std::string_view separator = ";") -> std::vector<std::string_view>
11+
{
12+
return line
13+
| std::views::split(separator)
14+
| std::views::transform([](auto&& range) { return String::trimString(std::string_view{range}); })
15+
| std::ranges::to<std::vector<std::string_view>>();
16+
}
17+
18+
[[nodiscard]] inline auto parseCSV(std::string_view csv, std::string_view columnSeparator = ";", std::string_view lineSeparator = "\n") -> std::vector<std::vector<std::string_view>>
19+
{
20+
return csv
21+
| std::views::split(lineSeparator)
22+
| std::views::transform([lineSeparator](auto&& range) -> std::string_view {
23+
auto line = std::string_view{range};
24+
if (lineSeparator == "\n" and not std::empty(line) and line.back() == '\r')
25+
line.remove_suffix(1);
26+
return String::trimString(line);
27+
})
28+
| std::views::filter([](std::string_view line) { return not std::empty(line); })
29+
| std::views::transform([columnSeparator](std::string_view line) { return splitLine(line, columnSeparator); })
30+
| std::ranges::to<std::vector<std::vector<std::string_view>>>();
31+
}
32+
33+
template<class Object, class Mapping>
34+
[[nodiscard]] inline auto loadFile(
35+
const std::filesystem::path& filePath,
36+
std::size_t dropNFirstLines = 0,
37+
std::string_view columnSeparator = ";",
38+
std::string_view lineSeparator = "\n",
39+
std::vector<Object>&& elements = {})
40+
{
41+
auto fileContent = FileSystem::String::read(filePath);
42+
return toStructs<Object, Mapping>(parseCSV(fileContent, columnSeparator, lineSeparator), dropNFirstLines, std::move(elements));
43+
}
44+
45+
template<class Object, class Mapping>
46+
[[nodiscard]] inline auto loadDirectory(const std::filesystem::path& directory,
47+
std::size_t dropNFirstLines = 0,
48+
std::string_view columnSeparator = ";",
49+
std::string_view lineSeparator = "\n",
50+
std::vector<Object>&& elements = {})
51+
{
52+
FileSystem::forFilesWithExtension(directory, ".csv", [&](const auto& filePath) {
53+
elements = loadFile<Object, Mapping>(filePath, dropNFirstLines, columnSeparator, lineSeparator, std::move(elements));
54+
});
55+
return std::move(elements);
56+
}
57+
58+
namespace Literals
59+
{
60+
[[nodiscard]] inline auto operator""_csv(const char* cString, std::size_t size) -> std::vector<std::vector<std::string_view>>
61+
{
62+
return parseCSV(std::string_view{cString, size});
63+
}
64+
}
65+
}

‎modules/Language/Language.mpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ export import CppUtils.Language.GrammarParser;
66
export import CppUtils.Language.VirtualMachine;
77
export import CppUtils.Language.MetaEvaluator;
88
export import CppUtils.Language.CSV.Mapping;
9+
export import CppUtils.Language.CSV.Parsing;

‎tests/Language/CSV/Mapping.mpp‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -49,40 +49,40 @@ namespace CppUtils::UnitTest::Language::CSV::Mapping
4949
CppUtils::Type::Pair<"Signed integer"_token, &FileObject::d>,
5050
CppUtils::Type::Pair<"Floating point"_token, &FileObject::e>>;
5151

52+
namespace Conversion
53+
{
54+
constexpr auto doubleValue = [](std::size_t& member, std::string_view value) {
55+
std::from_chars(std::data(value), std::data(value) + std::size(value), member);
56+
member *= 2;
57+
};
58+
}
59+
5260
struct ObjectMapping final
5361
{
5462
using CSVMapping = UnitTest::Language::CSV::Mapping::CSVMapping;
5563
using StructMapping = UnitTest::Language::CSV::Mapping::StructMapping;
56-
using FunctionMapping = CppUtils::Type::Mapping<>;
5764
};
5865

5966
struct FileObjectMapping final
6067
{
6168
using CSVMapping = UnitTest::Language::CSV::Mapping::CSVMapping;
6269
using StructMapping = UnitTest::Language::CSV::Mapping::FileStructMapping;
63-
using FunctionMapping = CppUtils::Type::Mapping<>;
6470
};
6571

6672
struct ObjectFunctionMapping final
6773
{
6874
using CSVMapping = UnitTest::Language::CSV::Mapping::CSVMapping;
6975
using StructMapping = UnitTest::Language::CSV::Mapping::StructMapping;
7076
using FunctionMapping = CppUtils::Type::Mapping<
71-
CppUtils::Type::Pair<"Unsigned integer"_token, [](std::size_t& member, std::string_view value) {
72-
std::from_chars(std::data(value), std::data(value) + std::size(value), member);
73-
member *= 2;
74-
}>>;
77+
CppUtils::Type::Pair<"Unsigned integer"_token, Conversion::doubleValue>>;
7578
};
7679

7780
struct ObjectLambdaMapping final
7881
{
7982
using CSVMapping = UnitTest::Language::CSV::Mapping::CSVMapping;
8083
using StructMapping = UnitTest::Language::CSV::Mapping::StructMapping;
8184
using FunctionMapping = CppUtils::Type::Mapping<
82-
CppUtils::Type::Pair<"Unsigned integer"_token, [](std::size_t& member, std::string_view value) {
83-
std::from_chars(std::data(value), std::data(value) + std::size(value), member);
84-
member *= 2;
85-
}>>;
85+
CppUtils::Type::Pair<"Unsigned integer"_token, Conversion::doubleValue>>;
8686
};
8787

8888
struct ObjectFunctorMapping final
@@ -105,22 +105,22 @@ namespace CppUtils::UnitTest::Language::CSV::Mapping
105105

106106
auto _ = TestSuite{"Language/CSV/Mapping", {}, [](TestSuite& suite) {
107107
using namespace std::literals;
108+
using namespace CppUtils::Language::CSV::Literals;
108109

109110
suite.addTest("CSV line to Struct", [&] {
110-
constexpr auto csvLine = "foo;bar;42;-1;3.14"sv;
111+
const auto csvLine = "foo;bar;42;-1;3.14"_csv[0];
111112

112113
const auto object = CppUtils::Language::CSV::toStruct<Object, ObjectMapping>(csvLine);
113114

114115
suite.expectEqual(object, Object{"foo", "bar", 42, -1, 3.14f});
115116
});
116117

117118
suite.addTest("CSV to Structs", [&] {
118-
constexpr auto csv = R"(
119+
const auto objects = CppUtils::Language::CSV::toStructs<Object, ObjectMapping>(R"(
119120
a;b;c;d;e
120121
foo;bar;42;-1;3.14
121-
)"sv;
122-
123-
const auto objects = CppUtils::Language::CSV::toStructs<Object, ObjectMapping>(csv, ";", "\n", 2uz);
122+
)"_csv,
123+
2uz);
124124

125125
suite.expectEqual(objects, std::vector<Object>{{"foo", "bar", 42, -1, 3.14f}});
126126
});
@@ -148,7 +148,7 @@ namespace CppUtils::UnitTest::Language::CSV::Mapping
148148
const auto filePath = directory / "test.csv";
149149
CppUtils::FileSystem::String::write(filePath, "a;b;c;d;e\nfoo;bar;42;-1;3.14\n"sv);
150150

151-
const auto objects = CppUtils::Language::CSV::loadFile<FileObject, FileObjectMapping>(filePath, ";", "\n", 1uz);
151+
const auto objects = CppUtils::Language::CSV::loadFile<FileObject, FileObjectMapping>(filePath, 1uz);
152152

153153
suite.expectEqual(objects, std::vector<FileObject>{{"foo", "bar", 42, -1, 3.14f}});
154154
}};
@@ -159,7 +159,7 @@ namespace CppUtils::UnitTest::Language::CSV::Mapping
159159
const auto filePath = directory / "test_crlf.csv";
160160
CppUtils::FileSystem::String::write(filePath, "a;b;c;d;e\r\nfoo;bar;42;-1;3.14\r\n"sv);
161161

162-
const auto objects = CppUtils::Language::CSV::loadFile<FileObject, FileObjectMapping>(filePath, ";", "\n", 1uz);
162+
const auto objects = CppUtils::Language::CSV::loadFile<FileObject, FileObjectMapping>(filePath, 1uz);
163163

164164
suite.expectEqual(objects, std::vector<FileObject>{{"foo", "bar", 42, -1, 3.14f}});
165165
}};
@@ -170,7 +170,7 @@ namespace CppUtils::UnitTest::Language::CSV::Mapping
170170
CppUtils::FileSystem::String::write(directory / "test1.csv", "a;b;c;d;e\nfoo;bar;1;-10;1.1\n"sv);
171171
CppUtils::FileSystem::String::write(directory / "test2.csv", "a;b;c;d;e\nhello;world;2;-20;2.2\n"sv);
172172

173-
auto objects = CppUtils::Language::CSV::loadDirectory<FileObject, FileObjectMapping>(directory, ";", "\n", 1uz);
173+
auto objects = CppUtils::Language::CSV::loadDirectory<FileObject, FileObjectMapping>(directory, 1uz);
174174
std::ranges::sort(objects, {}, &FileObject::b);
175175

176176
const auto expected = std::vector<FileObject>{
@@ -187,7 +187,7 @@ namespace CppUtils::UnitTest::Language::CSV::Mapping
187187
CppUtils::FileSystem::String::write(filePath, "a;b;c;d;e\nfoo;bar;42;-1;3.14\n"sv);
188188

189189
auto objects = std::vector<FileObject>{{"existing", "object", 0, 0, 0.0f}};
190-
objects = CppUtils::Language::CSV::loadFile<FileObject, FileObjectMapping>(filePath, ";", "\n", 1uz, std::move(objects));
190+
objects = CppUtils::Language::CSV::loadFile<FileObject, FileObjectMapping>(filePath, 1uz, ";", "\n", std::move(objects));
191191

192192
const auto expected = std::vector<FileObject>{
193193
{"existing", "object", 0, 0, 0.0f},
@@ -198,23 +198,23 @@ namespace CppUtils::UnitTest::Language::CSV::Mapping
198198
});
199199

200200
suite.addTest("FunctionMapping", [&] {
201-
constexpr auto csvLine = "foo;bar;21;-1;3.14"sv;
201+
const auto csvLine = "foo;bar;21;-1;3.14"_csv[0];
202202

203203
const auto object = CppUtils::Language::CSV::toStruct<Object, ObjectFunctionMapping>(csvLine);
204204

205205
suite.expectEqual(object, Object{"foo", "bar", 42, -1, 3.14f});
206206
});
207207

208208
suite.addTest("LambdaMapping", [&] {
209-
constexpr auto csvLine = "foo;bar;21;-1;3.14"sv;
209+
const auto csvLine = "foo;bar;21;-1;3.14"_csv[0];
210210

211211
const auto object = CppUtils::Language::CSV::toStruct<Object, ObjectLambdaMapping>(csvLine);
212212

213213
suite.expectEqual(object, Object{"foo", "bar", 42, -1, 3.14f});
214214
});
215215

216216
suite.addTest("FunctorMapping (lambda with capture substitute)", [&] {
217-
constexpr auto csvLine = "foo;bar;10;-1;3.14"sv;
217+
const auto csvLine = "foo;bar;10;-1;3.14"_csv[0];
218218

219219
const auto object = CppUtils::Language::CSV::toStruct<Object, ObjectFunctorMapping>(csvLine);
220220

‎tests/Language/CSV/Parsing.mpp‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
export module CppUtils.UnitTests.Language.CSV.Parsing;
2+
3+
import CppUtils;
4+
5+
namespace CppUtils::UnitTest::Language::CSV::Parsing
6+
{
7+
auto _ = TestSuite{"Language/CSV/Parsing", {}, [](TestSuite& suite) {
8+
using namespace std::literals;
9+
using namespace CppUtils::Language::CSV::Literals;
10+
11+
suite.addTest("_csv literal", [&] {
12+
const auto csv = "a;b\nc;d"_csv;
13+
14+
suite.expectEqual(std::size(csv), 2uz);
15+
suite.expectEqual(csv[0], std::vector<std::string_view>{"a", "b"});
16+
suite.expectEqual(csv[1], std::vector<std::string_view>{"c", "d"});
17+
});
18+
19+
suite.addTest("parseCSV with multiple lines and spaces", [&] {
20+
const auto csv = CppUtils::Language::CSV::parseCSV(R"(
21+
header1 ; header2
22+
value1 ; value2
23+
)");
24+
25+
suite.expectEqual(std::size(csv), 2uz);
26+
suite.expectEqual(csv[0], std::vector<std::string_view>{"header1", "header2"});
27+
suite.expectEqual(csv[1], std::vector<std::string_view>{"value1", "value2"});
28+
});
29+
30+
suite.addTest("splitLine", [&] {
31+
const auto values = CppUtils::Language::CSV::splitLine(" foo ; bar ");
32+
33+
suite.expectEqual(std::size(values), 2uz);
34+
suite.expectEqual(values[0], "foo"sv);
35+
suite.expectEqual(values[1], "bar"sv);
36+
});
37+
}};
38+
}

0 commit comments

Comments
 (0)