Skip to content

Commit 920bba5

Browse files
doeixddoeixd
committed
added mergefn
1 parent 84a4b4a commit 920bba5

2 files changed

Lines changed: 134 additions & 72 deletions

File tree

‎README.md‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -434,8 +434,6 @@ const processProducts = pipe(
434434
products => sortBy(products, 'price') // Sort by price
435435
);
436436
```
437-
438-
439437
## API Documentation
440438

441439
### Core Class: CSV
@@ -528,8 +526,8 @@ const processProducts = pipe(
528526

529527
| Function | Description |
530528
|----------|-------------|
531-
| `arrayToObjArray(data, headerMap, headerRow?)` | Transform arrays to objects |
532-
| `objArrayToArray(data, headerMap, headers?, includeHeaders?)` | Transform objects to arrays |
529+
| `arrayToObjArray(data, headerMap, headerRow?, mergeFn?)` | Transform arrays to objects with optional value transformation |
530+
| `objArrayToArray(data, headerMap, headers?, includeHeaders?, transformFn?)` | Transform objects to arrays with optional value transformation |
533531
| `groupByField(data, field)` | Group objects by a field value |
534532

535533
#### Generator Functions
@@ -544,7 +542,18 @@ const processProducts = pipe(
544542

545543
| Function | Description |
546544
|----------|-------------|
547-
| `createHeaderMapFns(headerMap)` | Create functions for mapping between row arrays and objects |
545+
| `createHeaderMapFns(headerMap, mergeFn?)` | Create functions for mapping between row arrays and objects with optional value transformation |
546+
547+
## Types and Interfaces
548+
549+
### MergeFn
550+
551+
| Parameter | Type | Description |
552+
|-----------|------|-------------|
553+
| `obj` | `Partial<T>` | The partially constructed target object |
554+
| `key` | `string` | The target path where the value will be stored (e.g., 'profile.firstName') |
555+
| `value` | `any` | The original value from the source data |
556+
| **returns** | `any` | The transformed value to be stored in the target object |
548557

549558
## Options Interfaces
550559

@@ -556,6 +565,7 @@ const processProducts = pipe(
556565
| `csvOptions` | Object | CSV parsing options |
557566
| `transform` | Function | Function to transform raw content |
558567
| `headerMap` | HeaderMap | Header mapping configuration |
568+
| `mergeFn` | MergeFn | Function to customize value transformations during mapping |
559569
| `retry` | RetryOptions | Options for retry logic |
560570
| `validateData` | boolean | Enable data validation |
561571
| `allowEmptyValues` | boolean | Allow empty values in the CSV |
@@ -568,6 +578,7 @@ const processProducts = pipe(
568578
| `stringifyOptions` | Object | Options for stringifying |
569579
| `streaming` | boolean | Use streaming for large files |
570580
| `headerMap` | HeaderMap | Header mapping configuration |
581+
| `transformFn` | Function | Function to transform values during object-to-array conversion |
571582
| `streamingThreshold` | number | Threshold for using streaming |
572583
| `retry` | RetryOptions | Options for retry logic |
573584

@@ -579,6 +590,7 @@ const processProducts = pipe(
579590
| `transform` | Function | Function to transform rows |
580591
| `batchSize` | number | Size of batches (for csvBatchGenerator) |
581592
| `headerMap` | HeaderMap | Header mapping for transformation |
593+
| `mergeFn` | MergeFn | Function to customize value transformations during mapping |
582594
| `retry` | RetryOptions | Options for retry logic |
583595
| `useBuffering` | boolean | Use buffering for large files |
584596
| `bufferSize` | number | Size of buffer when useBuffering is true |

‎src/headers.ts‎

Lines changed: 117 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@ export type HeaderMap<T = any> = {
1414
[K in string | number]: keyof T | string
1515
};
1616

17+
/**
18+
* Type for the merge function that transforms values during mapping
19+
* @template T - The type of the target object
20+
*/
21+
export type MergeFn<T> = (obj: Partial<T>, key: string, value: any) => any;
22+
1723
/**
1824
* Options for retry logic
1925
*/
@@ -30,6 +36,7 @@ export interface RetryOptions {
3036
* Creates functions to map between row arrays and structured objects
3137
* @template T - The type of the target object
3238
* @param headerMap - Mapping between array indices or header names and object properties
39+
* @param mergeFn - Optional function to customize how values are merged into the target object
3340
* @returns Object containing mapping functions
3441
* @example
3542
* ```typescript
@@ -44,11 +51,16 @@ export interface RetryOptions {
4451
* 'last_name': 'profile.lastName'
4552
* };
4653
*
47-
* const { fromRowArr, toRowArr } = createHeaderMapFns<User>(headerMap);
54+
* // With custom merge function to trim strings
55+
* const { fromRowArr, toRowArr } = createHeaderMapFns<User>(
56+
* headerMap,
57+
* (obj, key, value) => typeof value === 'string' ? value.trim() : value
58+
* );
4859
* ```
4960
*/
5061
export function createHeaderMapFns<To extends Record<string, any>, RowArr extends any[] = any[]>(
51-
headerMap: { [K in number | string]: keyof To | string }
62+
headerMap: HeaderMap<To>,
63+
mergeFn?: MergeFn<To>
5264
) {
5365
// Validate the header map
5466
const validateHeaderMap = () => {
@@ -84,15 +96,19 @@ export function createHeaderMapFns<To extends Record<string, any>, RowArr extend
8496
for (let i = 0; i < rowArr.length; i++) {
8597
const toKey = headerMap[i];
8698
if (toKey) {
87-
setPath(to, toKey as string, rowArr[i]);
99+
const targetPath = String(toKey); // Ensure it's a string
100+
const value = mergeFn ? mergeFn(to, targetPath, rowArr[i]) : rowArr[i];
101+
setPath(to, targetPath, value);
88102
}
89103
}
90104
} else if (typeof rowArr === 'object' && rowArr !== null) {
91105
// Handle object input
92106
for (let [key, value] of Object.entries(rowArr)) {
93-
const toKey = headerMap?.[key];
107+
const toKey = headerMap[key];
94108
if (toKey) {
95-
setPath(to, toKey as string, value);
109+
const targetPath = String(toKey); // Ensure it's a string
110+
const processedValue = mergeFn ? mergeFn(to, targetPath, value) : value;
111+
setPath(to, targetPath, processedValue);
96112
}
97113
}
98114
} else {
@@ -106,15 +122,25 @@ export function createHeaderMapFns<To extends Record<string, any>, RowArr extend
106122
* Convert a structured object back to a row array
107123
* @param objAfterMapWasApplied - Structured object
108124
* @param headers - Array of header names in order (required for header-based mapping)
125+
* @param transformFn - Optional function to transform values when converting from object to row
109126
* @returns Row data as an array
110127
* @example
111128
* ```typescript
112129
* const user = { id: '123', profile: { firstName: 'John', lastName: 'Doe' } };
113130
* const row = toRowArr(user, ['user_id', 'first_name', 'last_name']);
114131
* // row = ['123', 'John', 'Doe']
132+
*
133+
* // With transform function
134+
* const csvRow = toRowArr(user, ['user_id', 'first_name', 'last_name'],
135+
* (value) => typeof value === 'string' ? value.toUpperCase() : value);
136+
* // csvRow = ['123', 'JOHN', 'DOE']
115137
* ```
116138
*/
117-
toRowArr: (objAfterMapWasApplied: To, headers: string[] = []): RowArr => {
139+
toRowArr: (
140+
objAfterMapWasApplied: To,
141+
headers: string[] = [],
142+
transformFn?: (value: any, key: string) => any
143+
): RowArr => {
118144
// Validate input
119145
if (!objAfterMapWasApplied || typeof objAfterMapWasApplied !== 'object') {
120146
throw new CSVError('Object must be a non-null object');
@@ -126,13 +152,12 @@ export function createHeaderMapFns<To extends Record<string, any>, RowArr extend
126152
if (isIndexBased) {
127153
// Index-based mapping
128154
for (let [rowIdx, path] of Object.entries(headerMap)) {
129-
let value = getPath(objAfterMapWasApplied, path as string);
155+
const targetPath = String(path);
156+
let value = getPath(objAfterMapWasApplied, targetPath);
157+
130158
if (typeof value !== 'undefined') {
131-
// Handle special types
132-
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
133-
// @ts-expect-error
134-
value = JSON.stringify(value);
135-
}
159+
// Process value
160+
value = processValueForOutput(value, targetPath, transformFn);
136161
row[parseInt(rowIdx)] = value;
137162
}
138163
}
@@ -145,14 +170,14 @@ export function createHeaderMapFns<To extends Record<string, any>, RowArr extend
145170
for (let i = 0; i < headers.length; i++) {
146171
const headerName = headers[i];
147172
const path = headerMap[headerName];
173+
148174
if (path) {
149-
let value = getPath(objAfterMapWasApplied, path as string);
175+
const targetPath = String(path);
176+
let value = getPath(objAfterMapWasApplied, targetPath);
177+
150178
if (typeof value !== 'undefined') {
151-
// Handle special types
152-
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
153-
// @ts-expect-error
154-
value = JSON.stringify(value);
155-
}
179+
// Process value
180+
value = processValueForOutput(value, targetPath, transformFn);
156181
row[i] = value;
157182
}
158183
}
@@ -164,6 +189,45 @@ export function createHeaderMapFns<To extends Record<string, any>, RowArr extend
164189
};
165190
}
166191

192+
/**
193+
* Helper function to process values for output, handling special cases
194+
* @param value - The value to process
195+
* @param key - The key or path associated with the value
196+
* @param transformFn - Optional function to transform the value
197+
* @returns Processed value
198+
*/
199+
function processValueForOutput(
200+
value: any,
201+
key: string,
202+
transformFn?: (value: any, key: string) => any
203+
): any {
204+
// Apply custom transformation if provided
205+
if (transformFn) {
206+
return transformFn(value, key);
207+
}
208+
209+
// Handle special types
210+
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
211+
return JSON.stringify(value);
212+
}
213+
214+
return value;
215+
}
216+
217+
/**
218+
* Helper function to convert array row to object row using headers
219+
* @param row - Array of values
220+
* @param headerRow - Array of header names
221+
* @returns Object with header names as keys
222+
*/
223+
function arrayRowToObjectRow(row: any[], headerRow: string[]): Record<string, any> {
224+
const objRow: Record<string, any> = {};
225+
for (let i = 0; i < row.length && i < headerRow.length; i++) {
226+
objRow[headerRow[i]] = row[i];
227+
}
228+
return objRow;
229+
}
230+
167231
/**
168232
* Transforms an array of arrays or objects into an array of structured objects
169233
* @template T - The type of the target object
@@ -190,7 +254,7 @@ export function createHeaderMapFns<To extends Record<string, any>, RowArr extend
190254
* { 0: 'id', 1: 'details.name', 2: 'details.price' }
191255
* );
192256
*
193-
* // With custom merge function to convert price to number
257+
* // With custom merge function to convert price to number
194258
* const productsWithPriceAsNumber = arrayToObjArray<Product>(
195259
* csvData.slice(1),
196260
* { 0: 'id', 1: 'details.name', 2: 'details.price' },
@@ -208,7 +272,7 @@ export function arrayToObjArray<T extends Record<string, any>>(
208272
data: any[],
209273
headerMap: HeaderMap<T>,
210274
headerRow?: string[],
211-
mergeFn?: (obj: Partial<T>, key: string, value: any) => any
275+
mergeFn?: MergeFn<T>
212276
): T[] {
213277
if (!Array.isArray(data)) {
214278
throw new CSVError('Data must be an array');
@@ -218,68 +282,53 @@ export function arrayToObjArray<T extends Record<string, any>>(
218282
return [];
219283
}
220284

221-
const { fromRowArr } = createHeaderMapFns<T>(headerMap);
285+
const { fromRowArr } = createHeaderMapFns<T>(headerMap, mergeFn);
222286

223-
// If first item is an array but keys are strings, we need header row
224-
const firstItem = data[0];
225-
const isArrayData = Array.isArray(firstItem);
226-
const hasStringKeys = Object.keys(headerMap).some(k => isNaN(Number(k)));
287+
// Check if we need to validate header row
288+
validateHeadersIfNeeded(data, headerMap, headerRow);
227289

228-
if (isArrayData && hasStringKeys && !headerRow) {
229-
throw new CSVError('Header row is required for string-keyed header map with array data');
230-
}
231-
232-
if (mergeFn) {
233-
return data.map(row => {
234-
// Convert row to an object if working with arrays and string header maps
235-
let objRow: Record<string, any> = {};
236-
if (isArrayData && hasStringKeys && headerRow) {
237-
for (let i = 0; i < row.length && i < headerRow.length; i++) {
238-
objRow[headerRow[i]] = row[i];
239-
}
240-
} else if (isArrayData) {
241-
// For array data with numeric indices
242-
objRow = [...row];
243-
} else {
244-
// For object data
245-
objRow = {...row};
246-
}
247-
248-
// Apply mappings with custom merge function
249-
const result = {} as T;
250-
for (const [sourceKey, targetPath] of Object.entries(headerMap)) {
251-
const key = isArrayData && !hasStringKeys ? parseInt(sourceKey) : sourceKey;
252-
const value = isArrayData ? row[key as number] : objRow[key as string];
253-
if (value !== undefined) {
254-
const processedValue = mergeFn(result, targetPath as string, value);
255-
setPath(result, targetPath as string, processedValue);
256-
}
257-
}
258-
return result;
259-
});
260-
}
261-
262290
return data.map(row => {
263291
// If working with arrays and string header maps, convert to object first
292+
const isArrayData = Array.isArray(row);
293+
const hasStringKeys = Object.keys(headerMap).some(k => isNaN(Number(k)));
294+
264295
if (isArrayData && hasStringKeys && headerRow) {
265-
const objRow: Record<string, any> = {};
266-
for (let i = 0; i < row.length && i < headerRow.length; i++) {
267-
objRow[headerRow[i]] = row[i];
268-
}
296+
const objRow = arrayRowToObjectRow(row, headerRow);
269297
return fromRowArr(objRow);
270298
}
271299

272300
return fromRowArr(row);
273301
});
274302
}
275303

304+
/**
305+
* Validates that header row is provided when needed
306+
* @param data - The data array
307+
* @param headerMap - The header mapping configuration
308+
* @param headerRow - The header row (optional)
309+
*/
310+
function validateHeadersIfNeeded<T>(
311+
data: any[],
312+
headerMap: HeaderMap<T>,
313+
headerRow?: string[]
314+
): void {
315+
const firstItem = data[0];
316+
const isArrayData = Array.isArray(firstItem);
317+
const hasStringKeys = Object.keys(headerMap).some(k => isNaN(Number(k)));
318+
319+
if (isArrayData && hasStringKeys && !headerRow) {
320+
throw new CSVError('Header row is required for string-keyed header map with array data');
321+
}
322+
}
323+
276324
/**
277325
* Transforms an array of structured objects into an array of arrays
278326
* @template T - The type of the source object
279327
* @param data - Array of structured objects to transform
280328
* @param headerMap - Mapping between object properties and array indices or header names
281329
* @param headers - Optional array of headers (required for header-based mapping)
282330
* @param includeHeaders - Whether to include headers as the first row
331+
* @param transformFn - Optional function to transform values when converting to rows
283332
* @returns Array of arrays
284333
* @example
285334
* ```typescript
@@ -305,7 +354,8 @@ export function objArrayToArray<T extends Record<string, any>>(
305354
data: T[],
306355
headerMap: HeaderMap,
307356
headers: string[] = [],
308-
includeHeaders: boolean = false
357+
includeHeaders: boolean = false,
358+
transformFn?: (value: any, key: string) => any
309359
): any[][] {
310360
if (!Array.isArray(data)) {
311361
throw new CSVError('Data must be an array');
@@ -318,14 +368,14 @@ export function objArrayToArray<T extends Record<string, any>>(
318368
// Create an inverse header map
319369
const inverseMap: HeaderMap<T> = {};
320370
for (const [key, value] of Object.entries(headerMap)) {
321-
if (typeof value === 'string') {
371+
if (typeof value === 'string' || typeof value === 'number') {
322372
inverseMap[value] = key;
323373
}
324374
}
325375

326376
const { toRowArr } = createHeaderMapFns<T>(inverseMap);
327377

328-
const rows = data.map(obj => toRowArr(obj, headers));
378+
const rows = data.map(obj => toRowArr(obj, headers, transformFn));
329379

330380
if (includeHeaders && headers.length > 0) {
331381
return [headers, ...rows];
@@ -366,7 +416,7 @@ export function groupByField<T extends Record<string, any>>(
366416
field: string
367417
): Record<string, T[]> {
368418
return data.reduce((groups, item) => {
369-
const key = String(getPath(item, field) || 'undefined');
419+
const key = String(getPath(item, field) ?? 'undefined');
370420
if (!groups[key]) {
371421
groups[key] = [];
372422
}

0 commit comments

Comments
 (0)