Skip to content

Data Exchange Definition export performance improvement #9073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
VolodySyn wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Data Exchange Definition export performance improvement #9073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dfd4c0b
Data Exchange Definition export performance improvement
VolodySyn Jul 3, 2026
29b983a
Merge branch 'main' of https://github.com/microsoft/BCApps into bugs/…
VolodySyn Jul 3, 2026
238ee27
Miapp
VolodySyn Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 101 additions & 0 deletions src/Layers/RU/BaseApp/Bank/Payment/PaymentExportMgt.Codeunit.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,107 @@ codeunit 1210 "Payment Export Mgt"
end;
end;

/// <summary>
/// Processes column mapping for payment export data exchange.
/// </summary>
/// <param name="DataExch">Data exchange record</param>
/// <param name="RecRef">Record reference containing source data</param>
/// <param name="DataExchFlowFieldGrBuff">FlowField grouping buffer for aggregated fields</param>
/// <param name="LineNo">Line number in the data exchange</param>
/// <param name="DataExchLineDefCode">Data exchange line definition code</param>
/// <param name="TableID">Table ID of the source record</param>
/// <param name="TempDataExchField">Temporary record for data exchange fields</param>
procedure ProcessColumnMapping(var DataExch: Record "Data Exch."; RecRef: RecordRef; var DataExchFlowFieldGrBuff: Record "Data Exch. FlowField Gr. Buff."; LineNo: Integer; DataExchLineDefCode: Code[20]; TableID: Integer; var TempDataExchField: Record "Data Exch. Field" temporary)
var
DataExchDef: Record "Data Exch. Def";
DataExchColumnDef: Record "Data Exch. Column Def";
DataExchFieldMapping: Record "Data Exch. Field Mapping";
TransformationRule: Record "Transformation Rule";
StringConversionManagement: Codeunit StringConversionManagement;
FieldRef: FieldRef;
ValueAsDestType: Variant;
ValueAsString: Text[250];
begin
if not DataExchDef.Get(DataExch."Data Exch. Def Code") then
Error(FormatNotDefinedErr, DataExch."Data Exch. Def Code");

PrepopulateColumns(DataExchDef, DataExchLineDefCode, DataExch."Entry No.", LineNo, TempDataExchField);

DataExchFieldMapping.SetRange("Data Exch. Def Code", DataExchDef.Code);
DataExchFieldMapping.SetRange("Data Exch. Line Def Code", DataExchLineDefCode);
DataExchFieldMapping.SetRange("Table ID", TableID);
DataExchFieldMapping.FindSet();

repeat
DataExchColumnDef.Get(DataExchDef.Code, DataExchLineDefCode, DataExchFieldMapping."Column No.");

if DataExchFieldMapping."Use Default Value" then
ValueAsString := DataExchFieldMapping."Default Value"
else begin
FieldRef := RecRef.Field(DataExchFieldMapping."Field ID");
if FieldRef.Class = FieldRef.Class::FlowField then
if DataExchFlowFieldGrBuff.Get(RecRef.RecordId, FieldRef.Number) then
FieldRef.Value := DataExchFlowFieldGrBuff.Value
else
FieldRef.CalcField();

CheckOptional(DataExchFieldMapping.Optional, FieldRef);
CastToDestinationType(ValueAsDestType, FieldRef.Value, DataExchColumnDef, DataExchFieldMapping.Multiplier);
ValueAsString := FormatToText(ValueAsDestType, DataExchDef, DataExchColumnDef);

if TransformationRule.Get(DataExchFieldMapping."Transformation Rule") then
ValueAsString := CopyStr(TransformationRule.TransformText(ValueAsString), 1, 250);

if DataExchColumnDef."Text Padding Required" and (DataExchColumnDef."Pad Character" <> '') and (not DataExchColumnDef."Blank Zero") then
ValueAsString :=
StringConversionManagement.GetPaddedString(
ValueAsString,
DataExchColumnDef.Length,
DataExchColumnDef."Pad Character",
DataExchColumnDef.Justification);
end;

OnProcessColumnMappingOnBeforeCheckLength(ValueAsString, DataExchFieldMapping, DataExchColumnDef);
CheckLength(ValueAsString, RecRef.Field(DataExchFieldMapping."Field ID"), DataExchDef, DataExchColumnDef);

TempDataExchField.Get(DataExch."Entry No.", LineNo, DataExchFieldMapping."Column No.");
TempDataExchField.SetValueWithoutModifying(ValueAsString);
TempDataExchField.Modify();
until DataExchFieldMapping.Next() = 0;
end;

local procedure PrepopulateColumns(DataExchDef: Record "Data Exch. Def"; DataExchLineDefCode: Code[20]; DataExchEntryNo: Integer; DataExchLineNo: Integer; var TempDataExchField: Record "Data Exch. Field" temporary)
var
DataExchLineDef: Record "Data Exch. Line Def";
DataExchColumnDef: Record "Data Exch. Column Def";
ColumnIndex: Integer;
begin
if DataExchDef."File Type" in
[DataExchDef."File Type"::"Fixed Text",
DataExchDef."File Type"::Xml,
DataExchDef."File Type"::Json]
then begin
DataExchColumnDef.SetRange("Data Exch. Def Code", DataExchDef.Code);
DataExchColumnDef.SetRange("Data Exch. Line Def Code", DataExchLineDefCode);
if not DataExchColumnDef.FindSet() then
Error(DataExchLineDefNotFoundErr, DataExchDef.Name, DataExchLineDefCode);
repeat
TempDataExchField.InsertRec(
DataExchEntryNo, DataExchLineNo, DataExchColumnDef."Column No.",
PadStr(DataExchColumnDef.Constant, DataExchColumnDef.Length), DataExchLineDefCode)
until DataExchColumnDef.Next() = 0;
end else begin
if not DataExchLineDef.Get(DataExchDef.Code, DataExchLineDefCode) then
Error(DataExchLineDefNotFoundErr, DataExchDef.Name, DataExchLineDefCode);
for ColumnIndex := 1 to DataExchLineDef."Column Count" do
if DataExchColumnDef.Get(DataExchDef.Code, DataExchLineDef.Code, ColumnIndex) then
TempDataExchField.InsertRec(
DataExchEntryNo, DataExchLineNo, ColumnIndex, DataExchColumnDef.Constant, DataExchLineDefCode)
else
TempDataExchField.InsertRec(DataExchEntryNo, DataExchLineNo, ColumnIndex, '', DataExchLineDefCode);
end;
end;

local procedure CheckOptional(Optional: Boolean; FieldRef: FieldRef)
var
Value: Variant;
Expand Down
101 changes: 101 additions & 0 deletions src/Layers/W1/BaseApp/Bank/Payment/PaymentExportMgt.Codeunit.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,107 @@ codeunit 1210 "Payment Export Mgt"
end;
end;

/// <summary>
/// Processes column mapping for payment export data exchange.
/// </summary>
/// <param name="DataExch">Data exchange record</param>
/// <param name="RecRef">Record reference containing source data</param>
/// <param name="DataExchFlowFieldGrBuff">FlowField grouping buffer for aggregated fields</param>
/// <param name="LineNo">Line number in the data exchange</param>
/// <param name="DataExchLineDefCode">Data exchange line definition code</param>
/// <param name="TableID">Table ID of the source record</param>
/// <param name="TempDataExchField">Temporary record for data exchange fields</param>
procedure ProcessColumnMapping(var DataExch: Record "Data Exch."; RecRef: RecordRef; var DataExchFlowFieldGrBuff: Record "Data Exch. FlowField Gr. Buff."; LineNo: Integer; DataExchLineDefCode: Code[20]; TableID: Integer; var TempDataExchField: Record "Data Exch. Field" temporary)
var
DataExchDef: Record "Data Exch. Def";
DataExchColumnDef: Record "Data Exch. Column Def";
DataExchFieldMapping: Record "Data Exch. Field Mapping";
TransformationRule: Record "Transformation Rule";
StringConversionManagement: Codeunit StringConversionManagement;
FieldRef: FieldRef;
ValueAsDestType: Variant;
ValueAsString: Text[250];
begin
if not DataExchDef.Get(DataExch."Data Exch. Def Code") then
Error(FormatNotDefinedErr, DataExch."Data Exch. Def Code");

PrepopulateColumns(DataExchDef, DataExchLineDefCode, DataExch."Entry No.", LineNo, TempDataExchField);

DataExchFieldMapping.SetRange("Data Exch. Def Code", DataExchDef.Code);
DataExchFieldMapping.SetRange("Data Exch. Line Def Code", DataExchLineDefCode);
DataExchFieldMapping.SetRange("Table ID", TableID);
DataExchFieldMapping.FindSet();

repeat
DataExchColumnDef.Get(DataExchDef.Code, DataExchLineDefCode, DataExchFieldMapping."Column No.");

if DataExchFieldMapping."Use Default Value" then
ValueAsString := DataExchFieldMapping."Default Value"
else begin
FieldRef := RecRef.Field(DataExchFieldMapping."Field ID");
if FieldRef.Class = FieldRef.Class::FlowField then
if DataExchFlowFieldGrBuff.Get(RecRef.RecordId, FieldRef.Number) then
FieldRef.Value := DataExchFlowFieldGrBuff.Value
else
FieldRef.CalcField();

CheckOptional(DataExchFieldMapping.Optional, FieldRef);
CastToDestinationType(ValueAsDestType, FieldRef.Value, DataExchColumnDef, DataExchFieldMapping.Multiplier);
ValueAsString := FormatToText(ValueAsDestType, DataExchDef, DataExchColumnDef);

if TransformationRule.Get(DataExchFieldMapping."Transformation Rule") then
ValueAsString := CopyStr(TransformationRule.TransformText(ValueAsString), 1, 250);

if DataExchColumnDef."Text Padding Required" and (DataExchColumnDef."Pad Character" <> '') and (not DataExchColumnDef."Blank Zero") then
ValueAsString :=
StringConversionManagement.GetPaddedString(
ValueAsString,
DataExchColumnDef.Length,
DataExchColumnDef."Pad Character",
DataExchColumnDef.Justification);
end;

OnProcessColumnMappingOnBeforeCheckLength(ValueAsString, DataExchFieldMapping, DataExchColumnDef);
CheckLength(ValueAsString, RecRef.Field(DataExchFieldMapping."Field ID"), DataExchDef, DataExchColumnDef);

TempDataExchField.Get(DataExch."Entry No.", LineNo, DataExchFieldMapping."Column No.");
TempDataExchField.SetValueWithoutModifying(ValueAsString);
TempDataExchField.Modify();
until DataExchFieldMapping.Next() = 0;
end;

local procedure PrepopulateColumns(DataExchDef: Record "Data Exch. Def"; DataExchLineDefCode: Code[20]; DataExchEntryNo: Integer; DataExchLineNo: Integer; var TempDataExchField: Record "Data Exch. Field" temporary)
var
DataExchLineDef: Record "Data Exch. Line Def";
DataExchColumnDef: Record "Data Exch. Column Def";
ColumnIndex: Integer;
begin
if DataExchDef."File Type" in
[DataExchDef."File Type"::"Fixed Text",
DataExchDef."File Type"::Xml,
DataExchDef."File Type"::Json]
then begin
DataExchColumnDef.SetRange("Data Exch. Def Code", DataExchDef.Code);
DataExchColumnDef.SetRange("Data Exch. Line Def Code", DataExchLineDefCode);
if not DataExchColumnDef.FindSet() then
Error(DataExchLineDefNotFoundErr, DataExchDef.Name, DataExchLineDefCode);
repeat
TempDataExchField.InsertRec(
DataExchEntryNo, DataExchLineNo, DataExchColumnDef."Column No.",
PadStr(DataExchColumnDef.Constant, DataExchColumnDef.Length), DataExchLineDefCode)
until DataExchColumnDef.Next() = 0;
end else begin
if not DataExchLineDef.Get(DataExchDef.Code, DataExchLineDefCode) then
Error(DataExchLineDefNotFoundErr, DataExchDef.Name, DataExchLineDefCode);
for ColumnIndex := 1 to DataExchLineDef."Column Count" do
if DataExchColumnDef.Get(DataExchDef.Code, DataExchLineDef.Code, ColumnIndex) then
TempDataExchField.InsertRec(
DataExchEntryNo, DataExchLineNo, ColumnIndex, DataExchColumnDef.Constant, DataExchLineDefCode)
else
TempDataExchField.InsertRec(DataExchEntryNo, DataExchLineNo, ColumnIndex, '', DataExchLineDefCode);
end;
end;

local procedure CheckOptional(Optional: Boolean; FieldRef: FieldRef)
var
Value: Variant;
Expand Down
44 changes: 40 additions & 4 deletions src/Layers/W1/BaseApp/System/DataExchange/ExportMapping.Codeunit.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,28 +10,64 @@ codeunit 1269 "Export Mapping"

trigger OnRun()
var
DataExchField: Record "Data Exch. Field";
DataExchMapping: Record "Data Exch. Mapping";
TempDataExchField: Record "Data Exch. Field" temporary;
TempDataExchFlowFieldGrBuff: Record "Data Exch. FlowField Gr. Buff." temporary;
PaymentExportMgt: Codeunit "Payment Export Mgt";
SourceRecRef: RecordRef;
Window: Dialog;
LineNo: Integer;
RowCount: Integer;
Divider: Integer;
begin
FindMapping(DataExchMapping, Rec."Data Exch. Def Code", Rec."Data Exch. Line Def Code");
GetSourceRecRefBuffer(SourceRecRef, TempDataExchFlowFieldGrBuff, DataExchMapping, ReadTableFilters(Rec));
Window.Open(ProgressMsg);
RowCount := SourceRecRef.Count();
case true of
RowCount > 1000:
Divider := 200;
RowCount > 100:
Divider := 10;
else
Divider := 1;
end;

Window.Open(MappingProgressMsg);
repeat
LineNo += 1;
Window.Update(1, LineNo);
PaymentExportMgt.ProcessColumnMapping(Rec, SourceRecRef, TempDataExchFlowFieldGrBuff, LineNo, Rec."Data Exch. Line Def Code", SourceRecRef.Number);
if LineNo mod Divider = 0 then
Window.Update(1, LineNo);

PaymentExportMgt.ProcessColumnMapping(Rec, SourceRecRef, TempDataExchFlowFieldGrBuff, LineNo, Rec."Data Exch. Line Def Code", SourceRecRef.Number, TempDataExchField);
until SourceRecRef.Next() = 0;
Window.Close();

Window.Open(SavingProgressMsg);
TempDataExchField.Reset();
if TempDataExchField.FindSet() then
repeat
if TempDataExchField."Line No." <> LineNo then begin
LineNo := TempDataExchField."Line No.";
if LineNo mod Divider = 0 then
Window.Update(1, LineNo);
end;

DataExchField.Init();
DataExchField.TransferFields(TempDataExchField);
if TempDataExchField."Value BLOB".HasValue() then begin
TempDataExchField.CalcFields("Value BLOB");
DataExchField."Value BLOB" := TempDataExchField."Value BLOB";
end;
DataExchField.Insert();
until TempDataExchField.Next() = 0;
Window.Close();
end;

var
#pragma warning disable AA0470
ProgressMsg: Label 'Processing line no. #1######.';
MappingProgressMsg: Label 'Mapping line no. #1######';
SavingProgressMsg: Label 'Saving line no. #1######';
MappingNotFoundErr: Label 'No mapping was found for format definition %1 and line definition %2.';
RecordsNotFoundErr: Label 'No records were found for source table %1 using the filters: %2.';
#pragma warning restore AA0470
Expand Down
Loading