Apply unescape on all fields

Signed-off-by: Dushyant Behl <dushyantbehl@in.ibm.com>

Author: dushyantbehl

tuning/config/configs.py

@@ -131,6 +131,17 @@ class DataArguments:
         },
     )
 
+    def __post_init__(self):
+        def unescape(s):
+            if s is not None and isinstance(s, str):
+                return s.encode("utf-8").decode("unicode_escape")
+            return s
+
+        self.chat_template = unescape(self.chat_template)
+        self.data_formatter_template = unescape(self.data_formatter_template)
+        self.response_template = unescape(self.response_template)
+        self.instruction_template = unescape(self.instruction_template)
+
 
 @dataclass
 class TrainingArguments(transformers.TrainingArguments):

tuning/data/data_preprocessing_utils.py

@@ -69,7 +69,6 @@ def get_data_collator(
         # otherwise template is not found. We will create issue to clean this out after we discuss
         # data formats and collators we will support.
         if response_template:
-            response_template = response_template.encode('utf-8').decode('unicode_escape')
             response_template_ids = tokenizer.encode(
                 response_template, add_special_tokens=False
             )[2:]

tuning/data/setup_dataprocessor.py

@@ -173,7 +173,7 @@ def _get_dataset_formatting_handlers(data_args, packing, is_padding_free=False):
             arguments={"fn_kwargs": fn_kwargs, "batched": False},
         )
     else:
-        fn_kwargs["template"] = data_args.data_formatter_template.encode('utf-8').decode('unicode_escape')
+        fn_kwargs["template"] = data_args.data_formatter_template
         handler = DataHandlerConfig(
             "apply_custom_data_formatting_template",
             arguments={"fn_kwargs": fn_kwargs, "batched": False},