pt_text_classification.py

# -*- coding: utf-8 -*-
"""pt-text-classification.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1QwXWUoFH8Rg4t5y9oqi34wrpxRiAHTY2
"""

from huggingface_hub import login
login()

!pip install  sentencepiece datasets peft  evaluate  -U accelerate -U transformers

from datasets import  load_dataset, Dataset, DatasetDict, concatenate_datasets

from transformers import (
    AutoTokenizer,
    AutoConfig,
    AutoModelForSequenceClassification,
    DataCollatorWithPadding,
    TrainingArguments,
    Trainer
)

from peft import PeftModel, PeftConfig, get_peft_model, LoraConfig
import evaluate
import torch
import numpy as np
import pandas as pd

data = pd.read_json("/kaggle/input/reviews/reviews.json")
data.text = data.text.astype(str)
data = data.sample(frac=1).reset_index(drop=True) ## 73665 rows


fifty = int(len(data) * 0.7)
data = data[:fifty]


id2c = {0: 'neg', 1:'neutral', 2: 'pos'}
c2id = {"neg": 0, "neutral": 1, "pos": 2}
data['sentiment'].unique()

def get_features(row):
    y = 0 if row['sentiment'] == 'neg' else 1 if row['sentiment'] == 'neutral' else 2
    return {'text': row['text'], 'label': y}


size = int(len(data) * 0.2)
treino = data[size:]
teste = data[:size]


train_dict_list = treino.apply(get_features, axis=1).tolist()

trainDataset = Dataset.from_dict({"text": [item['text'] for item in train_dict_list], "label": [item['label'] for item in train_dict_list]})

validation_dict_list = teste.apply(get_features, axis=1).tolist()

validationDataset = Dataset.from_dict({"text": [item['text'] for item in validation_dict_list], "label": [item['label'] for item in validation_dict_list]})

dataset = DatasetDict({"train": trainDataset, "validation": validationDataset})
dataset

model_repo = "distilbert/distilbert-base-multilingual-cased"

model = AutoModelForSequenceClassification.from_pretrained(model_repo, num_labels=3, id2label=id2c, label2id=c2id)
tokenizer = AutoTokenizer.from_pretrained(model_repo, add_prefix_space=True)

def tokenize_items(examples):
  text = examples["text"]

  tokenizer.truncation_side = "left"
  tokenized_inputs = tokenizer(
      text,
      return_tensors="pt",
      truncation=True,
      padding=True,
      max_length=512
  )

  # Convertendo tensores para matrizes numpy
  numpy_tokenized_inputs = {key: value.numpy() for key, value in tokenized_inputs.items()}

  return numpy_tokenized_inputs

if tokenizer.pad_token is None:
  tokenizer.add_special_tokens({'pad_token': '[PAD]'})
  model.resize_token_embeddings(len(tokenizer))
tokenized_dataset = dataset.map(tokenize_items, batched=True)
tokenized_dataset

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

from sklearn.metrics import f1_score
accuracy = evaluate.load('accuracy')

def compute_metrics(p):
  predictions, labels = p
  predictions = np.argmax(predictions, axis=1)
  f1 = f1_score(predictions, labels, average='macro')

  return {"accuracy": accuracy.compute(predictions=predictions, references=labels), "f1_score": f1}

## Untrained model predictions
text_list = ["Grande erro da rede globo hoje!!", "Gostei muito do jogo do corinthians hoje", "Que prova dificil"]


for text in text_list:
  inputs = tokenizer.encode(text, return_tensors="pt")
  logits = model(inputs).logits


  predictions = torch.argmax(logits)
  print(text + " - " + id2c[predictions.tolist()])

peft_config = LoraConfig(
                task_type="SEQ_CLS", ##sequence classification,
                r=4, ##intrinsic rank of trainable weight matrix
                lora_alpha=32, ##this is like learning rate
                lora_dropout=0.01, ## probality of dropout
                target_modules = ['q_lin'] ##we apply lora to query layer
              )

model = get_peft_model(model, peft_config)
model.print_trainable_parameters()

#hyperparameters
lr =  1e-3
batch_size = 4
num_epochs = 5

training_args = TrainingArguments(
    output_dir = model_repo + "-lora-multilingual-text-classification",
    learning_rate = lr,
    per_device_train_batch_size = batch_size,
    per_device_eval_batch_size = batch_size,
    num_train_epochs = num_epochs,
    weight_decay = 0.01,
    evaluation_strategy = "epoch",
    save_strategy="epoch",
    load_best_model_at_end=True
)

trainer = Trainer(
    model = model,
    args=training_args,
    train_dataset=tokenized_dataset["train"],
    eval_dataset=tokenized_dataset["validation"],
    tokenizer=tokenizer,
    data_collator=data_collator, ##this will dynamically pad examples in each batch
    compute_metrics=compute_metrics
)


trainer.train()

## Trained model predictionsd
text_list = ["Grande erro da rede globo hoje", "Gostei muito do jogo do corinthians hoje", "Que prova dificil"]

model.to("cpu")

for text in text_list:
  inputs1 = tokenizer.encode(text, return_tensors="pt")
  logits1 = model(inputs1).logits


  predictions1 = torch.argmax(logits1)
  str = text + " - " + id2c[predictions1.tolist()]
  str

inputs = tokenizer.encode("Nao gostei desse produto, pessima qualidade", return_tensors="pt")
logits = model(inputs).logits


predictions = torch.argmax(logits)
id2c[predictions.tolist()]

model.save_pretrained("pt-trained-model")

peft_model = "igoramf/lora-pt-sentiment-analysis"

model.push_to_hub(
    peft_model, use_auth_token=True
)