models/BertTunning.py

import torch
from torch import nn
from models.preprocess_stage.bert_model import model
from models.preprocess_stage.bert_model import preprocess_bert

MAX_LEN = 100
# DEVICE='cpu'

class BertTunnig(nn.Module):
    def __init__(self, bert_model):
        super().__init__()

        self.bert = bert_model
        for weights in self.bert.parameters():
            weights.requires_grad = False

        self.fc1 = nn.Linear(768, 256)
        self.drop1 = nn.Dropout(p=0.5)
        self.fc2 = nn.Linear(256, 32)
        self.fc_out = nn.Linear(32, 1)

    def forward(self, x, attention_mask):

        output = self.bert(x, attention_mask=attention_mask)[0][:, 0, :]
        output = self.fc1(output)

        output_drop = self.drop1(output)
        output = self.fc2(output_drop)

        output = self.fc_out(output)

        return torch.sigmoid(output)


model_tunning = BertTunnig(bert_model=model)
model_tunning.load_state_dict(torch.load('models/weights/BertTunnigWeights.pt', map_location=torch.device('cpu')))


def predict_2(text):

    preprocessed_text, attention_mask = preprocess_bert(text, MAX_LEN=MAX_LEN)
    preprocessed_text, attention_mask = torch.tensor(preprocessed_text).unsqueeze(0), torch.tensor([attention_mask])

    # model_tunning.to(DEVICE)
    with torch.inference_mode():

        predict = round(model_tunning(preprocessed_text, attention_mask=attention_mask).item())

    return predict