Use ROBERTa

[khatu2002]

Can i base on bert_spc to do ROBERTa model?
this is roberta model
import torch
import torch.nn as nn
class RoBERTa(nn.Module):
def init(self, roberta, opt):
super(RoBERTa, self).init()
self.roberta = roberta
self.dropout = nn.Dropout(opt.dropout)
self.dense = nn.Linear(opt.bert_dim, opt.polarities_dim)

def forward(self, inputs):
    concat_roberta_indices = inputs[0]
    text_roberta_indices = inputs[1]
    aspect_roberta_indices = inputs[2]

    text_outputs = self.roberta(input_ids=text_roberta_indices, attention_mask=(text_roberta_indices != 0).long())
    
    concat_outputs = self.roberta(input_ids=concat_roberta_indices, attention_mask=(concat_roberta_indices != 0).long())

    aspect_outputs = self.roberta(input_ids=aspect_roberta_indices, attention_mask=(aspect_roberta_indices != 0).long())

    text_pooled_output = text_outputs.pooler_output
    concat_pooled_output = concat_outputs.pooler_output
    aspect_pooled_output = aspect_outputs.pooler_output

    combined_output = torch.cat((text_pooled_output, concat_pooled_output, aspect_pooled_output), dim=-1)

    combined_output = self.dropout(combined_output)

    logits = self.dense(combined_output)
    
    return logits