Deep&Cross network (DCN)

paper: https://arxiv.org/pdf/1708.05123.pdf

Implementation of DCN

import torch
from torch import nn
class CrossNet(nn.Module):
  def __init__(self, fea_dim = 5,embed_dim = 5, layer_num=3):
    super(CrossNet, self).__init__()
    self.weight = nn.Parameter(torch.randn(layer_num, fea_dim))
    self.bias = nn.Parameter(torch.randn(layer_num, fea_dim))
    self.layer_num = layer_num
  def forward(self, x):
    #
    #x: dense input  after concating embedding
    in_fea = x[:,:]
    # print(x.shape)
    for j in range(len(x)):
      x0 = x[j,:]
      for i in range(self.layer_num):
        in_fea[j,:] = x0 * torch.dot(in_fea[j,:],self.weight[i,:])  + self.bias[i,:] + in_fea[j,:]
        #print(in_fea[j,:].shape)

    # for i in range(self.layer_num):
    #   # dot product of vector weight[i,:] with all rows of input
    #   in_fea = torch.tensordot(in_fea,self.weight[i,:],dims=([1],[0]))
    #   in_fea = torch.multiply(in_fea,x)
    #   in_fea  += self.bias[i,:] + in_fea
    #   #print(in_fea[j,:].shape)
         
    return in_fea


class DCN(nn.Module):
  def __init__(self, fea_dim = 5,emb_dim=8, dense_dim =5):
    super(DCN, self).__init__()
    # using weight vector in each layer
    #embedding matrix
    self.V = nn.Parameter(torch.randn(fea_dim, emb_dim))
    
    dnn_input_size = emb_dim * fea_dim + dense_dim
    layers =[nn.Linear(dnn_input_size,256),
             nn.ReLU(),
             nn.Dropout(0.5)
             ]
    # layers += layers
    #layers += [nn.Linear(256,1)]
    self.DNN = nn.Sequential(*layers)

    self.CrossN = CrossNet(dnn_input_size, layer_num=3)
    self.linear = None#nn.Linear()

  def forward(self,x_dense, x_sparse):
    # 
    #convert feature to dense features
    embedding_list = []
    for i in range(len(x_sparse)):
      emb_vec = torch.multiply( torch.unsqueeze(x_sparse[i],1),self.V)
      emb_vec = torch.flatten(emb_vec)
      embedding_list.append(emb_vec)
    embedding_list = torch.stack(embedding_list)
    dense_input = torch.cat([embedding_list,x_dense],dim=1)
    cross_out = self.CrossN(dense_input)
    dnn_out = self.DNN(dense_input)
    #out = dnn_out + cross_out
    cat_out = torch.cat([dnn_out, cross_out],dim=1)
    if self.linear == None:
      self.linear = nn.Linear(cat_out.shape[1],1)
    out =self.linear(cat_out)
    print(dnn_out,cross_out)
    return out


fea_dim = 5
emb_dim=8
dense_dim =5
batch = 5
dcn = DCN(fea_dim = 5,emb_dim=8, dense_dim =5)


x_sparse = torch.randn(batch,fea_dim)<0.5
x_dense = torch.randn(batch,dense_dim)
out = dcn(x_dense, x_sparse)
out 

Reference:

DeepCTR-Torch/interaction.py at 8265c75237e473c7f238fd6ba44cb09f55d1d9a9 · shenweichen/DeepCTR-TorchGitHub