SwishLayer.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using MyCaffe.basecode;
using MyCaffe.common;
using MyCaffe.param;
 
namespace MyCaffe.layers
{
    public class SwishLayer<T> : NeuronLayer<T>
    {
        SigmoidLayer<T> m_sigmoidLayer;
        Blob<T> m_blobSigmoidInput;
        Blob<T> m_blobSigmoidOutput;
        BlobCollection<T> m_colSigmoidBottom = new BlobCollection<T>();
        BlobCollection<T> m_colSigmoidTop = new BlobCollection<T>();
 
        public SwishLayer(CudaDnn<T> cuda, Log log, LayerParameter p)
            : base(cuda, log, p)
        {
            m_type = LayerParameter.LayerType.SWISH;
 
            m_blobSigmoidInput = new Blob<T>(cuda, log);
            m_blobSigmoidInput.Name = m_param.name + " sigmoid in";
            m_blobSigmoidOutput = new Blob<T>(cuda, log);
            m_blobSigmoidOutput.Name = m_param.name + " sigmoid out";
 
            LayerParameter sigmoidParam = new LayerParameter(LayerParameter.LayerType.SIGMOID);
            sigmoidParam.sigmoid_param.engine = p.swish_param.engine;
 
            m_sigmoidLayer = new SigmoidLayer<T>(cuda, log, sigmoidParam);
        }
 
        protected override void dispose()
        {
            m_blobSigmoidOutput.Dispose();
            m_blobSigmoidInput.Dispose();
 
            base.dispose();
        }
 
        public override void LayerSetUp(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            base.LayerSetUp(colBottom, colTop);
 
            m_colSigmoidBottom.Clear();
            m_colSigmoidBottom.Add(m_blobSigmoidInput);
            m_colSigmoidTop.Clear();
            m_colSigmoidTop.Add(m_blobSigmoidOutput);
            m_sigmoidLayer.LayerSetUp(m_colSigmoidBottom, m_colSigmoidTop);
        }
 
        public override void Reshape(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            base.Reshape(colBottom, colTop);
 
            m_blobSigmoidInput.ReshapeLike(colBottom[0]);
            m_sigmoidLayer.Reshape(m_colSigmoidBottom, m_colSigmoidTop);
        }
 
        protected override void forward(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            long hBottomData = colBottom[0].gpu_data;
            long hSigmoidInputData = m_blobSigmoidInput.mutable_gpu_data;
            long hTopData = colTop[0].mutable_gpu_data;
            int nCount = colBottom[0].count();
            double dfBeta = m_param.swish_param.beta;
 
            m_cuda.copy(nCount, hBottomData, hSigmoidInputData);
            m_cuda.scal(nCount, dfBeta, hSigmoidInputData);
            m_sigmoidLayer.Forward(m_colSigmoidBottom, m_colSigmoidTop);
            m_cuda.mul(nCount, hBottomData, m_blobSigmoidOutput.gpu_data, hTopData);
        }
 
        protected override void backward(BlobCollection<T> colTop, List<bool> rgbPropagateDown, BlobCollection<T> colBottom)
        {
            if (rgbPropagateDown[0])
            {
                long hTopData = colTop[0].gpu_data;
                long hTopDiff = colTop[0].gpu_diff;
                long hSigmoidOutputData = m_blobSigmoidOutput.gpu_data;
                long hBottomDiff = colBottom[0].mutable_gpu_diff;
                int nCount = colBottom[0].count();
                double dfBeta = m_param.swish_param.beta;
 
                m_cuda.swish_bwd(nCount, hTopDiff, hTopData, hSigmoidOutputData, hBottomDiff, dfBeta);
            }
        }
    }
}