MemoryDataLayer.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Diagnostics;
using MyCaffe.basecode;
using MyCaffe.db.image;
using MyCaffe.param;
using MyCaffe.common;
using MyCaffe.data;
 
namespace MyCaffe.layers
{
    public class MemoryDataLayer<T> : BaseDataLayer<T>
    {
        int m_nBatchSize;
        int m_nChannels;
        int m_nHeight;
        int m_nWidth;
        int m_nLabelChannels = 0;
        int m_nLabelHeight = 0;
        int m_nLabelWidth = 0;
        int m_nDataSize;
        int m_nLabelSize = 0;
        int m_nClipSize1 = 0;
        int m_nClipSize2 = 0;
        int m_nLabelIdx = -1;
        int m_nClipIdx = -1;
        Blob<T> m_blobData;
        Blob<T> m_blobLabel;
        Blob<T> m_blobClip = null;
        bool m_bHasNewData;
        int m_nPos = 0;
        int m_nN = 1;
 
        public event EventHandler<MemoryDataLayerGetDataArgs> OnGetData;
        public event EventHandler<MemoryDataLayerPackDataArgs<T>> OnDataPack;
 
        public MemoryDataLayer(CudaDnn<T> cuda, Log log, LayerParameter p)
            : base(cuda, log, p, null)
        {
            m_type = LayerParameter.LayerType.MEMORYDATA;
            m_blobData = new Blob<T>(cuda, log);
            m_blobData.Name = m_param.name + " data";
            m_blobLabel = new Blob<T>(cuda, log);
            m_blobLabel.Name = m_param.name + " label";
        }
 
        protected override void dispose()
        {
            base.dispose();
        }
 
        protected override void DataLayerSetUp(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            m_nBatchSize = (int)m_param.memory_data_param.batch_size;
            m_nChannels = (int)m_param.memory_data_param.channels;
            m_nHeight = (int)m_param.memory_data_param.height;
            m_nWidth = (int)m_param.memory_data_param.width;
            m_nDataSize = m_nChannels * m_nHeight * m_nWidth;
 
            m_log.CHECK_GT(m_nBatchSize * m_nDataSize, 0, "batch_size, channels, height, and width must be specified and positive in memory_data_param.");
 
            if (m_param.memory_data_param.label_type != LayerParameterBase.LABEL_TYPE.NONE)
            {
                m_nLabelChannels = (int)m_param.memory_data_param.label_channels;
                m_nLabelHeight = (int)m_param.memory_data_param.label_height;
                m_nLabelWidth = (int)m_param.memory_data_param.label_width;
                m_nLabelSize = m_nLabelChannels * m_nLabelHeight * m_nLabelWidth;
 
                if (m_param.memory_data_param.label_type == LayerParameterBase.LABEL_TYPE.MULTIPLE)
                    m_log.CHECK_GT(m_nBatchSize * m_nLabelSize, 0, "batch_size, label_channels, label_height, and label_width must be specified and positive in memory_data_param when using label_type = MULTIPLE.");
            }
 
            if (m_param.memory_data_param.clip_length1 > 0)
                m_nClipSize1 = (int)m_param.memory_data_param.clip_length1;
 
            if (m_param.memory_data_param.clip_length2 > 0)
                m_nClipSize2 = (int)m_param.memory_data_param.clip_length2;
 
            if (m_nClipSize1 > 1 && m_nClipSize2 == 0)
                m_nClipSize2 = 1;
 
            if (m_nClipSize2 > 1 && m_nClipSize1 == 0)
                m_nClipSize1 = 1;
 
            if (m_nClipSize1 > 0 || m_nClipSize2 > 0)
            {
                m_blobClip = new Blob<T>(m_cuda, m_log);
                m_blobClip.Name = m_param.name + " clip";
                m_blobClip.Reshape(new List<int>() { m_nClipSize1, m_nClipSize2 });
            }
 
            if (OnGetData != null)
            {
                OnGetData(this, new MemoryDataLayerGetDataArgs(true));
 
                if (m_param.memory_data_param.label_type != LayerParameterBase.LABEL_TYPE.NONE)
                {
                    m_log.CHECK_EQ(m_nLabelChannels, m_blobLabel.channels, "The actual label channels (" + m_blobLabel.channels.ToString() + ") do not match the 'memory_data_param.label_channels' setting of " + m_nLabelChannels.ToString() + ".");
                    m_log.CHECK_EQ(m_nLabelHeight, m_blobLabel.height, "The actual label channels (" + m_blobLabel.height.ToString() + ") do not match the 'memory_data_param.label_channels' setting of " + m_nLabelHeight.ToString() + ".");
                    m_log.CHECK_EQ(m_nLabelWidth, m_blobLabel.width, "The actual label channels (" + m_blobLabel.width.ToString() + ") do not match the 'memory_data_param.label_channels' setting of " + m_nLabelWidth.ToString() + ".");
                }
            }
            else
            {
                m_blobData.Reshape(m_nBatchSize, m_nChannels, m_nHeight, m_nWidth);
 
                if (m_param.memory_data_param.label_type != LayerParameterBase.LABEL_TYPE.NONE)
                {
                    if (m_param.memory_data_param.label_type == LayerParameterBase.LABEL_TYPE.SINGLE && m_blobClip != null)
                        m_blobLabel.Reshape(1, m_nLabelChannels, m_nLabelHeight, m_nLabelWidth);
                    else
                        m_blobLabel.Reshape(m_nBatchSize, m_nLabelChannels, m_nLabelHeight, m_nLabelWidth);
                }
            }
 
            colTop[0].Reshape(m_nBatchSize, m_nChannels, m_nHeight, m_nWidth);
            m_blobData.update_cpu_data();
 
            int nLabelIdx = -1;
            int nClipIdx = -1;
 
            for (int i = 0; i < m_param.top.Count; i++)
            {
                if (m_param.top[i].ToLower() == "label")
                    nLabelIdx = i;
 
                if (m_param.top[i].ToLower() == "clip")
                    nClipIdx = i;
            }
 
            if (m_param.memory_data_param.label_type != LayerParameterBase.LABEL_TYPE.NONE)
            {
                if (nLabelIdx == -1)
                    nLabelIdx = (nClipIdx == -1) ? 1 : nClipIdx + 1;
 
                m_nLabelIdx = nLabelIdx;
                colTop[nLabelIdx].ReshapeLike(m_blobLabel);
                m_blobLabel.update_cpu_data();
            }
 
            if (m_blobClip != null)
            {
                if (nClipIdx == -1)
                    throw new Exception("Could not find a top named 'clip'!");
 
                m_nClipIdx = nClipIdx;
                colTop[nClipIdx].ReshapeLike(m_blobClip);
                m_blobClip.update_cpu_data();
            }
        }
 
        public override void Reshape(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            base.Reshape(colBottom, colTop);
 
            m_blobData.Reshape(m_nBatchSize, m_nChannels, m_nHeight, m_nWidth);
 
            colTop[0].Reshape(m_nBatchSize, m_nChannels, m_nHeight, m_nWidth);
            m_blobData.update_cpu_data();
 
            if (m_param.memory_data_param.label_type != LayerParameterBase.LABEL_TYPE.NONE)
            {
                if (m_param.memory_data_param.label_type == LayerParameterBase.LABEL_TYPE.SINGLE && m_blobClip != null)
                    m_blobLabel.Reshape(1, m_nLabelChannels, m_nLabelHeight, m_nLabelWidth);
                else
                    m_blobLabel.Reshape(m_nBatchSize, m_nLabelChannels, m_nLabelHeight, m_nLabelWidth);
 
                colTop[m_nLabelIdx].ReshapeLike(m_blobLabel);
                m_blobLabel.update_cpu_data();
            }
 
            if (m_blobClip != null)
            {
                m_blobClip.Reshape(new List<int>() { m_nClipSize1, m_nClipSize2 });
                colTop[m_nClipIdx].ReshapeLike(m_blobClip);
                m_blobClip.update_cpu_data();
            }
        }
 
        public override int ExactNumBottomBlobs
        {
            get { return 0; }
        }
 
        public override int ExactNumTopBlobs
        {
            get { return -1; }
        }
 
        public override int MinTopBlobs
        {
            get { return 1; }
        }
 
        public override int MaxTopBlobs
        {
            get { return 3; }
        }
 
        public virtual void AddDatum(SimpleDatum sd, int nLblAxis = 1, bool bReset = false, bool bResizeBatch = false)
        {
            List<Datum> rgData = new List<Datum>();
            rgData.Add(new Datum(sd));
            AddDatumVector(rgData, null, nLblAxis, bReset, bResizeBatch);
        }
 
        public virtual void AddDatumVector(Datum[] rgData, Datum[] rgClip = null, int nLblAxis = 1, bool bReset = false, bool bResizeBatch = false)
        {
            AddDatumVector(rgData.ToList(), (rgClip == null) ? null : rgClip.ToList(), nLblAxis, bReset, bResizeBatch);
        }
 
        public virtual void AddDatumVector(List<Datum> rgData, List<Datum> rgClip = null, int nLblAxis = 1, bool bReset = false, bool bResizeBatch = false)
        {
            if (bReset)
                m_bHasNewData = false;
 
            m_log.CHECK(!m_bHasNewData, "Can't add data until current data has been consumed.");
            int nNum = rgData.Count;
            m_log.CHECK_GT(nNum, 0, "There are no datum to add.");
 
            if (rgClip == null)
            {
                if (bResizeBatch)
                    m_nBatchSize = rgData.Count;
 
                int nNumAligned = (int)Math.Floor((double)rgData.Count / (double)m_nBatchSize) * m_nBatchSize;
                m_log.CHECK_GT(nNumAligned, 0, "Three are not enough datum to add.");
 
                if (nNumAligned < nNum)
                {
                    m_log.WriteLine("WARNING: Clipping batch to batch aligned count of " + nNumAligned.ToString() + ".");
 
                    for (int i = nNumAligned; i < nNum; i++)
                    {
                        rgData.RemoveAt(rgData.Count - 1);
                    }
                }
 
                nNum = nNumAligned;
                m_log.CHECK_EQ(nNum % m_nBatchSize, 0, "The added data must be a multiple of the batch size.");
 
                m_blobData.Reshape(nNum, m_nChannels, m_nHeight, m_nWidth);
 
                // Apply data transformations (mirror, scale, crop...)
                m_transformer.Transform(rgData, m_blobData, m_cuda, m_log);
 
                if (m_param.memory_data_param.label_type != LayerParameterBase.LABEL_TYPE.NONE)
                {
                    List<int> rgLblShape = new List<int>();
                    int nLabelNum = nNum;
 
                    if (m_param.memory_data_param.label_type == LayerParameterBase.LABEL_TYPE.ONLY_ONE)
                        nLabelNum = 1;
 
                    rgLblShape.Add(nLabelNum);
                    rgLblShape.Add(1);
                    rgLblShape.Add(1);
                    rgLblShape.Add(1);
 
                    // Reshape label blob depending on label type.
                    if (m_param.memory_data_param.label_type == LayerParameterBase.LABEL_TYPE.MULTIPLE)
                    {
                        m_log.CHECK_GE(nLblAxis, 1, "The label axis must be greater than or equal to 1.");
                        m_log.CHECK_LE(nLblAxis, 4, "The label axis must be less than 4.");
 
                        List<float> rgLbl = BinaryData.UnPackFloatList(rgData[0].DataCriteria, rgData[0].DataCriteriaFormat);
                        rgLblShape[nLblAxis] = rgLbl.Count;
                    }
 
                    m_blobLabel.Reshape(rgLblShape);
 
                    // Copy labels - use DataCriteria for MULTIPLE labels
                    if (m_param.memory_data_param.label_type == LayerParameterBase.LABEL_TYPE.MULTIPLE)
                    {
                        T[] rgLabels = m_blobLabel.mutable_cpu_data;
                        int nIdx = 0;
 
                        for (int i = 0; i < nNum; i++)
                        {
                            List<float> rgLbl = BinaryData.UnPackFloatList(rgData[i].DataCriteria, rgData[i].DataCriteriaFormat);
                            for (int j = 0; j < rgLbl.Count; j++)
                            {
                                rgLabels[nIdx] = (T)Convert.ChangeType(rgLbl[j], typeof(T));
                                nIdx++;
                            }
                        }
                        m_blobLabel.mutable_cpu_data = rgLabels;
                    }
                    else
                    {
                        T[] rgLabels = m_blobLabel.mutable_cpu_data;
 
                        // For recurrent layers, single labels only use the last label in the sequence.
                        if (rgClip != null)
                        {
                            rgLabels[0] = (T)Convert.ChangeType(rgData[nNum - 1].label, typeof(T));
                        }
                        else
                        {
                            // Copy labels - use standard Datum label for SINGLE labels.
                            for (int i = 0; i < nNum; i++)
                            {
                                rgLabels[i] = (T)Convert.ChangeType(rgData[i].label, typeof(T));
                            }
                        }
 
                        m_blobLabel.mutable_cpu_data = rgLabels;
                    }
                }
            }
            else
            {
                if (OnDataPack == null)
                    throw new Exception("To properly handle data packing, you must connect the OnDataPack event and properly fill out the data and clip blobs with the ordering expected by the recurrent layers.");
 
                MemoryDataLayerPackDataArgs<T> args = new MemoryDataLayerPackDataArgs<T>(m_blobData, m_blobClip, m_blobLabel, rgData, rgClip);
                OnDataPack(this, args);
            }
 
            m_bHasNewData = true;
            m_nN = nNum;
        }
 
        public void Reset(Blob<T> data, Blob<T> labels, int n)
        {
            m_log.CHECK_EQ(n % m_nBatchSize, 0, "'n' must be a multiple of batch size.");
            m_nN = n;
 
            m_log.CHECK_GT(m_blobData.count(), 0, "There is no data.");
            m_blobData.ReshapeLike(data);
            m_cuda.copy(m_blobData.count(), data.gpu_data, m_blobData.mutable_gpu_data);
 
            if (m_param.memory_data_param.label_type != LayerParameterBase.LABEL_TYPE.NONE)
            {
                m_log.CHECK_GT(m_blobLabel.count(), 0, "There are no lables.");
                m_blobLabel.ReshapeLike(labels);
                m_cuda.copy(m_blobLabel.count(), labels.gpu_data, m_blobLabel.mutable_gpu_data);
            }
        }
 
        public void Copy(MemoryDataLayer<T> src)
        {
            m_nN = src.m_nN;
 
            m_blobData.ReshapeLike(src.m_blobData);
            m_cuda.copy(src.m_blobData.count(), src.m_blobData.gpu_data, m_blobData.mutable_gpu_data);
 
            if (m_param.memory_data_param.label_type != LayerParameterBase.LABEL_TYPE.NONE)
            {
                m_blobLabel.ReshapeLike(src.m_blobLabel);
                m_cuda.copy(src.m_blobLabel.count(), src.m_blobLabel.gpu_data, m_blobLabel.mutable_gpu_data);
            }
 
            if (m_blobClip != null)
            {
                m_blobClip.ReshapeLike(src.m_blobClip);
                m_cuda.copy(src.m_blobClip.count(), src.m_blobClip.gpu_data, m_blobClip.mutable_gpu_data);
            }
 
            m_bHasNewData = true;
        }
 
        public int batch_size
        {
            get { return (int)m_nBatchSize; }
            set
            {
                m_log.CHECK(!m_bHasNewData, "Can't change the batch size until current data has been consumed.");
                m_nBatchSize = value;
            }
        }
 
        public int channels
        {
            get { return m_nChannels; }
        }
 
        public int height
        {
            get { return m_nHeight; }
        }
 
        public int width
        {
            get { return m_nWidth; }
        }
 
        public int clip_size1
        {
            get { return m_nClipSize1; }
        }
 
        public int clip_size2
        {
            get { return m_nClipSize2; }
        }
 
        protected override void forward(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            int nSrcOffset;
 
            Blob<T> blobData = colTop[0];
            Blob<T> blobClip = null;
            Blob<T> blobLabel = null;
 
            for (int i=1; i<colTop.Count; i++)
            {
                if (blobClip == null && (colTop[i].type == BLOB_TYPE.CLIP || colTop[i].Name.ToLower().Contains("clip")))
                    blobClip = colTop[i];
                else
                    blobLabel = colTop[i];
            }
 
            blobData.Reshape(m_nBatchSize, m_nChannels, m_nHeight, m_nWidth);
 
            nSrcOffset = m_nPos * m_nDataSize;
            m_cuda.copy(blobData.count(), m_blobData.gpu_data, blobData.mutable_gpu_data, nSrcOffset, 0);
 
 
            if (m_param.memory_data_param.label_type != LayerParameterBase.LABEL_TYPE.NONE)
            {
                if (blobLabel == null)
                    m_log.WriteError(new Exception("Could not find the MemoryDataLayer 'label' top!"));
 
                int nLabelSize = m_nBatchSize;
                if (m_param.memory_data_param.label_type == LayerParameterBase.LABEL_TYPE.ONLY_ONE)
                    nLabelSize = 1;
 
                List<int> rgLabelShape = Utility.Clone<int>(m_blobLabel.shape());
                if (rgLabelShape.Count == 0)
                    rgLabelShape.Add(nLabelSize);
                else
                    rgLabelShape[0] = nLabelSize;
 
                blobLabel.Reshape(rgLabelShape);
 
                nSrcOffset = m_nPos * m_nLabelSize;
                m_cuda.copy(blobLabel.count(), m_blobLabel.gpu_data, blobLabel.mutable_gpu_data, nSrcOffset, 0);
            }
 
            if (m_blobClip != null)
            {
                if (blobClip == null)
                    m_log.WriteError(new Exception("Could not find the MemoryDataLayer 'clip' top!"));
 
                blobClip.CopyFrom(m_blobClip, false, true);
            }
 
            m_nPos = (m_nPos + m_nBatchSize) % m_nN;
 
            if (m_nPos == 0)
                m_bHasNewData = false;
        }
    }
 
    public class MemoryDataLayerGetDataArgs : EventArgs
    {
        bool m_bInitialization = true;
 
        public MemoryDataLayerGetDataArgs(bool bInit)
        {
            m_bInitialization = bInit;
        }
 
        public bool Initialization
        {
            get { return m_bInitialization; }
        }
    }
 
    public class MemoryDataLayerPackDataArgs<T> : EventArgs
    {
        Blob<T> m_blobData;
        Blob<T> m_blobClip;
        Blob<T> m_blobLabel;
        List<Datum> m_rgData;
        List<Datum> m_rgClip;
        LayerParameter.LayerType m_lstmType = LayerParameter.LayerType.LSTM;
 
        public MemoryDataLayerPackDataArgs(Blob<T> blobData, Blob<T> blobClip, Blob<T> blobLabel, List<Datum> rgData, List<Datum> rgClip, LayerParameter.LayerType type = LayerParameter.LayerType.LSTM)
        {
            m_blobData = blobData;
            m_blobClip = blobClip;
            m_blobLabel = blobLabel;
            m_rgData = rgData;
            m_rgClip = rgClip;
            m_lstmType = type;
        }
 
        public LayerParameter.LayerType LstmType
        {
            get { return m_lstmType; }
        }
 
        public Blob<T> Data
        {
            get { return m_blobData; }
        }
 
        public Blob<T> Clip
        {
            get { return m_blobClip; }
        }
 
        public Blob<T> Label
        {
            get { return m_blobLabel; }
        }
 
        public List<Datum> DataItems
        {
            get { return m_rgData; }
        }
 
        public List<Datum> ClipItems
        {
            get { return m_rgClip; }
        }
    }
}