AdaDeltaSolver.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.IO;
using MyCaffe.basecode;
using MyCaffe.db.image;
using MyCaffe.common;
using MyCaffe.param;
 
namespace MyCaffe.solvers
{
    public class AdaDeltaSolver<T> : SGDSolver<T>
    {
        public AdaDeltaSolver(CudaDnn<T> cuda, Log log, SolverParameter p, CancelEvent evtCancel, AutoResetEvent evtForceSnapshot, AutoResetEvent evtForceTest, IXDatabaseBase db, IXPersist<T> persist, int nSolverCount = 1, int nSolverRank = 0, Net<T> shareNet = null, onGetWorkspace getws = null, onSetWorkspace setws = null)
            : base(cuda, log, p, evtCancel, evtForceSnapshot, evtForceTest, db, persist, nSolverCount, nSolverRank, shareNet, getws, setws)
        {
            AdaDeltaPreSolve();
        }
 
        public void AdaDeltaPreSolve()
        {
            // Add the extra history entries for AdaDelta after those from
            // SGDSolver::PreSolve
            BlobCollection<T> colNetParams = m_net.learnable_parameters;
 
            for (int i = 0; i < colNetParams.Count; i++)
            {
                List<int> rgShape = colNetParams[i].shape();
                Blob<T> blob = new Blob<T>(m_cuda, m_log, rgShape);
                m_colHistory.Add(blob);
            }
        }
 
        public override void ComputeUpdateValue(int param_id, double dfRate, int nIterationOverride = -1)
        {
            BlobCollection<T> colNetParams = m_net.learnable_parameters;
 
            if (!colNetParams[param_id].DiffExists)
                return;
 
            List<double?> net_params_lr = m_net.params_lr;
            T fDelta = Utility.ConvertVal<T>(m_param.delta);
            T fMomentum = Utility.ConvertVal<T>(m_param.momentum);
            T fLocalRate = Utility.ConvertVal<T>(dfRate * net_params_lr[param_id].GetValueOrDefault(0));
            int nUpdateHistoryOffset = colNetParams.Count;
 
            // Compute the update to history, then copy it to the parameter diff.
            m_cuda.adadelta_update(colNetParams[param_id].count(), 
                                   colNetParams[param_id].mutable_gpu_diff, 
                                   m_colHistory[param_id].mutable_gpu_data, 
                                   m_colHistory[nUpdateHistoryOffset + param_id].mutable_gpu_data,
                                   fMomentum, 
                                   fDelta, 
                                   fLocalRate);
        }
    }
}