TrainerPG.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using MyCaffe.basecode;
using MyCaffe.common;
using MyCaffe.fillers;
using MyCaffe.layers;
using MyCaffe.param;
using MyCaffe.solvers;
 
namespace MyCaffe.trainers.pg.mt
{
    public class TrainerPG<T> : IxTrainerRL, IDisposable
    {
        IxTrainerCallback m_icallback;
        CryptoRandom m_random = new CryptoRandom();
        MyCaffeControl<T> m_mycaffe;
        PropertySet m_properties;
        int m_nThreads = 1;
        List<int> m_rgGpuID = new List<int>();
        Optimizer<T> m_optimizer = null;
 
        public TrainerPG(MyCaffeControl<T> mycaffe, PropertySet properties, CryptoRandom random, IxTrainerCallback icallback)
        {
            m_icallback = icallback;
            m_mycaffe = mycaffe;
            m_properties = properties;
            m_random = random;
 
            m_nThreads = m_properties.GetPropertyAsInt("Threads", 1);
            m_rgGpuID.Add(m_mycaffe.Cuda.GetDeviceID());
 
            string strGpuID = m_properties.GetProperty("GPUIDs", false);
            if (strGpuID != null && m_nThreads > 1)
            {
                int nDeviceCount = m_mycaffe.Cuda.GetDeviceCount();
 
                m_rgGpuID.Clear();
                string[] rgstrGpuIDs = strGpuID.Split(',');
                foreach (string strID in rgstrGpuIDs)
                {
                    int nDevId = int.Parse(strID);
 
                    if (nDevId < 0 || nDevId >= nDeviceCount)
                        throw new Exception("Invalid device ID - value must be within the range [0," + (nDeviceCount - 1).ToString() + "].");
 
                    m_rgGpuID.Add(nDevId);
                }
            }
        }
 
        public void Dispose()
        {
        }
 
        public bool Initialize()
        {
            m_mycaffe.CancelEvent.Reset();
            m_icallback.OnInitialize(new InitializeArgs(m_mycaffe));
            return true;
        }
 
        private void wait(int nWait)
        {
            int nWaitInc = 250;
            int nTotalWait = 0;
 
            while (nTotalWait < nWait)
            {
                m_icallback.OnWait(new WaitArgs(nWaitInc));
                nTotalWait += nWaitInc;
            }
        }
 
        public bool Shutdown(int nWait)
        {
            if (m_mycaffe != null)
            {
                m_mycaffe.CancelEvent.Set();
                wait(nWait);
            }
 
            m_icallback.OnShutdown();
 
            return true;
        }
 
        public ResultCollection RunOne(int nDelay = 1000)
        {
            m_mycaffe.CancelEvent.Reset();
            Agent<T> agent = new Agent<T>(0, m_icallback, m_mycaffe, m_properties, m_random, Phase.TRAIN, 0, 1);
            Tuple<int,int> res = agent.Run(nDelay);
 
            List<Result> rgActions = new List<Result>();
            for (int i = 0; i < res.Item2; i++)
            {
                if (res.Item1 == i)
                    rgActions.Add(new Result(i, 1.0));
                else
                    rgActions.Add(new Result(i, 0.0));
            }
 
            agent.Dispose();
 
            return new ResultCollection(rgActions, LayerParameter.LayerType.SOFTMAX);
        }
 
        public byte[] Run(int nN, PropertySet runProp, out string type)
        {
            m_mycaffe.CancelEvent.Reset();
            Agent<T> agent = new Agent<T>(0, m_icallback, m_mycaffe, m_properties, m_random, Phase.RUN, 0, 1);
            byte[] rgResults = agent.Run(nN, out type);
            agent.Dispose();
 
            return rgResults;
        }
 
        public bool Test(int nN, ITERATOR_TYPE type)
        {
            int nDelay = 1000;
            string strProp = m_properties.ToString();
 
            // Turn off the num-skip to run at normal speed.
            strProp += "EnableNumSkip=False;";
            PropertySet properties = new PropertySet(strProp);
 
            m_mycaffe.CancelEvent.Reset();
            Agent<T> agent = new Agent<T>(0, m_icallback, m_mycaffe, properties, m_random, Phase.TRAIN, 0, 1);           
            agent.Run(Phase.TEST, nN, type, TRAIN_STEP.NONE);
 
            agent.Dispose();
            Shutdown(nDelay);
 
            return true;
        }
 
        public bool Train(int nN, ITERATOR_TYPE type, TRAIN_STEP step)
        {
            List<Agent<T>> rgAgents = new List<Agent<T>>();
            int nGpuIdx = 0;
 
            m_mycaffe.CancelEvent.Reset();
 
            if (m_nThreads > 1)
                m_optimizer = new Optimizer<T>(m_mycaffe);
 
            for (int i = 0; i < m_nThreads; i++)
            {
                int nGpuID = m_rgGpuID[nGpuIdx];
 
                Agent<T> agent = new Agent<T>(i, m_icallback, m_mycaffe, m_properties, m_random, Phase.TRAIN, nGpuID, m_nThreads);
                agent.OnApplyUpdates += Agent_OnApplyUpdates;
                rgAgents.Add(agent);
 
                nGpuIdx++;
                if (nGpuIdx == m_rgGpuID.Count)
                    nGpuIdx = 0;
            }
 
            if (m_optimizer != null)
                m_optimizer.Start(new WorkerStartArgs(0, Phase.TRAIN, nN, type, step));
 
            WorkerStartArgs args = new WorkerStartArgs(1, Phase.TRAIN, nN, type, step);            
            foreach (Agent<T> agent in rgAgents)
            {
                agent.Start(args);
            }
 
            while (!m_mycaffe.CancelEvent.WaitOne(250))
            {
            }
 
            foreach (Agent<T> agent in rgAgents)
            {
                agent.Stop(1000);
                agent.Dispose();
            }
 
            if (m_optimizer != null)
            {
                m_optimizer.Stop(1000);
                m_optimizer.Dispose();
                m_optimizer = null;
            }
 
            Shutdown(3000);
 
            return false;
        }
 
        private void Agent_OnApplyUpdates(object sender, ApplyUpdateArgs<T> e)
        {
            if (m_optimizer != null)
                m_optimizer.ApplyUpdates(e.MyCaffeWorker, e.Iteration);
        }
    }
 
    class WorkerStartArgs
    {
        int m_nCycleDelay;
        Phase m_phase;
        int m_nN;
        ITERATOR_TYPE m_type;
        TRAIN_STEP m_step = TRAIN_STEP.NONE;
 
        public WorkerStartArgs(int nCycleDelay, Phase phase, int nN, ITERATOR_TYPE type, TRAIN_STEP step)
        {
            m_nCycleDelay = nCycleDelay;
            m_phase = phase;
            m_nN = nN;
            m_type = type;
            m_step = step;
        }
 
        public TRAIN_STEP Step
        {
            get { return m_step; }
        }
 
        public int CycleDelay
        {
            get { return m_nCycleDelay; }
        }
 
        public Phase Phase
        {
            get { return m_phase; }
        }
 
        public int N
        {
            get { return m_nN; }
        }
 
        public ITERATOR_TYPE IterationType
        {
            get { return m_type; }
        }
    }
 
    class Worker
    {
        protected int m_nIndex = -1;
        protected AutoResetEvent m_evtCancel = new AutoResetEvent(false);
        protected ManualResetEvent m_evtDone = new ManualResetEvent(false);
        protected Task m_workTask = null;
 
        public Worker(int nIdx)
        {
            m_nIndex = nIdx;
        }
 
        protected virtual void doWork(object arg)
        {
        }
 
        public void Start(WorkerStartArgs args)
        {
            if (m_workTask == null)
                m_workTask = Task.Factory.StartNew(new Action<object>(doWork), args);
        }
 
        public void Stop(int nWait)
        {
            m_evtCancel.Set();
            m_workTask = null;
            m_evtDone.WaitOne(nWait);
        }
    }
 
    class Optimizer<T> : Worker, IDisposable
    {
        MyCaffeControl<T> m_mycaffePrimary;
        MyCaffeControl<T> m_mycaffeWorker;
        int m_nIteration;
        double m_dfLearningRate;
        AutoResetEvent m_evtApplyUpdates = new AutoResetEvent(false);
        ManualResetEvent m_evtDoneApplying = new ManualResetEvent(false);
        object m_syncObj = new object();
 
        public Optimizer(MyCaffeControl<T> mycaffePrimary)
            : base(0)
        {
            m_mycaffePrimary = mycaffePrimary;
        }
 
        public void Dispose()
        {
        }
 
        protected override void doWork(object arg)
        {
            WorkerStartArgs args = arg as WorkerStartArgs;
 
            m_mycaffePrimary.Cuda.SetDeviceID();
 
            List<WaitHandle> rgWait = new List<WaitHandle>();
            rgWait.Add(m_evtApplyUpdates);
            rgWait.AddRange(m_mycaffePrimary.CancelEvent.Handles);
 
            int nWait = WaitHandle.WaitAny(rgWait.ToArray());
 
            while (nWait == 0)
            {
                if (args.Step != TRAIN_STEP.FORWARD)
                {
                    m_mycaffePrimary.CopyGradientsFrom(m_mycaffeWorker);
                    m_mycaffePrimary.Log.Enable = false;
                    m_dfLearningRate = m_mycaffePrimary.ApplyUpdate(m_nIteration);
                    m_mycaffePrimary.Log.Enable = true;
                    m_mycaffeWorker.CopyWeightsFrom(m_mycaffePrimary);
                }
 
                m_evtDoneApplying.Set();
 
                nWait = WaitHandle.WaitAny(rgWait.ToArray());
 
                if (args.Step != TRAIN_STEP.NONE)
                    break;
            }
        }
 
        public double ApplyUpdates(MyCaffeControl<T> mycaffeWorker, int nIteration)
        {
            lock (m_syncObj)
            {
                m_mycaffeWorker = mycaffeWorker;
                m_nIteration = nIteration;
 
                m_evtDoneApplying.Reset();
                m_evtApplyUpdates.Set();
 
                List<WaitHandle> rgWait = new List<WaitHandle>();
                rgWait.Add(m_evtDoneApplying);
                rgWait.AddRange(m_mycaffePrimary.CancelEvent.Handles);
 
                int nWait = WaitHandle.WaitAny(rgWait.ToArray());
                if (nWait != 0)
                    return 0;
 
                return m_dfLearningRate;
            }
        }
    }
 
    class Agent<T> : Worker, IDisposable 
    {
        IxTrainerCallback m_icallback;
        Brain<T> m_brain;
        PropertySet m_properties;
        CryptoRandom m_random;
        float m_fGamma;
        bool m_bAllowDiscountReset = false;
        bool m_bUseRawInput = false;
        int m_nEpsSteps = 0;
        double m_dfEpsStart = 0;
        double m_dfEpsEnd = 0;
        double m_dfExplorationRate = 0;
        int m_nEpisodeBatchSize = 1;
        double m_dfEpisodeElitePercentile = 1;
        static object m_syncObj = new object();
        bool m_bShowActionProb = false;
        bool m_bVerbose = false;
 
        public event EventHandler<ApplyUpdateArgs<T>> OnApplyUpdates;
 
        public Agent(int nIdx, IxTrainerCallback icallback, MyCaffeControl<T> mycaffe, PropertySet properties, CryptoRandom random, Phase phase, int nGpuID, int nThreadCount)
            : base(nIdx)
        {
            m_icallback = icallback;
            m_brain = new Brain<T>(mycaffe, properties, random, phase, nGpuID, nThreadCount);
            m_brain.OnApplyUpdate += brain_OnApplyUpdate;
            m_properties = properties;
            m_random = random;
 
            m_fGamma = (float)properties.GetPropertyAsDouble("Gamma", 0.99);
            m_bAllowDiscountReset = properties.GetPropertyAsBool("AllowDiscountReset", false);
            m_bUseRawInput = properties.GetPropertyAsBool("UseRawInput", false);
            m_nEpsSteps = properties.GetPropertyAsInt("EpsSteps", 0);
            m_dfEpsStart = properties.GetPropertyAsDouble("EpsStart", 0);
            m_dfEpsEnd = properties.GetPropertyAsDouble("EpsEnd", 0);
            m_nEpisodeBatchSize = m_properties.GetPropertyAsInt("EpisodeBatchSize", 1);
            m_dfEpisodeElitePercentile = properties.GetPropertyAsDouble("EpisodeElitePercent", 1.0);
            m_bShowActionProb = properties.GetPropertyAsBool("ShowActionProb", false);
            m_bVerbose = properties.GetPropertyAsBool("Verbose", false);
 
            if (m_dfEpsStart < 0 || m_dfEpsStart > 1)
                throw new Exception("The 'EpsStart' is out of range - please specify a real number in the range [0,1]");
 
            if (m_dfEpsEnd < 0 || m_dfEpsEnd > 1)
                throw new Exception("The 'EpsEnd' is out of range - please specify a real number in the range [0,1]");
 
            if (m_dfEpsEnd > m_dfEpsStart)
                throw new Exception("The 'EpsEnd' must be less than the 'EpsStart' value.");
        }
 
        private void brain_OnApplyUpdate(object sender, ApplyUpdateArgs<T> e)
        {
            if (OnApplyUpdates != null)
                OnApplyUpdates(sender, e);
        }
 
        public void Dispose()
        {
            if (m_brain != null)
            {
                m_brain.Dispose();
                m_brain = null;
            }
        }
 
        protected override void doWork(object arg)
        {
            try
            {
                WorkerStartArgs args = arg as WorkerStartArgs;
 
                lock (m_syncObj)
                {
                    m_brain.Create();
                }
 
                m_evtDone.Reset();
                m_evtCancel.Reset();
                Run(args.Phase, args.N, args.IterationType, args.Step);
                m_evtDone.Set();
            }
            catch (Exception excpt)
            {
                m_brain.OutputLog.WriteError(excpt);
            }
 
            m_brain.Cancel.Set();
        }
 
        private double getEpsilon(int nEpisode)
        {
            if (m_nEpsSteps == 0)
                return 0;
 
            if (nEpisode >= m_nEpsSteps)
                return m_dfEpsEnd;
 
            return m_dfEpsStart + (double)(nEpisode * (m_dfEpsEnd - m_dfEpsStart)/m_nEpsSteps);
        }
 
        private StateBase getData(Phase phase, int nIdx, int nAction, bool? bResetOverride = null)
        {
            GetDataArgs args = m_brain.getDataArgs(phase, nIdx, nAction, bResetOverride);
            m_icallback.OnGetData(args);
            return args.State;
        }
 
        private int getAction(int nEpisode, SimpleDatum sd, SimpleDatum sdClip, int nActionCount, TRAIN_STEP step, out float[] rgfAprob)
        {
            if (step == TRAIN_STEP.NONE)
            {
                m_dfExplorationRate = getEpsilon(nEpisode);
 
                if (m_dfExplorationRate > 0 && m_random.NextDouble() < m_dfExplorationRate)
                {
                    rgfAprob = new float[nActionCount];
                    int nAction = m_random.Next(nActionCount);
                    rgfAprob[nAction] = 1.0f;
                    return nAction;
                }
            }
 
            return m_brain.act(sd, sdClip, out rgfAprob);
        }
 
        private int updateStatus(int nIteration, int nEpisodeCount, double dfRunningReward, double dfRewardSum, double dfLoss, double dfLearningRate)
        {
            GetStatusArgs args = new GetStatusArgs(m_nIndex, nIteration, nEpisodeCount, 1000000, dfRunningReward, dfRewardSum, m_dfExplorationRate, 0, dfLoss, dfLearningRate);
            m_icallback.OnUpdateStatus(args);
            return args.NewFrameCount;
        }
 
        public Tuple<int, int> Run(int nDelay = 1000)
        {
            // Reset the environment and get the initial state.
            getData(Phase.RUN, m_nIndex, -1);
            Thread.Sleep(nDelay);
 
            StateBase state = getData(Phase.RUN, m_nIndex, -1, false);
            float[] rgfAprob;
 
            m_brain.Create();
 
            int a = m_brain.act(state.Data, state.Clip, out rgfAprob);
 
            return new Tuple<int, int>(a, state.ActionCount);
        }
 
        public byte[] Run(int nIterations, out string type)
        {
            IxTrainerCallbackRNN icallback = m_icallback as IxTrainerCallbackRNN;
            if (icallback == null)
                throw new Exception("The Run method requires an IxTrainerCallbackRNN interface to convert the results into the native format!");
 
            m_brain.Create();
 
            StateBase s = getData(Phase.RUN, m_nIndex, -1);
            int nIteration = 0;
            List<float> rgResults = new List<float>();
            int nLookahead = m_properties.GetPropertyAsInt("Lookahead", 0);
 
            while (!m_brain.Cancel.WaitOne(0) && (nIterations == -1 || nIteration < nIterations))
            {
                // Preprocess the observation.
                SimpleDatum x = m_brain.Preprocess(s, m_bUseRawInput);
 
                // Forward the policy network and sample an action.
                float[] rgfAprob;
                int nAction = m_brain.act(x, s.Clip, out rgfAprob);
 
                if (m_bShowActionProb && m_bVerbose)
                {
                    string strOut = "Action Prob: " + Utility.ToString<float>(rgfAprob.ToList(), 4) + " -> " + nAction.ToString();
                    m_brain.OutputLog.WriteLine(strOut);
                }
 
                int nSeqLen = m_brain.RecurrentSequenceLength;
                int nItemLen = s.Data.ItemCount / nSeqLen;
                int nData1Idx = s.Data.ItemCount - (nItemLen * (nLookahead + 1));
 
                rgResults.Add(s.Data.TimeStamp.ToFileTime());
                rgResults.Add((float)s.Data.GetDataAtF(nData1Idx));
                rgResults.Add(nAction);
 
                // Take the next step using the action
                s = getData(Phase.RUN, m_nIndex, nAction);
                nIteration++;
 
                m_brain.OutputLog.Progress = ((double)nIteration / (double)nIterations);
            }
 
            ConvertOutputArgs args = new ConvertOutputArgs(nIterations, rgResults.ToArray());
            icallback.OnConvertOutput(args);
 
            type = args.RawType;
            return args.RawOutput;
        }
 
        private bool isAtIteration(int nN, ITERATOR_TYPE type, int nIteration, int nEpisode)
        {
            if (nN == -1)
                return false;
 
            if (type == ITERATOR_TYPE.EPISODE)
            {
                if (nEpisode < nN)
                    return false;
 
                return true;
            }
            else
            {
                if (nIteration < nN)
                    return false;
 
                return true;
            }
        }
 
        public void Run(Phase phase, int nN, ITERATOR_TYPE type, TRAIN_STEP step)
        {
            MemoryCache rgMemoryCache = new MemoryCache(m_nEpisodeBatchSize);
            Memory rgMemory = new Memory();
            double? dfRunningReward = null;
            double dfEpisodeReward = 0;
            int nEpisode = 0;
            int nIteration = 0;
 
            m_brain.Create();
 
            StateBase s = getData(phase, m_nIndex, -1);
 
            while (!m_brain.Cancel.WaitOne(0) && !isAtIteration(nN, type, nIteration, nEpisode))
            {
                // Preprocess the observation.
                SimpleDatum x = m_brain.Preprocess(s, m_bUseRawInput);
 
                // Forward the policy network and sample an action.
                float[] rgfAprob;
                int action = getAction(nIteration, x, s.Clip, s.ActionCount, step, out rgfAprob);
 
                if (m_bShowActionProb && m_bVerbose)
                {
                    string strOut = "Action Prob: " + Utility.ToString<float>(rgfAprob.ToList(), 4) + " -> " + action.ToString();
                    m_brain.OutputLog.WriteLine(strOut);
                }
 
                if (step == TRAIN_STEP.FORWARD)
                    return;
 
                // Take the next step using the action
                StateBase s_ = getData(phase, m_nIndex, action);
                dfEpisodeReward += s_.Reward;
 
                if (phase == Phase.TRAIN)
                {
                    // Build up episode memory, using reward for taking the action.
                    rgMemory.Add(new MemoryItem(s, x, action, rgfAprob, (float)s_.Reward));
 
                    // An episode has finished.
                    if (s_.Done)
                    {
                        nEpisode++;
                        nIteration++;
 
                        if (rgMemoryCache.Add(rgMemory))
                        {
                            if (m_bShowActionProb)
                                m_brain.OutputLog.WriteLine("---learning---");
 
                            rgMemoryCache.PurgeNonElite(m_dfEpisodeElitePercentile);
 
                            for (int i=0; i<rgMemoryCache.Count; i++)
                            {
                                Memory rgMemory1 = rgMemoryCache[i];
 
                                m_brain.Reshape(rgMemory1);
 
                                // Compute the discounted reward (backwards through time)
                                float[] rgDiscountedR = rgMemory1.GetDiscountedRewards(m_fGamma, m_bAllowDiscountReset);
                                // Rewards are normalized when set to be unit normal (helps control the gradient estimator variance)
                                m_brain.SetDiscountedR(rgDiscountedR);
 
                                // Sigmoid models, set the probabilities up font.
                                if (!m_brain.UsesSoftMax)
                                {
                                    // Get the action probabilities.
                                    float[] rgfAprobSet = rgMemory1.GetActionProbabilities();
                                    // The action probabilities are used to calculate the initial gradient within the loss function.
                                    m_brain.SetActionProbabilities(rgfAprobSet);
                                }
 
                                // Get the action one-hot vectors.  When using Softmax, this contains the one-hot vector containing
                                // each action set (e.g. 3 actions with action 0 set would return a vector <1,0,0>).  
                                // When using a binary probability (e.g. with Sigmoid), the each action set only contains a
                                // single element which is set to the action value itself (e.g. 0 for action '0' and 1 for action '1')
                                float[] rgfAonehotSet = rgMemory1.GetActionOneHotVectors();
                                m_brain.SetActionOneHotVectors(rgfAonehotSet);
 
                                // Train for one iteration, which triggers the loss function.
                                List<Datum> rgData = rgMemory1.GetData();
                                List<Datum> rgClip = rgMemory1.GetClip();
 
                                m_brain.SetData(rgData, rgClip);
 
                                bool bApplyGradients = (i == rgMemoryCache.Count - 1) ? true : false;
                                m_brain.Train(nIteration, step, bApplyGradients);
 
                                // Update reward running
                                if (!dfRunningReward.HasValue)
                                    dfRunningReward = dfEpisodeReward;
                                else
                                    dfRunningReward = dfRunningReward * 0.99 + dfEpisodeReward * 0.01;
 
                                nEpisode = updateStatus(nIteration, nEpisode, dfRunningReward.Value, dfEpisodeReward, m_brain.LastLoss, m_brain.LearningRate);
                                dfEpisodeReward = 0;
                            }
 
                            rgMemoryCache.Clear();
                        }
 
                        s = getData(phase, m_nIndex, -1);
                        rgMemory = new Memory();
 
                        if (step != TRAIN_STEP.NONE)
                            return;
                    }
                    else
                    {
                        s = s_;
                    }
                }
                else
                {
                    if (s_.Done)
                    {
                        nEpisode++;
 
                        // Update reward running
                        if (!dfRunningReward.HasValue)
                            dfRunningReward = dfEpisodeReward;
                        else
                            dfRunningReward = dfRunningReward * 0.99 + dfEpisodeReward * 0.01;
 
                        nEpisode = updateStatus(nIteration, nEpisode, dfRunningReward.Value, dfEpisodeReward, m_brain.LastLoss, m_brain.LearningRate);
                        dfEpisodeReward = 0;
 
                        s = getData(phase, m_nIndex, -1);
                    }
                    else
                    {
                        s = s_;
                    }
 
                    nIteration++;
                }
            }
        }
    }
 
    class Brain<T> : IDisposable
    {
        MyCaffeControl<T> m_mycaffePrimary;
        MyCaffeControl<T> m_mycaffeWorker;
        Net<T> m_net;
        Solver<T> m_solver;
        MemoryDataLayer<T> m_memData;
        MemoryLossLayer<T> m_memLoss;
        SoftmaxLayer<T> m_softmax = null;
        SoftmaxCrossEntropyLossLayer<T> m_softmaxCe = null;
        bool m_bSoftmaxCeSetup = false;
        PropertySet m_properties;
        CryptoRandom m_random;
        BlobCollection<T> m_colAccumulatedGradients = new BlobCollection<T>();
        Blob<T> m_blobDiscountedR;
        Blob<T> m_blobPolicyGradient;
        Blob<T> m_blobActionOneHot;
        Blob<T> m_blobDiscountedR1;
        Blob<T> m_blobPolicyGradient1;
        Blob<T> m_blobActionOneHot1;
        Blob<T> m_blobLoss;
        Blob<T> m_blobAprobLogit;
        bool m_bSkipLoss;
        int m_nMiniBatch = 10;
        SimpleDatum m_sdLast = null;
        double m_dfLastLoss = 0;
        double m_dfLearningRate = 0;
        Phase m_phase;
        int m_nGpuID = 0;
        int m_nThreadCount = 1;
        bool m_bCreated = false;
        bool m_bUseAcceleratedTraining = false;
        int m_nRecurrentSequenceLength = 0;
        List<Datum> m_rgData = null;
        List<Datum> m_rgClip = null;
 
        public event EventHandler<ApplyUpdateArgs<T>> OnApplyUpdate;
 
        public Brain(MyCaffeControl<T> mycaffe, PropertySet properties, CryptoRandom random, Phase phase, int nGpuID, int nThreadCount)
        {
            m_properties = properties;
            m_random = random;
            m_phase = phase;
            m_nGpuID = nGpuID;
            m_nThreadCount = nThreadCount;
            m_mycaffePrimary = mycaffe;
 
            int nMiniBatch = mycaffe.CurrentProject.GetBatchSize(phase);
            if (nMiniBatch != 0)
                m_nMiniBatch = nMiniBatch;
 
            m_nMiniBatch = m_properties.GetPropertyAsInt("MiniBatch", m_nMiniBatch);
 
            double? dfRate = mycaffe.CurrentProject.GetSolverSettingAsNumeric("base_lr");
            if (dfRate.HasValue)
                m_dfLearningRate = dfRate.Value;
 
            m_bUseAcceleratedTraining = properties.GetPropertyAsBool("UseAcceleratedTraining", false);
        }
 
        public void Create()
        {
            if (m_bCreated)
                return;
 
            m_mycaffePrimary.Log.Enable = false;
 
            if (m_nThreadCount == 1)
            {
                m_mycaffeWorker = m_mycaffePrimary;
                m_mycaffePrimary.Cuda.SetDeviceID();
            }
            else
            {
                m_mycaffeWorker = m_mycaffePrimary.Clone(m_nGpuID);
            }
 
            m_mycaffePrimary.Log.Enable = true;
 
            m_mycaffeWorker.Cuda.SetDeviceID();
 
            m_net = m_mycaffeWorker.GetInternalNet(m_phase);
            m_solver = m_mycaffeWorker.GetInternalSolver();
 
            m_memData = m_net.FindLayer(LayerParameter.LayerType.MEMORYDATA, null) as MemoryDataLayer<T>;
            m_memLoss = m_net.FindLayer(LayerParameter.LayerType.MEMORY_LOSS, null) as MemoryLossLayer<T>;
            m_softmax = m_net.FindLayer(LayerParameter.LayerType.SOFTMAX, null) as SoftmaxLayer<T>;
 
            if (m_memData == null)
                throw new Exception("Could not find the MemoryData Layer!");
 
            if (m_memLoss == null && m_phase != Phase.RUN)
                throw new Exception("Could not find the MemoryLoss Layer!");
 
            m_memData.OnDataPack += memData_OnDataPack;
 
            if (m_memLoss != null)
                m_memLoss.OnGetLoss += memLoss_OnGetLoss;
 
            m_blobDiscountedR = new Blob<T>(m_mycaffeWorker.Cuda, m_mycaffeWorker.Log);
            m_blobPolicyGradient = new Blob<T>(m_mycaffeWorker.Cuda, m_mycaffeWorker.Log);
            m_blobActionOneHot = new Blob<T>(m_mycaffeWorker.Cuda, m_mycaffeWorker.Log);
            m_blobDiscountedR1 = new Blob<T>(m_mycaffeWorker.Cuda, m_mycaffeWorker.Log);
            m_blobPolicyGradient1 = new Blob<T>(m_mycaffeWorker.Cuda, m_mycaffeWorker.Log);
            m_blobActionOneHot1 = new Blob<T>(m_mycaffeWorker.Cuda, m_mycaffeWorker.Log);
            m_blobLoss = new Blob<T>(m_mycaffeWorker.Cuda, m_mycaffeWorker.Log);
            m_blobAprobLogit = new Blob<T>(m_mycaffeWorker.Cuda, m_mycaffeWorker.Log);
 
            if (m_softmax != null)
            {
                LayerParameter p = new LayerParameter(LayerParameter.LayerType.SOFTMAXCROSSENTROPY_LOSS);
                p.loss_weight.Add(1);
                p.loss_weight.Add(0);
                p.loss_param.normalization = LossParameter.NormalizationMode.NONE;
                m_softmaxCe = new SoftmaxCrossEntropyLossLayer<T>(m_mycaffeWorker.Cuda, m_mycaffeWorker.Log, p);
            }
 
            m_colAccumulatedGradients = m_net.learnable_parameters.Clone();
            m_colAccumulatedGradients.SetDiff(0);
 
            m_bCreated = true;
        }
 
        private void dispose(ref Blob<T> b)
        {
            if (b != null)
            {
                b.Dispose();
                b = null;
            }
        }
 
        public void Dispose()
        {
            if (m_memLoss != null)
                m_memLoss.OnGetLoss -= memLoss_OnGetLoss;
 
            if (m_memData != null)
                m_memData.OnDataPack -= memData_OnDataPack;
 
            dispose(ref m_blobDiscountedR);
            dispose(ref m_blobPolicyGradient);
            dispose(ref m_blobActionOneHot);
            dispose(ref m_blobDiscountedR1);
            dispose(ref m_blobPolicyGradient1);
            dispose(ref m_blobActionOneHot1);
            dispose(ref m_blobLoss);
            dispose(ref m_blobAprobLogit);
 
            if (m_colAccumulatedGradients != null)
            {
                m_colAccumulatedGradients.Dispose();
                m_colAccumulatedGradients = null;
            }
 
            if (m_mycaffeWorker != m_mycaffePrimary && m_mycaffeWorker != null)
                m_mycaffeWorker.Dispose();
 
            m_mycaffeWorker = null;
        }
 
        public int RecurrentSequenceLength
        {
            get { return m_nRecurrentSequenceLength; }
        }
 
        public Log OutputLog
        {
            get { return m_mycaffePrimary.Log; }
        }
 
        public bool UsesSoftMax
        {
            get { return (m_softmax == null) ? false : true; }
        }
 
        public int Reshape(Memory mem)
        {
            int nNum = mem.Count;
            int nChannels = mem[0].Data.Channels;
            int nHeight = mem[0].Data.Height;
            int nWidth = mem[0].Data.Height;
            int nActionProbs = 1;
            int nFound = 0;
 
            for (int i = 0; i < m_net.output_blobs.Count; i++)
            {
                if (m_net.output_blobs[i].type != BLOB_TYPE.LOSS)
                {
                    int nCh = m_net.output_blobs[i].channels;
                    nActionProbs = Math.Max(nCh, nActionProbs);
                    nFound++;
                }
            }
 
            if (nFound == 0)
                throw new Exception("Could not find a non-loss output!  Your model should output the loss and the action probabilities.");
 
            m_blobDiscountedR.Reshape(nNum, nActionProbs, 1, 1);
            m_blobPolicyGradient.Reshape(nNum, nActionProbs, 1, 1);
            m_blobActionOneHot.Reshape(nNum, nActionProbs, 1, 1);
            m_blobDiscountedR1.Reshape(nNum, nActionProbs, 1, 1);
            m_blobPolicyGradient1.Reshape(nNum, nActionProbs, 1, 1);
            m_blobActionOneHot1.Reshape(nNum, nActionProbs, 1, 1);
            m_blobLoss.Reshape(1, 1, 1, 1);
 
            return nActionProbs;
        }
 
        public void SetDiscountedR(float[] rg)
        {
            double dfMean = m_blobDiscountedR.mean(rg);
            double dfStd = m_blobDiscountedR.std(dfMean, rg);
            int nC = m_blobDiscountedR.channels;
 
            // Fill all items in each channel with the same discount value.
            if (nC > 1)
            {
                List<float> rgR = new List<float>();
 
                for (int i = 0; i < rg.Length; i++)
                {
                    for (int j = 0; j < nC; j++)
                    {
                        rgR.Add(rg[i]);
                    }
                }
 
                rg = rgR.ToArray();
            }
 
            m_blobDiscountedR.SetData(Utility.ConvertVec<T>(rg));
            m_blobDiscountedR.NormalizeData(dfMean, dfStd);
        }
 
        public void SetActionProbabilities(float[] rg)
        {
            m_blobPolicyGradient.SetData(Utility.ConvertVec<T>(rg));           
        }
 
        public void SetActionOneHotVectors(float[] rg)
        {
            m_blobActionOneHot.SetData(Utility.ConvertVec<T>(rg));
        }
 
        public void SetData(List<Datum> rgData, List<Datum> rgClip)
        {
            if (m_nRecurrentSequenceLength != 1 && rgData.Count > 1 && rgClip != null)
            {
                m_rgData = rgData;
                m_rgClip = rgClip;
            }
            else
            {
                m_memData.AddDatumVector(rgData, rgClip, 1, true, true);
                m_rgData = null;
                m_rgClip = null;
            }
        }
 
        public GetDataArgs getDataArgs(Phase phase, int nIdx, int nAction, bool? bResetOverride = null)
        {
            bool bReset = (nAction == -1) ? true : false;
            return new GetDataArgs(phase, nIdx, m_mycaffePrimary, m_mycaffePrimary.Log, m_mycaffePrimary.CancelEvent, bReset, nAction, false, true);
        }
 
        public double LastLoss
        {
            get { return m_dfLastLoss; }
        }
 
        public double LearningRate
        {
            get { return m_dfLearningRate; }
        }
 
        public Log Log
        {
            get { return m_mycaffePrimary.Log; }
        }
 
        public CancelEvent Cancel
        {
            get { return m_mycaffePrimary.CancelEvent; }
        }
 
        public SimpleDatum Preprocess(StateBase s, bool bUseRawInput)
        {
            SimpleDatum sd = new SimpleDatum(s.Data, true);
 
            if (bUseRawInput)
                return sd;
 
            if (m_sdLast == null)
                sd.Zero();
            else
                sd.Sub(m_sdLast);
 
            m_sdLast = s.Data;
 
            return sd;
        }
 
        public int act(SimpleDatum sd, SimpleDatum sdClip, out float[] rgfAprob)
        {
            List<Datum> rgData = new List<Datum>();
            rgData.Add(new Datum(sd));
            List<Datum> rgClip = null;
 
            if (sdClip != null)
            {
                rgClip = new List<Datum>();
                rgClip.Add(new Datum(sdClip));
            }
 
            double dfLoss;
            float fRandom = (float)m_random.NextDouble(); // Roll the dice.
 
            m_memData.AddDatumVector(rgData, rgClip, 1, true, true);
            m_bSkipLoss = true;
            BlobCollection<T> res = m_net.Forward(out dfLoss);
            m_bSkipLoss = false;
 
            rgfAprob = null;
 
            for (int i = 0; i < res.Count; i++)
            {
                if (res[i].type != BLOB_TYPE.LOSS)
                {
                    int nStart = 0;
                    // When using recurrent learning, only act on the last outputs.
                    if (m_nRecurrentSequenceLength > 1 && res[i].num > 1)
                    {
                        int nCount = res[i].count();
                        int nOutput = nCount / res[i].num;
                        nStart = nCount - nOutput;
 
                        if (nStart < 0)
                            throw new Exception("The start must be zero or greater!");
                    }
 
                    rgfAprob = Utility.ConvertVecF<T>(res[i].update_cpu_data(), nStart);
                    break;
                }
            }
 
            if (rgfAprob == null)
                throw new Exception("Could not find a non-loss output!  Your model should output the loss and the action probabilities.");
 
            // Select the action from the probability distribution.
            float fSum = 0;
            for (int i = 0; i < rgfAprob.Length; i++)
            {
                fSum += rgfAprob[i];
 
                if (fRandom < fSum)
                    return i;
            }
 
            if (rgfAprob.Length == 1)
                return 1;
 
            return rgfAprob.Length - 1;
        }
 
        private void prepareBlob(Blob<T> b1, Blob<T> b)
        {
            b1.CopyFrom(b, 0, 0, b1.count(), true, true);
            b.Reshape(1, b.channels, b.height, b.width);
        }
 
        private void copyBlob(int nIdx, Blob<T> src, Blob<T> dst)
        {
            int nCount = dst.count();
            dst.CopyFrom(src, nIdx * nCount, 0, nCount, true, false);
        }
 
        public void Train(int nIteration, TRAIN_STEP step, bool bApplyGradients = true)
        {
            // Run data/clip groups > 1 in non batch mode.
            if (m_nRecurrentSequenceLength != 1 && m_rgData != null && m_rgData.Count > 1 && m_rgClip != null)
            {
                prepareBlob(m_blobActionOneHot1, m_blobActionOneHot);
                prepareBlob(m_blobDiscountedR1, m_blobDiscountedR);
                prepareBlob(m_blobPolicyGradient1, m_blobPolicyGradient);
 
                for (int i = 0; i < m_rgData.Count; i++)
                {
                    copyBlob(i, m_blobActionOneHot1, m_blobActionOneHot);
                    copyBlob(i, m_blobDiscountedR1, m_blobDiscountedR);
                    copyBlob(i, m_blobPolicyGradient1, m_blobPolicyGradient);
 
                    List<Datum> rgData1 = new List<Datum>() { m_rgData[i] };
                    List<Datum> rgClip1 = new List<Datum>() { m_rgClip[i] };
 
                    m_memData.AddDatumVector(rgData1, rgClip1, 1, true, true);
                   
                    m_solver.Step(1, step, true, m_bUseAcceleratedTraining, true, true);
                    m_colAccumulatedGradients.Accumulate(m_mycaffeWorker.Cuda, m_net.learnable_parameters, true);
                }
 
                m_blobActionOneHot.ReshapeLike(m_blobActionOneHot1);
                m_blobDiscountedR.ReshapeLike(m_blobDiscountedR1);
                m_blobPolicyGradient.ReshapeLike(m_blobPolicyGradient1);
 
                m_rgData = null;
                m_rgClip = null;
            }
            else
            {
                m_solver.Step(1, step, true, m_bUseAcceleratedTraining, true, true);
                m_colAccumulatedGradients.Accumulate(m_mycaffeWorker.Cuda, m_net.learnable_parameters, true);
            }
 
            if (nIteration % m_nMiniBatch == 0 || bApplyGradients || step == TRAIN_STEP.BACKWARD || step == TRAIN_STEP.BOTH)
            {
                m_net.learnable_parameters.CopyFrom(m_colAccumulatedGradients, true);
                m_colAccumulatedGradients.SetDiff(0);
 
                if (m_mycaffePrimary == m_mycaffeWorker)
                {
                    m_dfLearningRate = m_solver.ApplyUpdate(nIteration);
                }
                else
                {
                    ApplyUpdateArgs<T> args = new ApplyUpdateArgs<T>(nIteration, m_mycaffeWorker);
                    OnApplyUpdate(this, args);
                    m_dfLearningRate = args.LearningRate;
                }
 
                m_net.ClearParamDiffs();
            }
        }
 
        private T[] unpackLabel(Datum d)
        {
            if (d.DataCriteria == null)
                return null;
 
            if (d.DataCriteriaFormat == SimpleDatum.DATA_FORMAT.LIST_FLOAT)
            {
                List<float> rgf = BinaryData.UnPackFloatList(d.DataCriteria, SimpleDatum.DATA_FORMAT.LIST_FLOAT);
                return Utility.ConvertVec<T>(rgf.ToArray());
            }
            else if (d.DataCriteriaFormat == SimpleDatum.DATA_FORMAT.LIST_DOUBLE)
            {
                List<double> rgf = BinaryData.UnPackDoubleList(d.DataCriteria, SimpleDatum.DATA_FORMAT.LIST_DOUBLE);
                return Utility.ConvertVec<T>(rgf.ToArray());
            }
 
            return null;
        }
 
        private void memData_OnDataPack(object sender, MemoryDataLayerPackDataArgs<T> e)
        {            
            List<int> rgDataShape = e.Data.shape();
            List<int> rgClipShape = e.Clip.shape();
            List<int> rgLabelShape = e.Label.shape();
            int nBatch = e.DataItems.Count;
            int nSeqLen = rgDataShape[0];
 
            e.Data.Log.CHECK_GT(nSeqLen, 0, "The sequence lenth must be greater than zero!");
            e.Data.Log.CHECK_EQ(nBatch, e.ClipItems.Count, "The data and clip should have the same number of items.");
            e.Data.Log.CHECK_EQ(nSeqLen, rgClipShape[0], "The data and clip should have the same sequence count.");
 
            rgDataShape[1] = nBatch;  // LSTM uses sizing: seq, batch, data1, data2
            rgClipShape[1] = nBatch;
            rgLabelShape[1] = nBatch;
 
            e.Data.Reshape(rgDataShape);
            e.Clip.Reshape(rgClipShape);
            e.Label.Reshape(rgLabelShape);
 
            T[] rgRawData = new T[e.Data.count()];
            T[] rgRawClip = new T[e.Clip.count()];
            T[] rgRawLabel = new T[e.Label.count()];
 
            int nDataSize = e.Data.count(2);
            T[] rgDataItem = new T[nDataSize];
            T dfClip;
            int nIdx;
 
            for (int i = 0; i < nBatch; i++)
            {
                Datum data = e.DataItems[i];
                Datum clip = e.ClipItems[i];
 
                T[] rgLabel = unpackLabel(data);
 
                for (int j = 0; j < nSeqLen; j++)
                {
                    dfClip = clip.GetDataAt<T>(j);
 
                    for (int k = 0; k < nDataSize; k++)
                    {
                        rgDataItem[k] = data.GetDataAt<T>(j * nDataSize + k);
                    }
 
                    // LSTM: Create input data, the data must be in the order
                    // seq1_val1, seq2_val1, ..., seqBatch_Size_val1, seq1_val2, seq2_val2, ..., seqBatch_Size_valSequence_Length
                    if (e.LstmType == LayerParameter.LayerType.LSTM)
                        nIdx = nBatch * j + i;  
 
                    // LSTM_SIMPLE: Create input data, the data must be in the order
                    // seq1_val1, seq1_val2, ..., seq1_valBatchSize, seq2_val1, seq2_val2, ..., seqSequenceLength_valBatchSize
                    else
                        nIdx = i * nBatch + j;  
 
                    Array.Copy(rgDataItem, 0, rgRawData, nIdx * nDataSize, nDataSize);
                    rgRawClip[nIdx] = dfClip;
 
                    if (rgLabel != null)
                    {
                        if (rgLabel.Length == nSeqLen)
                            rgRawLabel[nIdx] = rgLabel[j];
                        else if (rgLabel.Length == 1)
                        {
                            if (j == nSeqLen - 1)
                                rgRawLabel[0] = rgLabel[0];
                        }
                        else
                        {
                            throw new Exception("The Solver SequenceLength parameter does not match the actual sequence length!  The label length '" + rgLabel.Length.ToString() + "' must be either '1' for SINGLE labels, or the sequence length of '" + nSeqLen.ToString() + "' for MULTI labels.  Stopping training.");
                        }
                    }
                }
            }
 
            e.Data.mutable_cpu_data = rgRawData;
            e.Clip.mutable_cpu_data = rgRawClip;
            e.Label.mutable_cpu_data = rgRawLabel;
            m_nRecurrentSequenceLength = nSeqLen;
        }
 
 
        private void memLoss_OnGetLoss(object sender, MemoryLossLayerGetLossArgs<T> e)
        {
            if (m_bSkipLoss)
                return;
 
            int nCount = m_blobActionOneHot.count();
            long hActionOneHot = m_blobActionOneHot.gpu_data;
            long hPolicyGrad = 0;
            long hDiscountedR = m_blobDiscountedR.gpu_data;
            double dfLoss;
            int nDataSize = e.Bottom[0].count(1);
            bool bUsingEndData = false;
 
            // When using a recurrent model and receiving data with more than one sequence,
            // copy and only use the last sequence data.
            if (m_nRecurrentSequenceLength > 1)
            {
                if (e.Bottom[0].num > 1)
                {
                    m_blobAprobLogit.CopyFrom(e.Bottom[0], false, true);
                    m_blobAprobLogit.CopyFrom(e.Bottom[0], true);
 
                    List<int> rgShape = e.Bottom[0].shape();
                    rgShape[0] = 1;
                    e.Bottom[0].Reshape(rgShape);
                    e.Bottom[0].CopyFrom(m_blobAprobLogit, (m_blobAprobLogit.num - 1) * nDataSize, 0, nDataSize, true, true);
                    bUsingEndData = true;
                }
            }
 
            long hBottomDiff = e.Bottom[0].mutable_gpu_diff;
 
            // Calculate the initial gradients (policy grad initially just contains the action probabilities)
            if (m_softmax != null)
            {
                BlobCollection<T> colBottom = new BlobCollection<T>();
                BlobCollection<T> colTop = new BlobCollection<T>();
 
                colBottom.Add(e.Bottom[0]);             // aprob logit
                colBottom.Add(m_blobActionOneHot);      // action one-hot vectors
                colTop.Add(m_blobLoss);
                colTop.Add(m_blobPolicyGradient);
 
                if (!m_bSoftmaxCeSetup)
                {
                    m_softmaxCe.Setup(colBottom, colTop);
                    m_bSoftmaxCeSetup = true;
                }
 
                dfLoss = m_softmaxCe.Forward(colBottom, colTop);
                m_softmaxCe.Backward(colTop, new List<bool>() { true, false }, colBottom);
                hPolicyGrad = colBottom[0].gpu_diff;
            }
            else
            {
                hPolicyGrad = m_blobPolicyGradient.mutable_gpu_data;
 
                // Calculate (a=0) ? 1-aprob : 0-aprob
                m_mycaffeWorker.Cuda.add_scalar(nCount, -1.0, hActionOneHot); // invert one hot
                m_mycaffeWorker.Cuda.abs(nCount, hActionOneHot, hActionOneHot); 
                m_mycaffeWorker.Cuda.mul_scalar(nCount, -1.0, hPolicyGrad);   // negate Aprob
                m_mycaffeWorker.Cuda.add(nCount, hActionOneHot, hPolicyGrad, hPolicyGrad);  // gradient = ((a=0)?1:0) - Aprob
                dfLoss = Utility.ConvertVal<T>(m_blobPolicyGradient.sumsq_data());
 
                m_mycaffeWorker.Cuda.mul_scalar(nCount, -1.0, hPolicyGrad); // invert for ApplyUpdate subtracts the gradients
            }
 
            // Modulate the gradient with the advantage (PG magic happens right here.)
            m_mycaffeWorker.Cuda.mul(nCount, hPolicyGrad, hDiscountedR, hPolicyGrad);
 
            e.Loss = dfLoss;
            e.EnableLossUpdate = false; // dont apply loss to loss weight.
 
            if (hPolicyGrad != hBottomDiff)
                m_mycaffeWorker.Cuda.copy(nCount, hPolicyGrad, hBottomDiff);
 
            // When using recurrent model with more than one sequence of data, only
            // copy the diff to the last in the sequence and zero out the rest in the sequence.
            if (m_nRecurrentSequenceLength > 1 && bUsingEndData)
            {
                m_blobAprobLogit.SetDiff(0);
                m_blobAprobLogit.CopyFrom(e.Bottom[0], 0, (m_blobAprobLogit.num - 1) * nDataSize, nDataSize, false, true);
                e.Bottom[0].CopyFrom(m_blobAprobLogit, false, true);
                e.Bottom[0].CopyFrom(m_blobAprobLogit, true);
            }
 
            m_dfLastLoss = e.Loss;
        }
    }
 
    class MemoryCache : IEnumerable<Memory>
    {
        int m_nMax;
        List<Memory> m_rgMemory = new List<Memory>();
 
        public MemoryCache(int nMax)
        {
            m_nMax = nMax;
        }
 
        public int Count
        {
            get { return m_rgMemory.Count; }
        }
 
        public Memory this[int nIdx]
        {
            get { return m_rgMemory[nIdx]; }
        }
 
        public bool Add(Memory mem)
        {
            m_rgMemory.Add(mem);
 
            if (m_rgMemory.Count == m_nMax)
                return true;
 
            return false;
        }
 
        public void Clear()
        {
            m_rgMemory.Clear();
        }
 
        public void PurgeNonElite(double dfElitePercent)
        {
            if (dfElitePercent <= 0.0 || dfElitePercent >= 1.0)
                return;
 
            double dfMin = m_rgMemory.Min(p => p.RewardSum);
            double dfMax = m_rgMemory.Max(p => p.RewardSum);
            double dfRange = dfMax - dfMin;
            double dfCutoff = dfMin + ((1.0 - dfElitePercent) * dfRange);
            List<Memory> rgMem = m_rgMemory.OrderByDescending(p => p.RewardSum).ToList();
            List<Memory> rgElite = new List<Memory>();
 
            for (int i = 0; i < rgMem.Count; i++)
            {
                double dfSum = rgMem[i].RewardSum;
 
                if (dfSum >= dfCutoff)
                    rgElite.Add(rgMem[i]);
                else
                    break;
            }
 
            m_rgMemory = rgElite;
        }
 
        public IEnumerator<Memory> GetEnumerator()
        {
            return m_rgMemory.GetEnumerator();
        }
 
        IEnumerator IEnumerable.GetEnumerator()
        {
            return m_rgMemory.GetEnumerator();
        }
    }
 
    class Memory
    {
        List<MemoryItem> m_rgItems = new List<MemoryItem>();
        int m_nEpisodeNumber = 0;
        double m_dfRewardSum = 0;
 
        public Memory()
        {
        }
 
        public int Count
        {
            get { return m_rgItems.Count; }
        }
 
        public void Add(MemoryItem item)
        {
            m_dfRewardSum += item.Reward;
            m_rgItems.Add(item);
        }
 
        public void Clear()
        {
            m_dfRewardSum = 0;
            m_rgItems.Clear();
        }
 
        public MemoryItem this[int nIdx]
        {
            get { return m_rgItems[nIdx]; }
            set { m_rgItems[nIdx] = value; }
        }
 
        public int EpisodeNumber
        {
            get { return m_nEpisodeNumber; }
            set { m_nEpisodeNumber = value; }
        }
 
        public double RewardSum
        {
            get { return m_dfRewardSum; }
            set { m_dfRewardSum = value; }
        }
 
        public float[] GetDiscountedRewards(float fGamma, bool bAllowReset)
        {
            float[] rgR = m_rgItems.Select(p => p.Reward).ToArray();
            float fRunningAdd = 0;
            float[] rgDiscountedR = new float[rgR.Length];
 
            for (int t = Count - 1; t >= 0; t--)
            {
                if (bAllowReset && rgR[t] != 0)
                    fRunningAdd = 0;
 
                fRunningAdd = fRunningAdd * fGamma + rgR[t];
                rgDiscountedR[t] = fRunningAdd;
            }
 
            return rgDiscountedR;
        }
 
        public float[] GetActionProbabilities()
        {
            List<float> rgfAprob = new List<float>();
 
            for (int i = 0; i < m_rgItems.Count; i++)
            {
                rgfAprob.AddRange(m_rgItems[i].Aprob);
            }
 
            return rgfAprob.ToArray();
        }
 
        public float[] GetActionOneHotVectors()
        {
            List<float> rgfAonehot = new List<float>();
 
            for (int i = 0; i < m_rgItems.Count; i++)
            {
                float[] rgfOneHot = new float[m_rgItems[0].Aprob.Length];
 
                if (rgfOneHot.Length == 1)
                    rgfOneHot[0] = m_rgItems[i].Action;
                else
                    rgfOneHot[m_rgItems[i].Action] = 1;
 
                rgfAonehot.AddRange(rgfOneHot);
            }
 
            return rgfAonehot.ToArray();
        }
 
        public List<Datum> GetData()
        {
            List<Datum> rgData = new List<Datum>();
 
            for (int i = 0; i < m_rgItems.Count; i++)
            {
                rgData.Add(new Datum(m_rgItems[i].Data));
            }
 
            return rgData;
        }
 
        public List<Datum> GetClip()
        {
            if (m_rgItems.Count == 0)
                return null;
 
            if (m_rgItems[0].State.Clip == null)
                return null;
 
            List<Datum> rgData = new List<Datum>();
 
            for (int i = 0; i < m_rgItems.Count; i++)
            {
                if (m_rgItems[i].State.Clip == null)
                    return null;
 
                rgData.Add(new Datum(m_rgItems[i].State.Clip));
            }
 
            return rgData;
        }
    }
 
    class MemoryItem
    {
        StateBase m_state;
        SimpleDatum m_x;
        int m_nAction;
        float[] m_rgfAprob;
        float m_fReward;
 
        public MemoryItem(StateBase s, SimpleDatum x, int nAction, float[] rgfAprob, float fReward)
        {
            m_state = s;
            m_x = x;
            m_nAction = nAction;
            m_rgfAprob = rgfAprob;
            m_fReward = fReward;
        }
 
        public StateBase State
        {
            get { return m_state; }
        }
 
        public SimpleDatum Data
        {
            get { return m_x; }
        }
 
        public int Action
        {
            get { return m_nAction; }
        }
 
        public float Reward
        {
            get { return m_fReward; }
        }
 
        public float[] Aprob
        {
            get { return m_rgfAprob; }
        }
 
        public override string ToString()
        {
            return "action = " + m_nAction.ToString() + " reward = " + m_fReward.ToString("N2") + " aprob = " + tostring(m_rgfAprob);
        }
 
        private string tostring(float[] rg)
        {
            string str = "{";
 
            for (int i = 0; i < rg.Length; i++)
            {
                str += rg[i].ToString("N5");
                str += ",";
            }
 
            str = str.TrimEnd(',');
            str += "}";
 
            return str;
        }
    }
}