Database.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Drawing;
using System.Data.SqlClient;
using System.Diagnostics;
using System.Net;
using System.Runtime.InteropServices;
using System.Threading;
using MyCaffe.basecode.descriptors;
using MyCaffe.basecode;
using System.Data.Entity.Infrastructure;
using System.IO;
using System.Threading.Tasks;
 
namespace MyCaffe.db.image
{
    public class Database : IDisposable
    {
        protected Source m_src = null;
        DNNEntities m_entities = null;
        List<Label> m_rgLabelCache;
        protected string m_strPrimaryImgPath = null;
        protected string m_strSecondaryImgPath = null;
        protected bool m_bEnableFileBasedData = false;
        protected int m_nLastIndex = -1;
        object m_objSync = new object();
        Dictionary<int, string> m_rgSecondarySourcePath = new Dictionary<int, string>();
        object m_objRawImgSync = new object();
        Dictionary<string, string> m_rgstrDatabaseFilePath = new Dictionary<string, string>();
        object m_objRemoteSync = new object();
        FileWriter m_fileWriter = null;
 
        public enum FORCE_LOAD
        {
            NONE,
            FROM_FILE,
            FROM_DB
        }
 
        public Database()
        {
        }
 
        public void Dispose()
        {
            if (m_fileWriter != null)
                m_fileWriter.Cancel();
        }
 
        public bool VerifyDataConnection(string strSrc, ConnectInfo ci)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                string strPrimaryPath = m_strPrimaryImgPath;
 
                try
                {
                    int nSrcId = GetSourceID(strSrc, ci);
 
                    List<RawImage> rgImg = entities.RawImages.Where(p => p.SourceID == nSrcId).Take(1).ToList();
                    if (rgImg.Count == 0)
                        throw new Exception("No images found for data source = '" + strSrc + "' on connection: " + ci.ToString("signalpop."));
 
                    byte[] rgRawData = rgImg[0].Data;
                    int nVirtualId = rgImg[0].VirtualID.GetValueOrDefault(0);
                    if (rgRawData == null && nVirtualId > 0)
                    {
                        int nSrcId1 = rgImg[0].OriginalSourceID.GetValueOrDefault(rgImg[0].SourceID.GetValueOrDefault(0));
                        rgImg = entities.RawImages.Where(p => p.ID == nVirtualId).Take(1).ToList();
                        if (rgImg.Count == 0)
                            throw new Exception("No images found for data source = '" + strSrc + "' on connection: " + ci.ToString("signalpop."));
 
                        rgRawData = rgImg[0].Data;
                        strSrc = GetSourceName(nSrcId1);
                    }
 
                    m_strPrimaryImgPath = getImagePathBase(strSrc, entities);
                    byte[] rgData = getRawImage(rgRawData, null, ci, entities, true);
                    if (rgData == null || rgData.Length == 0)
                        throw new Exception("The image at image ID = " + rgImg[0].ID.ToString() + " for data source = '" + strSrc + "' on connection: " + ci.ToString("signalpop.") + " is empty!");
                }
                catch (Exception excpt)
                {
                    throw new Exception("Failed to get raw image data for data source = '" + strSrc + "' on connection: " + ci.ToString("signalpop.") + " with error = '" + excpt.Message + "'", excpt);
                }
                finally
                {
                    m_strPrimaryImgPath = strPrimaryPath;
                }
 
                return true;
            }
        }
 
        public int LastIndex
        {
            get { return m_nLastIndex; }
        }
 
        private string convertWs(string str, char chReplacement)
        {
            string strOut = "";
 
            if (str == null)
                return strOut;
 
            foreach (char ch in str)
            {
                if (char.IsWhiteSpace(ch))
                    strOut += chReplacement;
                else
                    strOut += ch;
            }
 
            return strOut;
        }
 
        public void SaveChanges()
        {
            if (m_entities != null)
                m_entities.SaveChanges();
        }
 
        public Source CurrentSource
        {
            get { return m_src; }
        }
 
        public virtual void Open(int nSrcId, FORCE_LOAD nForceLoad = FORCE_LOAD.NONE, ConnectInfo ci = null)
        {
            m_src = GetSource(nSrcId, ci);
            if (m_src == null)
                throw new Exception("Could not find the source with ID = " + nSrcId.ToString());
 
            m_entities = EntitiesConnection.CreateEntities(ci);
            m_rgLabelCache = loadLabelCache(m_src.ID);
 
            if (ci == null)
                ci = EntitiesConnection.GlobalDatabaseConnectInfo;
 
            if (ci.Location == ConnectInfo.TYPE.AZURE)
                nForceLoad = FORCE_LOAD.FROM_DB;
 
            setImagePath(nForceLoad);
        }
 
        public void Open(string strSrc, bool bForceLoadImageFilePath = false)
        {
            m_src = GetSource(strSrc);
            m_entities = EntitiesConnection.CreateEntities();
            m_rgLabelCache = loadLabelCache(m_src.ID);
 
            setImagePath((bForceLoadImageFilePath) ? FORCE_LOAD.FROM_FILE : FORCE_LOAD.NONE);
        }
 
        protected virtual void setImagePath(FORCE_LOAD nForceLoad)
        {
            if (nForceLoad == FORCE_LOAD.FROM_DB)
            {
                m_bEnableFileBasedData = false;
                return;
            }
 
            m_strPrimaryImgPath = getImagePath();
 
            if (m_src.SaveImagesToFile.GetValueOrDefault(false) || nForceLoad == FORCE_LOAD.FROM_FILE)
            {
                m_bEnableFileBasedData = true;
 
                if (string.IsNullOrEmpty(EntitiesConnection.GlobalDatabaseConnectInfo.Password))
                {
                    if (!Directory.Exists(m_strPrimaryImgPath))
                        Directory.CreateDirectory(m_strPrimaryImgPath);
                }
            }
        }
 
        protected virtual string getImagePath(string strSrcName = null)
        {
            return getImagePathBase(strSrcName);
        }
 
        string getImagePathBase(string strSrcName = null, DNNEntities entities = null)
        {
            if (strSrcName == null)
            {
                if (m_src == null)
                    return null;
 
                strSrcName = m_src.Name;
 
                if (m_src.CopyOfSourceID > 0)
                    strSrcName = GetSourceName(m_src.CopyOfSourceID.GetValueOrDefault());
            }
 
            if (entities == null)
                entities = m_entities;
 
            return GetDatabaseImagePath(entities.Database.Connection.Database) + strSrcName + "\\";
        }
 
        public virtual void Close()
        {
            m_src = null;
 
            if (m_entities != null)
            {
                m_entities.Dispose();
                m_entities = null;
            }
 
            m_strPrimaryImgPath = null;
            m_bEnableFileBasedData = false;
        }
 
        public void Refresh()
        {
            int nSrcId = m_src.ID;
            Close();
            Open(nSrcId);
        }
 
        public string GetDatabaseFilePath(string strName)
        {
            if (EntitiesConnection.GlobalDatabaseConnectInfo.Server == "NONE")
                return "";
 
            if (m_rgstrDatabaseFilePath.ContainsKey(strName))
                return m_rgstrDatabaseFilePath[strName];
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "SELECT physical_name FROM sys.master_files WHERE name = '" + strName + "'";
                DbRawSqlQuery<string> qry = entities.Database.SqlQuery<string>(strCmd);
                List<string> rgStr = qry.ToList();
 
                if (rgStr.Count == 0)
                    return null;
 
                FileInfo fi = new FileInfo(rgStr[0]);
                string strDir = fi.DirectoryName;
 
                string strPath = strDir + "\\";
 
                m_rgstrDatabaseFilePath.Add(strName, strPath);
 
                return strPath;
            }
        }
 
        public string GetDatabaseImagePath(string strName)
        {
            string strDir = GetDatabaseFilePath(strName);
            string strPath = strDir + "Images\\" + strName + "\\";
            return strPath;
        }
 
 
        //---------------------------------------------------------------------
        //  Labels
        //---------------------------------------------------------------------
        #region Labels
 
        private List<Label> loadLabelCache(int nSrcId)
        {
            return m_entities.Labels.Where(p => p.SourceID == nSrcId).ToList();
        }
 
        public void UpdateLabelName(int nLabel, string strName, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Label> rg = entities.Labels.Where(p => p.SourceID == nSrcId && p.ActiveLabel == nLabel).ToList();
                Label l;
 
                if (rg.Count == 0)
                {
                    l = new Label();
                    l.Label1 = nLabel;
                    l.ActiveLabel = nLabel;
                    l.ImageCount = 0;
                    l.SourceID = nSrcId;
 
                    entities.Labels.Add(l);
                }
                else
                {
                    l = rg[0];
                }
 
                l.Name = strName;
 
                entities.SaveChanges();
            }
        }
 
        public Label GetLabel(int nID)
        {
            foreach (Label l in m_rgLabelCache)
            {
                if (l.ID == nID)
                    return l;
            }
 
            return null;
        }
 
        public string GetLabelName(int nLabel, int nSrcId = 0)
        {
            if (nSrcId == 0)
            {
                foreach (Label l in m_rgLabelCache)
                {
                    if (l.Label1 == nLabel)
                        return l.Name;
                }
 
                return null;
            }
            else
            {
                using (DNNEntities entities = EntitiesConnection.CreateEntities())
                {
                    List<Label> rg = entities.Labels.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Label1 == nLabel).ToList();
 
                    if (rg.Count == 0)
                        return null;
 
                    return rg[0].Name;
                }
            }
        }
 
        public Label FindLabelInCache(int nLabel)
        {
            foreach (Label l in m_rgLabelCache)
            {
                if (l.ActiveLabel == nLabel)
                    return l;
            }
 
            return null;
        }
 
        public int GetLabelID(int nLabel)
        {
            foreach (Label l in m_rgLabelCache)
            {
                if (l.ActiveLabel == nLabel)
                    return l.ID;
            }
 
            return 0;
        }
 
        public int GetLabelCount(int nLabel)
        {
            foreach (Label l in m_rgLabelCache)
            {
                if (l.ActiveLabel == nLabel)
                    return l.ImageCount.GetValueOrDefault(0);
            }
 
            return 0;
        }
 
        public void AddLabelToCache(int nLabel)
        {
            Label l = FindLabelInCache(nLabel);
 
            if (l == null)
            {
                l = new Label();
                l.ImageCount = 1;
                l.Name = "";
                l.SourceID = m_src.ID;
                l.Label1 = nLabel;
                l.ActiveLabel = nLabel;
                m_rgLabelCache.Add(l);
            }
            else
            {
                l.ImageCount++;
            }
        }
 
        public void SaveLabelCache()
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                foreach (Label l in m_rgLabelCache)
                {
                    List<Label> rgLabels = entities.Labels.Where(p => p.SourceID == m_src.ID && p.Label1 == l.Label1).ToList();
 
                    if (rgLabels.Count == 0)
                    {
                        if (!l.ActiveLabel.HasValue)
                            l.ActiveLabel = l.Label1;
 
                        entities.Labels.Add(l);
                    }
                }
 
                entities.SaveChanges();
            }
        }
 
        public void UpdateLabelCounts(Dictionary<int, int> rgCounts, DNNEntities entities)
        {
            if (entities == null)
                entities = m_entities;
 
            foreach (Label l in m_rgLabelCache)
            {
                l.ImageCount = 0;
            }
 
            foreach (KeyValuePair<int, int> kv in rgCounts)
            {
                List<Label> rgLabel = m_rgLabelCache.Where(p => p.ActiveLabel == kv.Key).ToList();
 
                if (rgLabel.Count > 0)
                    rgLabel[0].ImageCount = kv.Value;
            }
 
            entities.SaveChanges();
        }
 
        public void LoadLabelCounts(Dictionary<int, int> rgCounts, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Label> rgLabels = entities.Labels.AsNoTracking().Where(p => p.SourceID == nSrcId).ToList();
 
                foreach (Label l in rgLabels)
                {
                    int nCount = entities.RawImages.Where(p => p.SourceID == nSrcId && p.ActiveLabel == l.ActiveLabel && p.Active == true).Count();
                    rgCounts.Add(l.ActiveLabel.GetValueOrDefault(), nCount);
                }
            }
        }
 
        public string GetLabelCountsAsText(int nSrcId = 0, ConnectInfo ci = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            if (ci == null)
                ci = EntitiesConnection.GlobalDatabaseConnectInfo;
 
            if (ci.Location == ConnectInfo.TYPE.AZURE || !string.IsNullOrEmpty(ci.Password))
                return "n\a";
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Label> rgLabels = entities.Labels.AsNoTracking().Where(p => p.SourceID == nSrcId).OrderBy(p => p.Label1).ToList();
 
                string strOut = "(" + rgLabels.Count.ToString() + ") {";
 
                foreach (Label l in rgLabels)
                {
                    strOut += l.ImageCount.GetValueOrDefault().ToString();
                    strOut += ",";
                }
 
                strOut = strOut.TrimEnd(',');
                strOut += "}";
 
                return strOut;
            }
        }
 
        public void UpdateLabelCounts(Dictionary<int, int> rgCounts, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Label> rgLabels = entities.Labels.Where(p => p.SourceID == nSrcId).ToList();
 
                foreach (Label l in rgLabels)
                {
                    l.ImageCount = 0;
                }
 
                foreach (KeyValuePair<int, int> kv in rgCounts)
                {
                    List<Label> rgLabel = rgLabels.Where(p => p.ActiveLabel == kv.Key).ToList();
 
                    if (rgLabel.Count > 0)
                        rgLabel[0].ImageCount = kv.Value;
                }
 
                entities.SaveChanges();
            }
        }
 
        public void UpdateLabelCounts(int nSrcId = 0, int nProjectId = 0, ConnectInfo ci = null)
        {
            Dictionary<int, double> rgLabelBoosts = null;
            double dfTotal = 0;
 
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            if (nProjectId > 0)
                rgLabelBoosts = new Dictionary<int, double>();
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Label> rgLabels = entities.Labels.Where(p => p.SourceID == nSrcId).ToList();
 
                foreach (Label l in rgLabels)
                {
                    int nCount = entities.RawImages.Where(p => p.SourceID == nSrcId && p.ActiveLabel == l.ActiveLabel && p.Active == true).Count();
                    l.ImageCount = nCount;
 
                    if (nProjectId > 0)
                    {
                        rgLabelBoosts.Add(l.ActiveLabel.GetValueOrDefault(0), nCount);
                        dfTotal += nCount;
                    }
                }
 
                entities.SaveChanges();
            }
 
            if (nProjectId > 0)
            {
                if (dfTotal == 0)
                    throw new Exception("There are no images for label boost!");
 
                foreach (int nKey in rgLabelBoosts.Keys)
                {
                    AddLabelBoost(nProjectId, nKey, rgLabelBoosts[nKey] / dfTotal, nSrcId);
                }
            }
        }
 
 
        public List<Label> GetLabels(bool bSort = true, bool bWithImagesOnly = false, int nSrcId = 0, ConnectInfo ci = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Label> rgLabels = entities.Labels.AsNoTracking().Where(p => p.SourceID == nSrcId).ToList();
 
                if (bWithImagesOnly)
                    rgLabels = rgLabels.Where(p => p.ImageCount > 0).ToList();
 
                if (bSort)
                    rgLabels = rgLabels.OrderBy(p => p.Label1).ToList();
 
                return rgLabels;
            }
        }
 
        public void DeleteLabels(int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "DELETE FROM [DNN].[dbo].[Labels] WHERE (SourceID = " + nSrcId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public int AddLabel(int nLabel, string strName = "", int nSrcId = 0, ConnectInfo ci = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Label> rgLabel = entities.Labels.Where(p => p.SourceID == nSrcId && p.Label1 == nLabel).ToList();
                Label l; 
        
                if (rgLabel.Count > 0)
                {
                    l = rgLabel[0];
                }
                else
                {
                    l = new Label();
                    l.Label1 = nLabel;
                    l.SourceID = nSrcId;
                }
 
                l.ActiveLabel = nLabel;
                l.ImageCount = 0;
                l.Name = strName;
 
                if (rgLabel.Count == 0)
                    entities.Labels.Add(l);
 
                entities.SaveChanges();
 
                return l.ID;
            }
        }
 
        public void AddLabelBoost(int nProjectId, int nLabel, double dfBoost, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<LabelBoost> rgLabelBoosts = entities.LabelBoosts.Where(p => p.ProjectID == nProjectId && p.SourceID == nSrcId && p.ActiveLabel == nLabel).ToList();
                LabelBoost lb;
 
                if (rgLabelBoosts.Count > 0)
                {
                    lb = rgLabelBoosts[0];
                }
                else
                {
                    lb = new LabelBoost();
                    lb.ActiveLabel = nLabel;
                    lb.ProjectID = nProjectId;
                    lb.SourceID = nSrcId;
                }
 
                lb.Boost = (decimal)dfBoost;
 
                if (rgLabelBoosts.Count == 0)
                    entities.LabelBoosts.Add(lb);
 
                entities.SaveChanges();
            }
        }
 
        public void SetLabelMapping(LabelMapping map, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.CommandTimeout = 180;
                string strCmd = "UPDATE [DNN].[dbo].[RawImages] SET [ActiveLabel] = " + map.NewLabel.ToString() + " WHERE (SourceID = " + nSrcId.ToString() + ")";
                strCmd += " AND (OriginalLabel = " + map.OriginalLabel.ToString() + ")";
 
                if (map.ConditionBoostEquals.HasValue)
                    strCmd += " AND (ActiveBoost = " + map.ConditionBoostEquals.Value.ToString() + ")";
 
                entities.Database.ExecuteSqlCommand(strCmd);
 
                if (map.ConditionBoostEquals.HasValue)
                {
                    if (map.NewLabelConditionFalse.HasValue)
                    {
                        strCmd = "UPDATE [DNN].[dbo].[RawImages] SET [ActiveLabel] = " + map.NewLabelConditionFalse.Value.ToString() + " WHERE (SourceID = " + nSrcId.ToString() + ")";
 
                        strCmd += " AND (OriginalLabel = " + map.OriginalLabel.ToString() + ")";
                        strCmd += " AND (ActiveBoost != " + map.ConditionBoostEquals.Value.ToString() + ")";
                        entities.Database.ExecuteSqlCommand(strCmd);
                    }
                }
            }
        }
 
        public void UpdateLabelMapping(int nNewLabel, List<int> rgOriginalLabels, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "UPDATE [DNN].[dbo].[RawImages] SET [ActiveLabel] = " + nNewLabel.ToString() + " WHERE (SourceID = " + nSrcId.ToString() + ") AND (";
 
                for (int i = 0; i < rgOriginalLabels.Count; i++)
                {
                    strCmd += "OriginalLabel = " + rgOriginalLabels[i].ToString();
 
                    if (i < rgOriginalLabels.Count - 1)
                        strCmd += " OR ";
                }
 
                strCmd += ")";
 
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public void ResetLabels(int nProjectId = 0, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.CommandTimeout = 180;
 
                string strResetCmd = "UPDATE [DNN].[dbo].[RawImages] SET [ActiveLabel] = [OriginalLabel] WHERE (SourceID = " + nSrcId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strResetCmd);
 
                if (nProjectId > 0)
                    DeleteLabelBoosts(nProjectId, nSrcId);
            }
        }
 
        public void DeleteLabelBoosts(int nProjectId, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "DELETE FROM [DNN].[dbo].[LabelBoosts] WHERE (ProjectID = " + nProjectId.ToString() + ") AND (SourceID = " + nSrcId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public void DeleteLabelBoosts(int nProjectId)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "DELETE FROM [DNN].[dbo].[LabelBoosts] WHERE (ProjectID = " + nProjectId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public void ResetLabelBoosts(int nProjectId)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<LabelBoost> rgLabels = entities.LabelBoosts.Where(p => p.ProjectID == nProjectId).ToList();
 
                foreach (LabelBoost l in rgLabels)
                {
                    l.Boost = 1;
                }
 
                entities.SaveChanges();
            }
        }
 
        public List<LabelBoost> GetLabelBoosts(int nProjectId, bool bSort = true, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<LabelBoost> rgBoosts = entities.LabelBoosts.AsNoTracking().Where(p => p.ProjectID == nProjectId && p.SourceID == nSrcId).ToList();
 
                if (bSort)
                    rgBoosts = rgBoosts.OrderBy(p => p.ActiveLabel).ToList();
 
                return rgBoosts;
            }
        }
 
        public string GetLabelBoostsAsText(int nProjectId, int nSrcId = 0, bool bSort = true)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            List<LabelBoost> rgLb = GetLabelBoosts(nProjectId, bSort, nSrcId);
            string strOut = "";
 
            foreach (LabelBoost lb in rgLb)
            {
                strOut += lb.Boost.GetValueOrDefault().ToString("N2");
                strOut += ",";
            }
 
            return strOut.TrimEnd(',');
        }
 
        public void ActivateLabels(List<int> rgLabels, bool bActive, params int[] rgSrcId)
        {
            string strSQL = "UPDATE [dbo].[RawImages] SET [Active] = " + ((bActive) ? "1" : "0") + " WHERE (";
 
            for (int i = 0; i < rgSrcId.Length; i++)
            {
                strSQL += "(SourceID = " + rgSrcId[i].ToString() + ")";
 
                if (i < rgSrcId.Length - 1)
                    strSQL += " OR ";
            }
 
            strSQL += ") AND (";
 
            for (int i = 0; i < rgLabels.Count; i++)
            {
                strSQL += "(ActiveLabel = " + rgLabels[i].ToString() + ")";
 
                if (i < rgLabels.Count - 1)
                    strSQL += " OR ";
            }
 
            strSQL += ")";
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.ExecuteSqlCommand(strSQL);
            }
        }
 
        public void UpdateLabelBoost(int? nTgtLbl, bool bTgtLblExact, int? nTgtBst, bool bTgtBstExact, int? nNewLbl, int? nNewBst, params int[] rgSrcId)
        {
            if (rgSrcId.Length == 0)
                throw new Exception("You must specify at least one source ID!");
 
            string strSQL = "UPDATE [dbo].[RawImages] SET [Active] = 1";
 
            if (nNewLbl.HasValue)
                strSQL += ", [ActiveLabel] = " + nNewLbl.Value.ToString();
 
            if (nNewBst.HasValue)
                strSQL += ", [ActiveBoost] = " + nNewBst.Value.ToString();
 
            strSQL += " WHERE (";
 
            for (int i = 0; i < rgSrcId.Length; i++)
            {
                strSQL += "([SourceID] = " + rgSrcId[i].ToString() + ")";
 
                if (i < rgSrcId.Length - 1)
                    strSQL += " OR ";
            }
 
            if (nTgtBst.HasValue || nTgtLbl.HasValue)
                strSQL += ") AND (";
 
            if (nTgtLbl.HasValue)
            {
                strSQL += "(";
                strSQL += "[ActiveLabel] ";
                strSQL += (bTgtLblExact) ? "=" : ">=";
                strSQL += nTgtLbl.Value.ToString();
                strSQL += ")";
            }
 
            if (nTgtBst.HasValue)
            {
                if (nTgtLbl.HasValue)
                    strSQL += " OR ";
 
                strSQL += "(";
                strSQL += "[ActiveBoost] ";
                strSQL += (bTgtBstExact) ? "=" : ">=";
                strSQL += nTgtBst.Value.ToString();
                strSQL += ")";
            }
 
            strSQL += ")";
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.ExecuteSqlCommand(strSQL);
            }
        }
 
        #endregion
 
 
        //---------------------------------------------------------------------
        //  Images
        //---------------------------------------------------------------------
        #region RawImages
 
        public int GetImageCount()
        {
            return m_entities.RawImages.AsNoTracking().Where(p => p.SourceID == m_src.ID).Count();
        }
 
        public List<DbItem> GetAllRawImageIndexes(bool bBoostedOnly, bool bIncludeActive = true, bool bIncludeInactive = false)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                int nSrcID = m_src.ID;
                IQueryable<RawImage> iQuery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcID);
 
                if (bIncludeActive && !bIncludeInactive)
                    iQuery = iQuery.Where(p => p.Active == true);
                else if (!bIncludeActive && bIncludeInactive)
                    iQuery = iQuery.Where(p => p.Active == false);
 
                if (bBoostedOnly)
                    iQuery = iQuery.Where(p => p.ActiveBoost > 0);
 
                iQuery = iQuery.OrderBy(p => p.Idx);
 
                return iQuery.Select(p => new DbItem { id = p.ID, virtualid = p.VirtualID, index = p.Idx, label = p.ActiveLabel, boost = p.ActiveBoost, time = p.TimeStamp, desc = p.Description, originalsrcid = p.OriginalSourceID, active = p.Active }).ToList();
            }
        }
 
        public List<RawImage> QueryRawImages(params int[] rgSrcId)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<int?> rgSrcId1 = new List<int?>();
 
                foreach (int id in rgSrcId)
                {
                    rgSrcId1.Add(id);
                }
 
                return entities.RawImages.AsNoTracking().Where(p => rgSrcId1.Contains(p.SourceID)).ToList();   
            }
        }
 
        public List<RawImageParameter> QueryRawImageParameters(int nSrcId, string strName)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImageParameter> iQuery = entities.RawImageParameters.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Name == strName);
                return iQuery.ToList();
            }
        }
 
        public List<RawImageParameter> QueryRawImageParameters(int nImageID)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImageParameter> iQuery = entities.RawImageParameters.AsNoTracking().Where(p => p.RawImageID == nImageID);
                return iQuery.ToList();
            }
        }
 
 
        public List<RawImage> QueryRawImages(int nSrcId, bool? bActive = null, int nBoostVal = 0, bool bExactBoostVal = false)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImage> iQuery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId);
 
                if (bActive.HasValue)
                {
                    bool bVal = bActive.Value;
                    iQuery = iQuery.Where(p => p.Active == bVal);
                }
 
                if (nBoostVal > 0)
                {
                    if (bExactBoostVal)
                        iQuery = iQuery.Where(p => p.ActiveBoost == nBoostVal);
                    else
                        iQuery = iQuery.Where(p => p.ActiveBoost >= nBoostVal);
                }
 
                return iQuery.ToList();
            }
        }
 
        public List<RawImage> GetRawImagesAt(int nIdx, int nCount, int nSrcId = 0, string strDescription = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            string strCmd = "SELECT TOP " + nCount.ToString() + " * FROM [DNN].[dbo].[RawImages] WHERE (SourceID = " + nSrcId.ToString() + ") AND (Idx >= " + nIdx.ToString() + ") AND (Active = 1)";
 
            if (!String.IsNullOrEmpty(strDescription))
                strCmd += " AND (Description = " + strDescription + ")";
 
            strCmd += " ORDER BY Idx";
 
            return m_entities.Database.SqlQuery<RawImage>(strCmd).ToList();
        }
 
        public List<RawImage> GetRawImagesAt(List<int> rgImageIdx, int nSrcId = 0, string strDescription = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            if (rgImageIdx.Count > 100)
                throw new Exception("You can only query up to 100 images at a time when using the list of image indexes.");
 
            string strCmd = "SELECT * FROM [DNN].[dbo].[RawImages] WHERE (SourceID = " + nSrcId.ToString() + ") AND (";
 
            for (int i = 0; i < rgImageIdx.Count; i++)
            {
                strCmd += "Idx = " + rgImageIdx[i].ToString();
 
                if (i < rgImageIdx.Count-1)
                    strCmd += " OR ";
            }
 
            strCmd += ") AND (Active = 1)";
 
            if (!String.IsNullOrEmpty(strDescription))
                strCmd += " AND (Description = " + strDescription + ")";
 
            return m_entities.Database.SqlQuery<RawImage>(strCmd).ToList();
        }
 
 
        public List<RawImage> GetRawImagesAtID(List<int> rgImageID, int nSrcId = 0, string strDescription = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            if (rgImageID.Count > 100)
                throw new Exception("You can only query up to 100 images at a time when using the list of image indexes.");
 
            string strCmd = "SELECT * FROM [DNN].[dbo].[RawImages] WHERE (SourceID = " + nSrcId.ToString() + ") AND (";
 
            for (int i = 0; i < rgImageID.Count; i++)
            {
                strCmd += "ID = " + rgImageID[i].ToString();
 
                if (i < rgImageID.Count - 1)
                    strCmd += " OR ";
            }
 
            strCmd += ") AND (Active = 1)";
 
            if (!String.IsNullOrEmpty(strDescription))
                strCmd += " AND (Description = " + strDescription + ")";
 
            lock (m_objRemoteSync)
            {
                return m_entities.Database.SqlQuery<RawImage>(strCmd).ToList();
            }
        }
 
        public RawImage GetRawImageAt(int nIdx, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            string strCmd = "SELECT TOP 1 * FROM [DNN].[dbo].[RawImages] WHERE (SourceID = " + nSrcId.ToString() + ") AND (Idx = " + nIdx.ToString() + ") AND (Active = 1)";
            List<RawImage> rgImg = null;
 
            lock (m_objSync)
            {
                rgImg = m_entities.Database.SqlQuery<RawImage>(strCmd).ToList();
            }
 
            if (rgImg == null || rgImg.Count == 0)
                return null;
 
            return rgImg[0];
        }
 
        public int GetRawImageID(DateTime dt, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId && p.TimeStamp == dt && p.Active == true).ToList();
 
                if (rgImg.Count == 0)
                    return 0;
 
                return rgImg[0].ID;
            }
        }
 
        public byte[] GetRawImageData(RawImage img, bool bLoadDataCriteria, bool bLoadDebugData, out byte[] rgDataCriteria, out int? nDataCriteriaFmtId, out byte[] rgDebugData, out int? nDebugDataFmtId)
        {
            if (img.VirtualID == 0)
            {
                rgDataCriteria = (bLoadDataCriteria) ? getRawImage(img.DataCriteria, img.OriginalSourceID) : img.DataCriteria;
                nDataCriteriaFmtId = img.DataCriteriaFormatID;
                rgDebugData = (bLoadDebugData) ? getRawImage(img.DebugData, img.OriginalSourceID) : img.DebugData;
                nDebugDataFmtId = img.DebugDataFormatID;
                return getRawImage(img.Data, img.OriginalSourceID);
            }
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.AsNoTracking().Where(p => p.ID == img.VirtualID).ToList();
 
                if (rgImg.Count == 0)
                {
                    rgDataCriteria = null;
                    nDataCriteriaFmtId = null;
                    rgDebugData = null;
                    nDebugDataFmtId = null;
                    return null;
                }
 
                rgDataCriteria = (bLoadDataCriteria) ? getRawImage(rgImg[0].DataCriteria, img.OriginalSourceID, null, entities) : rgImg[0].DataCriteria;
                nDataCriteriaFmtId = rgImg[0].DataCriteriaFormatID;
                rgDebugData = (bLoadDebugData) ? getRawImage(rgImg[0].DebugData, img.OriginalSourceID, null, entities) : rgImg[0].DebugData;
                nDebugDataFmtId = rgImg[0].DebugDataFormatID;
 
                return getRawImage(rgImg[0].Data, img.OriginalSourceID, null, entities);
            }
        }
 
        public byte[] GetRawImageDataCriteria(RawImage img, out int? nDataCriteriaFmtId)
        {
            if (img.VirtualID == 0)
            {
                nDataCriteriaFmtId = img.DataCriteriaFormatID;
                return getRawImage(img.DataCriteria, img.OriginalSourceID);
            }
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.AsNoTracking().Where(p => p.ID == img.VirtualID).ToList();
 
                if (rgImg.Count == 0)
                {
                    nDataCriteriaFmtId = null;
                    return null;
                }
 
                nDataCriteriaFmtId = rgImg[0].DataCriteriaFormatID;
                return getRawImage(rgImg[0].DataCriteria, img.OriginalSourceID, null, entities);
            }
        }
 
        public byte[] GetRawImageDebugData(RawImage img, out int? nDebugDataFormatId)
        {
            if (img.VirtualID == 0)
            {
                nDebugDataFormatId = img.DebugDataFormatID;
                return getRawImage(img.DebugData, img.OriginalSourceID);
            }
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.AsNoTracking().Where(p => p.ID == img.VirtualID).ToList();
 
                if (rgImg.Count == 0)
                {
                    nDebugDataFormatId = null;
                    return null;
                }
 
                nDebugDataFormatId = rgImg[0].DebugDataFormatID;
                return getRawImage(rgImg[0].DebugData, img.OriginalSourceID, null, entities);
            }
        }
 
        public byte[] GetRawImageDebugData(byte[] rgData, int? nOriginalSourceID = null)
        {
            return getRawImage(rgData, nOriginalSourceID);
        }
 
        public byte[] GetRawImageDebugData(int nImgID, int? nOriginalSourceID = null)
        {
            byte[] rgData = null;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                var data = entities.RawImages.AsNoTracking().Where(p => p.ID == nImgID).Select(p => new { p.DebugData, p.VirtualID }).FirstOrDefault();
                if (data.DebugData == null)
                {
                    nImgID = data.VirtualID.GetValueOrDefault(0);
                    if (nImgID == 0)
                        return null;
 
                    rgData = entities.RawImages.AsNoTracking().Where(p => p.ID == nImgID).Select(p => p.DebugData).FirstOrDefault();
                    if (rgData == null)
                        return null;
                }
                else
                {
                    rgData = data.DebugData;
                }
            }
 
            return getRawImage(rgData, nOriginalSourceID);
        }
 
        public byte[] GetRawImageDataCriteria(byte[] rgData, int? nOriginalSourceID = null)
        {
            return getRawImage(rgData, nOriginalSourceID);
        }
 
        public byte[] GetRawImageDataCriteria(int nImgID, int? nOriginalSourceID = null)
        {
            byte[] rgData = null;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                var data = entities.RawImages.AsNoTracking().Where(p => p.ID == nImgID).Select(p => new { p.DataCriteria, p.VirtualID }).FirstOrDefault();
                if (data.DataCriteria == null)
                {
                    nImgID = data.VirtualID.GetValueOrDefault(0);
                    if (nImgID == 0)
                        return null;
 
                    rgData = entities.RawImages.AsNoTracking().Where(p => p.ID == nImgID).Select(p => p.DataCriteria).FirstOrDefault();
                    if (rgData == null)
                        return null;
                }
                else
                {
                    rgData = data.DataCriteria;
                }
            }
 
            return getRawImage(rgData, nOriginalSourceID);
        }
 
        protected byte[] getRawImage(byte[] rgData, int? nSecondarySrcId = null, ConnectInfo ci = null, DNNEntities entities = null, bool bTestConnection = false)
        {
            if (rgData == null || rgData.Length < 5)
                return rgData;
 
            string strFile = getImagePath(rgData);
            if (strFile == null)
                return rgData;
 
            if (m_strPrimaryImgPath == null)
                throw new Exception("You must open the database on a datasource.");
 
            // Get the file.
            if (nSecondarySrcId == null)
            {
                if (isRemote(ci))
                {
                    string strFile1 = m_strPrimaryImgPath + strFile;
 
                    if (bTestConnection)
                    {
                        string strDbPath = GetDatabaseFilePath("DNN");
                        int nIdx = strFile1.IndexOf(strDbPath);
                        if (nIdx == 0)
                            strFile1 = strFile1.Substring(strDbPath.Length);
                    }
 
                    return getRemoteData(strFile1, entities);
                }
                else
                {
                    return File.ReadAllBytes(m_strPrimaryImgPath + strFile);
                }
            }
 
            string strPath = m_strPrimaryImgPath;
 
            if (!File.Exists(strPath + strFile))
            {                
                lock (m_objRawImgSync)
                {
                    int nSecondarySrcId1 = nSecondarySrcId.Value;
 
                    if (m_rgSecondarySourcePath.ContainsKey(nSecondarySrcId1))
                    {
                        m_strSecondaryImgPath = m_rgSecondarySourcePath[nSecondarySrcId1];
                    }
                    else
                    {
                        m_strSecondaryImgPath = getImagePath(GetSourceName(nSecondarySrcId1));
                        m_rgSecondarySourcePath.Add(nSecondarySrcId1, m_strSecondaryImgPath);
                    }
                }
 
                strPath = m_strSecondaryImgPath;
            }
 
            try
            {
                if (isRemote(ci))
                    return getRemoteData(strPath + strFile, entities);
                else
                    return File.ReadAllBytes(strPath + strFile);
            }
            catch (Exception excpt)
            {
                throw excpt;
            }
        }
 
        private bool isRemote(ConnectInfo ci)
        {
            if (ci == null)
                ci = EntitiesConnection.GlobalDatabaseConnectInfo;
 
            if (string.IsNullOrEmpty(ci.Password))
                return false;
            else
                return true;
        }
 
        private byte[] getRemoteData(string strInfo, DNNEntities entities = null)
        {
            if (entities == null)
                entities = m_entities;
 
            string strCmd = "EXEC [dbo].[GetRawData] @strInfo = N'" + strInfo + "'";
            DbRawSqlQuery<byte[]> qry = entities.Database.SqlQuery<byte[]>(strCmd);
 
            lock (m_objRemoteSync)
            {
                return qry.Single();
            }
        }
 
        protected string getImagePath(byte[] rgData)
        {
            if (rgData == null)
                return null;
 
            if (rgData.Length < 5)
                return null;
 
            if (rgData[0] != 'F' ||
                rgData[1] != 'I' ||
                rgData[2] != 'L' ||
                rgData[3] != 'E' ||
                rgData[4] != ':')
                return null;
 
            return Encoding.ASCII.GetString(rgData, 5, rgData.Length - 5);
        }
 
        public bool ChangeRawImageSourceID(int nID, int nNewSrcID, bool bSave = true)
        {
            List<RawImage> rg = m_entities.RawImages.Where(p => p.ID == nID).ToList();
 
            if (rg.Count == 0)
                return false;
 
            if (rg[0].VirtualID == 0)
                return false;
 
            rg[0].SourceID = nNewSrcID;
 
            if (bSave)
                m_entities.SaveChanges();
 
            return true;
        }
 
        public bool UpdateActiveLabel(int nID, int nLabel, bool bActivate = true, bool bSaveChanges = true)
        {
            List<RawImage> rg = m_entities.RawImages.Where(p => p.ID == nID).ToList();
 
            if (rg.Count == 0)
                return false;
 
            rg[0].ActiveLabel = nLabel;
            rg[0].Active = bActivate;
 
            if (bSaveChanges)
                m_entities.SaveChanges();
 
            return true;
        }
 
        public void UpdateActiveLabelByID(int nID, int nLabel)
        {
            string strCmd = "UPDATE [dbo].[RawImages] SET [ActiveLabel] = " + nLabel.ToString() + ",[Active] = 1";
            strCmd += " WHERE ([ID] = " + nID.ToString() + ")";
 
            m_entities.Database.ExecuteSqlCommand(strCmd);
        }
 
        public void UpdateActiveLabelByIndex(int nSrcId, int nIdx, int nLabel)
        {
            string strCmd = "UPDATE [dbo].[RawImages] SET [ActiveLabel] = " + nLabel.ToString() + ",[Active] = 1";
            strCmd += " WHERE ([SourceID] = " + nSrcId.ToString() + ") AND ([Idx] = " + nIdx.ToString() + ")";
 
            m_entities.Database.ExecuteSqlCommand(strCmd);
        }
 
        public void UpdateAllActiveLabels(int nSrcId, int nLabel, int? nOriginalLabel)
        {
            string strCmd = "UPDATE [dbo].[RawImages] SET [ActiveLabel] = " + nLabel.ToString() + ",[Active] = 1";
            strCmd += " WHERE ([SourceID] = " + nSrcId.ToString() + ")";
 
            if (nOriginalLabel.HasValue)
                strCmd += " AND ([OriginalLabel] = " + nOriginalLabel.Value.ToString() + ")";
 
            m_entities.Database.ExecuteSqlCommand(strCmd);
        }
 
        public void ResetAllActiveLabels(int nSrcId)
        {
            string strCmd = "UPDATE [dbo].[RawImages] SET [ActiveLabel] = [OriginalLabel], [Active] = 1";
            strCmd += " WHERE ([SourceID] = " + nSrcId.ToString() + ")";
            m_entities.Database.ExecuteSqlCommand(strCmd);
        }
 
        public void DisableLabel(int nSrcId, int nLabel, bool bOriginalLabel = false)
        {
            string strCmd = "UPDATE [dbo].[RawImages] SET [Active] = 0";
            strCmd += " WHERE ([SourceID] = " + nSrcId.ToString() + ")";
 
            if (bOriginalLabel)
                strCmd += " AND ([OriginalLabel] = " + nLabel.ToString() + ")";
            else
                strCmd += " AND ([ActiveLabel] = " + nLabel.ToString() + ")";
 
            m_entities.Database.ExecuteSqlCommand(strCmd);
        }
 
        public void DisableAllLabels(int nSrcId)
        {
            string strCmd = "UPDATE [dbo].[RawImages] SET [Active] = 0";
            strCmd += " WHERE ([SourceID] = " + nSrcId.ToString() + ")";
            m_entities.Database.ExecuteSqlCommand(strCmd);
        }
 
        public int DisableAllNonMatchingImages(int nSrcId, int nWidth, int nHeight)
        {
            string strCmd = "SELECT COUNT([ID]) FROM[DNN].[dbo].[RawImages] WHERE(SourceID = " + nSrcId.ToString() + ") AND (Width != " + nWidth.ToString() + " OR Height != " + nHeight.ToString() + ")";
            DbRawSqlQuery<int> qry = m_entities.Database.SqlQuery<int>(strCmd);
            List<int> rgVal = qry.ToList();
            int nCount = rgVal[0];
 
            strCmd = "UPDATE [dbo].[RawImages] SET [Active] = 0 WHERE(SourceID = " + nSrcId.ToString() + ") AND (Width != " + nWidth.ToString() + " OR Height != " + nHeight.ToString() + ")";
            m_entities.Database.ExecuteSqlCommand(strCmd);
 
            return nCount;
        }
 
        public void ActivateRawImageByIndex(int nSrcId, int nIdx, bool bActive)
        {
            string strCmd = "UPDATE [dbo].[RawImages] SET [Active] = " + ((bActive) ? "1" : "0");
            strCmd += " WHERE ([SourceID] = " + nSrcId.ToString() + ") AND ([Idx] = " + nIdx.ToString() + ")";
 
            m_entities.Database.ExecuteSqlCommand(strCmd);
        }
 
 
        public void UpdateActiveLabelDirect(int nID, int nLabel)
        {
            string strSql = "UPDATE [dbo].[RawImages] SET [ActiveLabel] = " + nLabel.ToString() + ",[Active] = 1 WHERE (ID = " + nID.ToString() + ")";
 
            if (m_entities != null)
            {
                m_entities.Database.ExecuteSqlCommand(strSql);
                return;
            }
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.ExecuteSqlCommand(strSql);
            }
        }
 
        public bool UpdateRawImageDescription(int nID, string strDescription)
        {
            List<RawImage> rg = m_entities.RawImages.Where(p => p.ID == nID).ToList();
 
            if (rg.Count == 0)
                return false;
 
            rg[0].Description = strDescription;
            m_entities.SaveChanges();
 
            return true;
        }
 
        public bool UpdateRawImageDescriptionFromParameter(int nID, string strParamName)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rg = entities.RawImages.Where(p => p.ID == nID).ToList();
 
                if (rg.Count == 0)
                    return false;
 
                List<RawImageParameter> rgP = entities.RawImageParameters.Where(p => p.RawImageID == nID && p.Name == strParamName).ToList();
 
                if (rgP.Count == 0)
                    return false;
 
                if (rgP[0].TextValue == null || rgP[0].TextValue.Length == 0)
                    return false;
 
                if (rg[0].Description == rgP[0].TextValue)
                    return false;
 
                rg[0].Description = rgP[0].TextValue;
                entities.SaveChanges();
 
                return true;
            }
        }
 
        public List<int> QueryAllRawImageIDs(int nSrcId = 0, int nMax = int.MaxValue, int nLabel = -1, int nBoost = -1, bool bBoostIsExact = false, bool bAnnotatedOnly = false, bool bActiveOnly = true)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strTop = (nMax == int.MaxValue) ? "" : "TOP " + nMax.ToString();
                string strCmd = "SELECT " + strTop + " ID FROM RawImages WHERE (SourceID = " + nSrcId.ToString() + ")";
 
                if (nLabel != -1)
                    strCmd += " AND (ActiveLabel = " + nLabel.ToString() + ")";
 
                if (nBoost != -1)
                {
                    string strCompare = (bBoostIsExact) ? "=" : ">=";
                    strCmd += " AND (ActiveBoost " + strCompare + " " + nBoost.ToString() + ")";
                }
 
                if (bAnnotatedOnly)
                    strCmd += " AND (DataCriteriaFormatID = " + ((int)SimpleDatum.DATA_FORMAT.ANNOTATION_DATA).ToString() + ")";
 
                if (bActiveOnly)
                    strCmd += " AND (Active = 1)";
 
                return entities.Database.SqlQuery<int>(strCmd).ToList();
            }
        }
 
        public RawImage CreateRawImage(int nIdx, SimpleDatum d, int nBackgroundWritingThreadCount, string strDescription = null, int? nOriginalSourceID = null, bool bActive = true)
        {
            if (string.IsNullOrEmpty(strDescription))
                strDescription = d.Description;
 
            DateTime dtMin = new DateTime(1980, 1, 1);
            RawImage img = new RawImage();
            bool bEncoded = false;
            img.Channels = d.Channels;
            img.Height = d.Height;
            img.Width = d.Width;
            img.SourceID = m_src.ID;
            img.TimeStamp = (d.TimeStamp < dtMin) ? dtMin : d.TimeStamp;
            img.Idx = nIdx;
            img.OriginalBoost = (short)d.Boost;
            img.ActiveBoost = (short)d.Boost;
            img.GroupID = d.GroupID;
            img.ActiveLabel = d.Label;
            img.OriginalLabel = d.Label;
            img.SourceID = m_src.ID;
            img.Active = bActive;
            img.AutoLabel = d.AutoLabeled;
            img.Description = strDescription;
 
            if (nOriginalSourceID.HasValue)
                img.OriginalSourceID = nOriginalSourceID.Value;
 
            string strGuid = Guid.NewGuid().ToString();
 
            if (d.VirtualID > 0)
            {
                img.VirtualID = d.VirtualID;
                img.Encoded = d.IsRealData;
            }
            else
            {
                img.VirtualID = 0;
                img.Data = setImageByteData(d.GetByteData(out bEncoded), null, strGuid, nBackgroundWritingThreadCount);
                img.Encoded = bEncoded;
            }
 
            if (d.DebugData != null)
            {
                img.DebugData = setImageByteData(d.DebugData, "dbg", strGuid);
                img.DebugDataFormatID = (byte)d.DebugDataFormat;
            }
 
            if (d.annotation_group != null || d.annotation_type != SimpleDatum.ANNOTATION_TYPE.NONE)
                d.SaveAnnotationDataToDataCriteria();
 
            if (d.DataCriteria != null)
            {
                img.DataCriteria = setImageByteData(d.DataCriteria, "criteria", strGuid);
                img.DataCriteriaFormatID = (byte)d.DataCriteriaFormat;
            }
 
            return img;
        }
 
        public bool WaitForFileWriter(int nWait = int.MaxValue)
        {
            if (m_fileWriter == null)
                return true;
 
            return m_fileWriter.WaitUntilCompletion(nWait);
        }
 
        protected byte[] setImageByteData(byte[] rgImg, string strType = null, string strGuid = null, int nBackgroundWritingThreadCount = 0)
        {
            if (rgImg == null)
                return null;
 
            if (!m_bEnableFileBasedData || rgImg.Length < 100)
                return rgImg;
 
            if (strGuid == null)
                strGuid = Guid.NewGuid().ToString();
 
            if (!Directory.Exists(m_strPrimaryImgPath))
                Directory.CreateDirectory(m_strPrimaryImgPath);
 
            string strTypeExt = (strType == null) ? "" : "." + strType;
            string strFile = strGuid + strTypeExt + ".bin";
 
            if (nBackgroundWritingThreadCount > 0)
            {
                if (m_fileWriter == null)
                    m_fileWriter = new FileWriter(nBackgroundWritingThreadCount);
                m_fileWriter.Add(m_strPrimaryImgPath + strFile, rgImg);
            }
            else
            {
                File.WriteAllBytes(m_strPrimaryImgPath + strFile, rgImg);
            }
 
            string strTag = "FILE:" + strFile;
            return Encoding.ASCII.GetBytes(strTag);
        }
 
        private void fileWriterThread()
        {
        }
 
        public bool ConvertRawImagesSaveToFile(int nIdx, int nCount, CancelEvent evtCancel = null)
        {
            if (m_strPrimaryImgPath == null)
                m_strPrimaryImgPath = getImagePath();
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.Where(p => p.SourceID == m_src.ID && p.Idx >= nIdx && p.Active == true).OrderBy(p => p.Idx).Take(nCount).ToList();
                string strPath;
                string strImgPath;
                string strTag;
                string strName;
                byte[] rgData;
                List<int?> rgId = new List<int?>();
                Dictionary<int, string> rgNames = new Dictionary<int, string>();
 
                for (int i=0; i<rgImg.Count; i++)
                {
                    if (evtCancel != null && evtCancel.WaitOne(0))
                        return false;
 
                    strName = Guid.NewGuid().ToString();
                    rgNames.Add(rgImg[i].ID, strName);
 
                    rgId.Add(rgImg[i].ID);
                    rgData = rgImg[i].Data;
                    if (rgData != null)
                    {
                        strPath = getImagePath(rgData);
                        if (strPath == null)
                        {
                            strImgPath = strName + ".bin";
                            File.WriteAllBytes(m_strPrimaryImgPath + strImgPath, rgData);
                            strTag = "FILE:" + strImgPath;
                            rgImg[i].Data = Encoding.ASCII.GetBytes(strTag);
                        }
                    }
 
                    rgData = rgImg[i].DebugData;
                    if (rgData != null)
                    {
                        strPath = getImagePath(rgData);
                        if (strPath == null)
                        {
                            strImgPath = strName + ".dbg.bin";
                            File.WriteAllBytes(m_strPrimaryImgPath + strImgPath, rgData);
                            strTag = "FILE:" + strImgPath;
                            rgImg[i].DebugData = Encoding.ASCII.GetBytes(strTag);
                        }
                    }
 
                    rgData = rgImg[i].DataCriteria;
                    if (rgData != null)
                    {
                        strPath = getImagePath(rgData);
                        if (strPath == null)
                        {
                            strImgPath = strName + ".criteria.bin";
                            File.WriteAllBytes(m_strPrimaryImgPath + strImgPath, rgData);
                            strTag = "FILE:" + strImgPath;
                            rgImg[i].DataCriteria = Encoding.ASCII.GetBytes(strTag);
                        }
                    }
                }
 
                entities.SaveChanges();
 
                List<RawImageParameter> rgParam = entities.RawImageParameters.Where(p => rgId.Contains(p.RawImageID)).ToList();
                for (int i = 0; i < rgParam.Count; i++)
                {
                    rgData = rgParam[i].Value;
                    if (rgData != null)
                    {
                        strPath = getImagePath(rgData);
                        if (strPath == null)
                        {
                            int nRawImgId = rgParam[i].RawImageID.GetValueOrDefault(0);
                            string strName1 = rgNames[nRawImgId];
                            strImgPath = strName1 + ".param_" + rgParam[i].Name + ".bin";
                            File.WriteAllBytes(m_strPrimaryImgPath + strImgPath, rgData);
                            strTag = "FILE:" + strImgPath;
                            rgParam[i].Value = Encoding.ASCII.GetBytes(strTag);
                        }
                    }
                }
 
                entities.SaveChanges();
            }
 
            return true;
        }
 
        public bool ConvertRawImagesSaveToDatabase(int nIdx, int nCount, CancelEvent evtCancel = null)
        {
            if (m_strPrimaryImgPath == null)
                m_strPrimaryImgPath = getImagePath();
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.Where(p => p.SourceID == m_src.ID && p.Idx >= nIdx && p.Active == true).OrderBy(p => p.Idx).Take(nCount).ToList();
                List<string> rgstrFiles = new List<string>();
                string strPath;
                byte[] rgData;
                List<int?> rgId = new List<int?>();
 
                for (int i = 0; i < rgImg.Count; i++)
                {
                    if (evtCancel != null && evtCancel.WaitOne(0))
                        return false;
 
                    rgData = rgImg[i].Data;
                    if (rgData != null)
                    {
                        strPath = getImagePath(rgData);
                        if (strPath != null)
                        {
                            rgImg[i].Data = File.ReadAllBytes(m_strPrimaryImgPath + strPath);
                            rgstrFiles.Add(m_strPrimaryImgPath + strPath);
                        }
                    }
 
                    rgData = rgImg[i].DebugData;
                    if (rgData != null)
                    {
                        strPath = getImagePath(rgData);
                        if (strPath != null)
                        {
                            rgImg[i].DebugData = File.ReadAllBytes(m_strPrimaryImgPath + strPath);
                            rgstrFiles.Add(m_strPrimaryImgPath + strPath);
                        }
                    }
 
                    rgData = rgImg[i].DataCriteria;
                    if (rgData != null)
                    {
                        strPath = getImagePath(rgData);
                        if (strPath != null)
                        {
                            rgImg[i].DataCriteria = File.ReadAllBytes(m_strPrimaryImgPath + strPath);
                            rgstrFiles.Add(m_strPrimaryImgPath + strPath);
                        }
                    }
                }
 
                entities.SaveChanges();
 
                List<RawImageParameter> rgParam = entities.RawImageParameters.Where(p => rgId.Contains(p.RawImageID)).ToList();
                for (int i = 0; i < rgParam.Count; i++)
                {
                    rgData = rgParam[i].Value;
                    if (rgData != null)
                    {
                        strPath = getImagePath(rgData);
                        if (strPath != null)
                        {
                            rgParam[i].Value = File.ReadAllBytes(m_strPrimaryImgPath + strPath);
                            rgstrFiles.Add(m_strPrimaryImgPath + strPath);
                        }
                    }
                }
 
                entities.SaveChanges();
 
                foreach (string strFile in rgstrFiles)
                {
                    File.Delete(strFile);
                }
 
                if (Directory.Exists(m_strPrimaryImgPath))
                {
                    if (Directory.GetFiles(m_strPrimaryImgPath).Length == 0)
                        Directory.Delete(m_strPrimaryImgPath);
                }
            }
 
            return true;
        }
 
        public void FixupRawImageCopy(int nImageID, int nSecondarySrcId)
        {
            if (m_strPrimaryImgPath == null)
                m_strPrimaryImgPath = getImagePath();
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.Where(p => p.ID == nImageID).ToList();
                if (rgImg.Count > 0)
                {
                    string strPath;
                    int nVirtId = rgImg[0].VirtualID.GetValueOrDefault(0);
 
                    if (nVirtId > 0)
                    {
                        List<RawImage> rgImg2 = entities.RawImages.Where(p => p.ID == nVirtId).ToList();
                        if (rgImg2.Count > 0)
                        {
                            strPath = getImagePath(rgImg2[0].Data);
                            if (!File.Exists(m_strPrimaryImgPath + strPath))
                            {
                                rgImg[0].OriginalSourceID = nSecondarySrcId;
                                entities.SaveChanges();
                            }
                        }
                    }
                    else if (rgImg[0].Data != null)
                    {
                        strPath = getImagePath(rgImg[0].Data);
                        if (!File.Exists(m_strPrimaryImgPath + strPath))
                        {
                            rgImg[0].OriginalSourceID = nSecondarySrcId;
                            entities.SaveChanges();
                        }
                    }
                }
            }
        }
 
        public void PutRawImageParameters(List<ParameterData> rgParam, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                entities.Configuration.AutoDetectChangesEnabled = false;
                entities.Configuration.ValidateOnSaveEnabled = false;
 
                foreach (ParameterData param in rgParam)
                {
                    IQueryable<RawImageParameter> iquery = entities.RawImageParameters.Where(p => p.SourceID == param.SourceID && p.RawImageID == param.ImageID && p.Name == param.Name);
 
                    int nCount = iquery.Count();
                    if (nCount == 0)
                    {
                        RawImageParameter rip = new RawImageParameter();
                        rip.RawImageID = param.ImageID;
                        rip.Name = param.Name;
                        rip.SourceID = param.SourceID;
                        rip.TextValue = param.Value;
                        rip.NumericValue2 = (float)param.NumericValue;
                        rip.Value = param.Data;
                        entities.RawImageParameters.Add(rip);
                    }
                    else
                    {
                        if (!param.OnlyAddNew)
                        {
                            List<RawImageParameter> rg = iquery.ToList();
                            rg[0].TextValue = param.Value;
                            rg[0].NumericValue2 = (float)param.NumericValue;
                            rg[0].Value = param.Data;
                        }
                    }
                }
 
                entities.SaveChanges();
            }
        }
 
        public void PutRawImages(List<RawImage> rgImg, List<List<ParameterData>> rgrgParam = null, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                entities.Configuration.AutoDetectChangesEnabled = false;
                entities.Configuration.ValidateOnSaveEnabled = false;
 
                foreach (RawImage img in rgImg)
                {
                    m_nLastIndex = Math.Max(m_nLastIndex, img.Idx.GetValueOrDefault(0));
                    entities.RawImages.Add(img);
                    AddLabelToCache(img.ActiveLabel.GetValueOrDefault());
                }
 
                entities.SaveChanges();
 
                if (rgrgParam != null && rgrgParam.Count == rgImg.Count)
                {
                    for (int i = 0; i < rgImg.Count; i++)
                    {
                        for (int j = 0; j < rgrgParam[i].Count; j++)
                        {
                            ParameterData p = rgrgParam[i][j];
                            string strName = p.Name;
                            string strVal = p.Value;
                            double? dfVal = p.NumericValue;
                            byte[] rgData = p.Data;
 
                            if (p.ImageID != 0)
                            {
                                RawImageParameter imgParam = GetRawImageParameterEx(p.ImageID, strName);
 
                                if (imgParam != null)
                                {
                                    strVal = imgParam.TextValue;
                                    rgData = imgParam.Value;
                                }
                            }
 
                            if (!String.IsNullOrEmpty(strVal) || dfVal.HasValue || rgData != null)
                                SetRawImageParameter(rgImg[i].ID, strName, strVal, dfVal, rgData, false, false, entities);
                        }
                    }
 
                    entities.SaveChanges();
                }
            }
        }
 
        public int PutRawImage(int nIdx, SimpleDatum d, string strDescription = null)
        {
            RawImage img = CreateRawImage(nIdx, d, 0, strDescription);
            m_entities.RawImages.Add(img);
            m_entities.SaveChanges();
 
            if (d is Datum)
            {
                string str = ((Datum)d).Tag as string;
                string strName = ((Datum)d).TagName;
 
                if (str != null && str.Length > 0 && strName != null && strName.Length > 0)
                    SetRawImageParameter(m_src.ID, img.ID, strName, str);
            }
 
            AddLabelToCache(d.Label);
 
            return img.ID;
        }
 
        public RawImage GetRawImage(int nID)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgItems = entities.RawImages.AsNoTracking().Where(p => p.ID == nID).ToList();
 
                if (rgItems.Count == 0)
                    return null;
 
                return rgItems[0];
            }
        }
 
        public int QueryRawImageCount(int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                return entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId).Count();
            }
        }
 
        public void DeleteRawImages(int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            string strCmd = "DELETE FROM RawImages WHERE (SourceID = " + nSrcId.ToString() + ")";
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.ExecuteSqlCommand(strCmd);
 
                string strImgPath = getImagePath();
                if (strImgPath != null)
                    deleteImages(strImgPath);
            }
        }
 
        private bool deleteImages(string strPath)
        {
            if (!Directory.Exists(strPath))
                return true;
 
            string[] rgstrFiles = Directory.GetFiles(strPath);
            foreach (string strFile in rgstrFiles)
            { 
                File.Delete(strFile);
            }
 
            Directory.Delete(strPath);
 
            return true;
        }
 
        public void DeleteRawImageResults(int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            string strCmd = "DELETE FROM RawImageResults WHERE (SourceID = " + nSrcId.ToString() + ")";
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public RawImageMean GetRawImageMean(int nSrcId = 0, ConnectInfo ci = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<RawImageMean> rgImg = entities.RawImageMeans.AsNoTracking().Where(p => p.SourceID == nSrcId).ToList();
 
                if (rgImg.Count == 0)
                    return null;
 
                return rgImg[0];
            }
        }
 
        public int PutRawImageMean(SimpleDatum sd, bool bUpdate, int nSrcId = 0, ConnectInfo ci = null)
        {
            if (sd == null)
                return 0;
 
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            if (isRemote(null))
                return 0;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                IQueryable<RawImageMean> iQuery = entities.RawImageMeans.Where(p => p.SourceID == nSrcId);
                if (iQuery != null)
                {
                    List<RawImageMean> rgMean = iQuery.ToList();
                    RawImageMean im = null;
 
                    if (rgMean.Count == 0)
                        im = new RawImageMean();
                    else
                        im = rgMean[0];
 
                    if (bUpdate || rgMean.Count == 0)
                    {
                        bool bEncoded = false;
                        im.Channels = sd.Channels;
                        im.Height = sd.Height;
                        im.Width = sd.Width;
                        im.SourceID = nSrcId;
                        im.Encoded = sd.IsRealData;
                        im.Data = sd.GetByteData(out bEncoded);
                    }
 
                    if (rgMean.Count == 0)
                        entities.RawImageMeans.Add(im);
 
                    if (rgMean.Count == 0 || bUpdate)
                        entities.SaveChanges();
 
                    return im.ID;
                }
 
                return 0;
            }
        }
 
        public bool CopyImageMean(int nSrcIdSrc, int nSrcIdDst, ConnectInfo ci = null)
        {
            RawImageMean src = GetRawImageMean(nSrcIdSrc);
            if (src == null)
                return false;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<RawImageMean> rgDst = entities.RawImageMeans.Where(p => p.SourceID == nSrcIdDst).ToList();
                RawImageMean dst;
 
                if (rgDst.Count == 0)
                {
                    dst = new RawImageMean();
                    dst.SourceID = nSrcIdDst;
                }
                else
                {
                    dst = rgDst[0];
                }
 
                dst.Encoded = src.Encoded;
                dst.Channels = src.Channels;
                dst.Height = src.Height;
                dst.Width = src.Width;
                dst.Data = src.Data;
 
                if (rgDst.Count == 0)
                    entities.RawImageMeans.Add(dst);
 
                entities.SaveChanges();
            }
 
            return true;
        }
 
        public void DeleteRawImageMeans(int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            string strCmd = "DELETE FROM RawImageMeans WHERE (SourceID = " + nSrcId.ToString() + ")";
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        private IQueryable<RawImage> getQuery(IQueryable<RawImage> iQuery, string strFilterVal, int? nBoostVal)
        {
            if (!string.IsNullOrEmpty(strFilterVal))
                iQuery = iQuery.Where(p => p.Description == strFilterVal);
 
            if (nBoostVal.HasValue)
            {
                if (nBoostVal.Value < 0)
                {
                    int nVal = Math.Abs(nBoostVal.Value);
                    iQuery = iQuery.Where(p => p.ActiveBoost == nVal);
                }
                else
                {
                    int nVal = nBoostVal.Value;
                    iQuery = iQuery.Where(p => p.ActiveBoost >= nVal);
                }
            }
            else
            {
                iQuery = iQuery.Where(p => p.ActiveBoost > 0);
            }
 
            return iQuery;
        }
 
        public int GetBoostCount(int nSrcId = 0, string strFilterVal = null, int? nBoostVal = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImage> iQuery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Active == true);
                iQuery = getQuery(iQuery, strFilterVal, nBoostVal);
                return iQuery.Count();
            }
        }
 
        public int ActivateFiltered(int nSrcId = 0, string strFilterVal = null, int? nBoostVal = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                if (!string.IsNullOrEmpty(strFilterVal) || nBoostVal.HasValue)
                {
                    string strCmd1 = "UPDATE[dbo].[RawImages] SET[Active] = 0 WHERE SourceID = " + nSrcId.ToString();
                    entities.Database.ExecuteSqlCommand(strCmd1);
 
                    string strCmd2 = "UPDATE[dbo].[RawImages] SET[Active] = 1 WHERE SourceID = " + nSrcId.ToString();
 
                    if (strFilterVal != null)
                        strCmd2 += " AND Description = '" + strFilterVal + "'";
 
                    if (nBoostVal.HasValue)
                    {
                        int nVal = nBoostVal.Value;
 
                        if (nVal < 0)
                        {
                            nVal = Math.Abs(nVal);
                            strCmd2 += " AND ActiveBoost = " + nVal.ToString();
                        }
                        else
                        {
                            strCmd2 += " AND ActiveBoost >= " + nVal.ToString();
                        }
                    }
 
                    entities.Database.ExecuteSqlCommand(strCmd2);
                }
                else
                {
                    string strCmd2 = "UPDATE[dbo].[RawImages] SET[Active] = 1 WHERE SourceID = " + nSrcId.ToString();
                    entities.Database.ExecuteSqlCommand(strCmd2);
                }
 
                return entities.RawImages.Where(p => p.Active == true).Count();
            }
        }
 
        public int GetImageCount(int nSrcId = 0, bool bActive = true, bool bInactive = true, ConnectInfo ci = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                IQueryable<RawImage> iQuery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId);
 
                if (bActive && bInactive)
                    return iQuery.Count();
 
                if (bActive)
                    iQuery = iQuery.Where(p => p.Active == true);
                else
                    iQuery = iQuery.Where(p => p.Active == false);
 
                return iQuery.Count();
            }
        }
 
        public void ResetAllBoosts(int nSrcId = 0, int nMinBoost = 0, bool bExactVal = false, int nDesiredBoostVal = -1)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "UPDATE RawImages SET ActiveBoost = ";
 
                if (nDesiredBoostVal >= 0)
                    strCmd += nDesiredBoostVal.ToString();
                else
                    strCmd += "OriginalBoost";
 
                strCmd += " WHERE(SourceID = " + nSrcId.ToString();
 
                if (nMinBoost >= 1)
                {
                    if (bExactVal)
                        strCmd += " AND ActiveBoost = " + nMinBoost.ToString();
                    else
                        strCmd += " AND ActiveBoost >= " + nMinBoost.ToString();
                }
 
                strCmd += ")";
 
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public void UpdateBoost(long nImageID, int nBoost)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "UPDATE RawImages SET ActiveBoost = " + nBoost.ToString() + " WHERE (ID = " + nImageID.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public void UpdateBoosts(int nSrcId, DateTime dtStart, DateTime dtEnd, int nBoost)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "UPDATE RawImages SET ActiveBoost = " + nBoost.ToString() + " WHERE (SourceID = " + nSrcId.ToString() + " AND TimeStamp >= '" + dtStart.ToString() + "' AND TimeStamp <= '" + dtEnd.ToString() + "')";
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public List<RawImage> ReindexRawImages(Log log, CancelEvent evtCancel, int nSrcId = 0)
        {
            if (nSrcId == 0 && m_src != null)
                nSrcId = m_src.ID;
 
            if (nSrcId == 0)
                return null;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                log.WriteLine("Resetting inactive indexes...");
                string strCmd = "UPDATE [dbo].[RawImages] SET [Idx] = -1 WHERE SourceID = " + nSrcId.ToString() + " AND Active = 0";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                log.WriteLine("Querying active images...");
 
                Stopwatch sw = new Stopwatch();
                List<RawImage> rgImg = entities.RawImages.Where(p => p.SourceID == nSrcId && p.Active == true).OrderBy(p => p.TimeStamp).ThenBy(p => p.ID).ToList();
 
                sw.Start();
                int nIdx = 0;
 
                for (int i = 0; i < rgImg.Count; i++)
                {
                    rgImg[i].Idx = nIdx;
                    nIdx++;
 
                    if (i % 1000 == 0)
                        entities.SaveChanges();
 
                    if (sw.Elapsed.TotalMilliseconds > 1000)
                    {
                        log.Progress = ((double)i / (double)rgImg.Count);
                        log.WriteLine("reindexing at " + log.Progress.ToString("P"));
                        sw.Restart();
                    }
 
                    if (evtCancel.WaitOne(0))
                        return null;
                }
 
                entities.SaveChanges();
 
                return rgImg;
            }
        }
 
        public bool UpdateRawImageSourceID(int nImageID, int nSrcID)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.Where(p => p.ID == nImageID).ToList();
                if (rgImg.Count > 0)
                {
                    if (rgImg[0].SourceID != nSrcID)
                    {
                        if (rgImg[0].OriginalSourceID == null)
                            rgImg[0].OriginalSourceID = rgImg[0].SourceID;
 
                        rgImg[0].SourceID = nSrcID;
                        
                        entities.SaveChanges();
                        return true;
                    }
                }
            }
 
            return false;
        }
 
        public bool ActivateRawImage(int nImageID, bool bActivate, bool bSave = true)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.Where(p => p.ID == nImageID).ToList();
                if (rgImg.Count == 0)
                    return false;
 
                if (rgImg[0].Active == bActivate)
                    return false;
 
                rgImg[0].Active = bActivate;
 
                if (bSave)
                    entities.SaveChanges();
            }
 
            return true;
        }
 
        public void ActivateAllRawImages(bool bActive, bool bAnnotatedOnly, params int[] rgSrcId)
        {
            if (rgSrcId.Length == 0)
                throw new Exception("You must specify at least one source iD.");
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strActive = (bActive) ? "1" : "0";
                string strCmd = "UPDATE RawImages SET [Active] = " + strActive + "  WHERE (";
 
                for (int i=0; i<rgSrcId.Length; i++)
                {
                    strCmd += "SourceID = " + rgSrcId[i].ToString();
 
                    if (i < rgSrcId.Length - 1)
                        strCmd += " OR ";
                }
 
                strCmd += ")";
 
                if (bAnnotatedOnly)
                {
                    strCmd += " AND ([DataCriteriaFormatID] = 9)";
                    strCmd += " AND (Convert(varbinary(8), [DataCriteria]) != 0x00000000)";
                }
 
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public void ActivateAllRawImages(bool bActive, bool bAnnotatedOnly, int? nTgtLabel, bool bTargetLabelExact, int? nTgtBoost, bool bTargetBoostExact, params int[] rgSrcId)
        {
            if (rgSrcId.Length == 0)
                throw new Exception("You must specify at least one source iD.");
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strActive = (bActive) ? "1" : "0";
                string strCmd = "UPDATE RawImages SET [Active] = " + strActive + "  WHERE (";
 
                for (int i = 0; i < rgSrcId.Length; i++)
                {
                    strCmd += "SourceID = " + rgSrcId[i].ToString();
 
                    if (i < rgSrcId.Length - 1)
                        strCmd += " OR ";
                }
 
                strCmd += ")";
 
                if (bAnnotatedOnly)
                {
                    strCmd += " AND ([DataCriteriaFormatID] = 9)";
                    strCmd += " AND (Convert(varbinary(8), [DataCriteria]) != 0x00000000)";
                }
 
                if (nTgtLabel.HasValue)
                {
                    strCmd += " AND ([ActiveLabel] ";
 
                    if (bTargetLabelExact)
                        strCmd += "=";
                    else
                        strCmd += ">=";
 
                    strCmd += nTgtLabel.Value.ToString() + ")";
                }
 
                if (nTgtBoost.HasValue)
                {
                    strCmd += " AND ([ActiveBoost] ";
 
                    if (bTargetBoostExact)
                        strCmd += "=";
                    else
                        strCmd += ">=";
 
                    strCmd += nTgtBoost.Value.ToString() + ")";
                }
 
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public void UpdateDatasetImageAnnotations(int nSrcId, int nImageId, AnnotationGroupCollection annotations, bool bSetLabelOnly)
        {
            using (DNNEntities entities = MyCaffe.db.image.EntitiesConnection.CreateEntities())
            {
                List<RawImage> rgImg = entities.RawImages.Where(p => p.ID == nImageId).ToList();
                if (rgImg.Count > 0)
                {
                    if (bSetLabelOnly)
                    {
                        if (annotations.Count > 0)
                            rgImg[0].ActiveLabel = annotations[0].group_label;
                        else
                            rgImg[0].ActiveLabel = 0;
                    }
                    else
                    {
                        if (annotations.Count > 0)
                        {
                            rgImg[0].DataCriteria = SimpleDatum.SaveAnnotationDataToDataCriteriaByteArray(SimpleDatum.ANNOTATION_TYPE.BBOX, annotations);
                            rgImg[0].DataCriteriaFormatID = (int)SimpleDatum.DATA_FORMAT.ANNOTATION_DATA;
                        }
                        else
                        {
                            rgImg[0].DataCriteria = null;
                            rgImg[0].DataCriteriaFormatID = (int)SimpleDatum.DATA_FORMAT.NONE;
                        }
                    }
 
                    entities.SaveChanges();
                }
 
                foreach (KeyValuePair<int, string> kv in annotations.Labels)
                {
                    List<Label> rgLabel = entities.Labels.Where(p => p.SourceID == nSrcId && p.Label1 == kv.Key).ToList();
 
                    if (rgLabel.Count == 0)
                    {
                        Label label = new Label();
                        label.Label1 = kv.Key;
                        label.ActiveLabel = kv.Key;
                        label.ImageCount = 0;
                        label.SourceID = nSrcId;
                        label.Name = kv.Value;
                        entities.Labels.Add(label);
                    }
                    else
                    {
                        rgLabel[0].Name = kv.Value;
                    }
 
                    entities.SaveChanges();
                }
            }
        }
 
        #endregion
 
 
        //---------------------------------------------------------------------
        //  RawImage Results
        //---------------------------------------------------------------------
        #region RawImage Results
 
        public int PutRawImageResults(int nSrcId, int nIdx, int nLabel, DateTime dt, List<Result> rgResults, bool bInvert, List<Tuple<DateTime, int>> rgExtra = null)
        {
            if (rgResults.Count == 0)
                throw new Exception("You must have at least one result!");
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImageResult> rg = entities.RawImageResults.Where(p => p.SourceID == nSrcId && p.Idx == nIdx).ToList();
                RawImageResult r;
 
                if (rg.Count == 0)
                {
                    r = new RawImageResult();
                    r.Idx = nIdx;
                    r.SourceID = nSrcId;
                }
                else
                {
                    r = rg[0];
                }
 
                r.Label = nLabel;
                r.ResultCount = rgResults.Count;
                r.Results = ResultDescriptor.CreateResults(rgResults, bInvert);
                r.TimeStamp = dt;
                r.BatchCount = 0;
 
                if (rgExtra != null && rgExtra.Count > 0)
                    r.ExtraData = PackExtraData(rgExtra);
 
                if (rg.Count == 0)
                    entities.RawImageResults.Add(r);
 
                entities.SaveChanges();
 
                return r.ID;
            }
        }
 
        public int PutRawImageResults(int nSrcId, int nIdx, int nLabel, DateTime dt, List<Tuple<SimpleDatum, List<Result>>> rgrgResults, List<Tuple<DateTime, int>> rgExtra = null)
        {
            if (rgrgResults.Count == 0 || rgrgResults[0].Item2.Count == 0)
                throw new Exception("You must have at least one result!");
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImageResult> rg = entities.RawImageResults.Where(p => p.SourceID == nSrcId && p.Idx == nIdx).ToList();
                RawImageResult r;
 
                if (rg.Count == 0)
                {
                    r = new RawImageResult();
                    r.Idx = nIdx;
                    r.SourceID = nSrcId;
                }
                else
                {
                    r = rg[0];
                }
 
                r.Label = nLabel;
                r.ResultCount = rgrgResults[0].Item2.Count;
                r.Results = ResultDescriptor.CreateResults(rgrgResults);
                r.TimeStamp = dt;
                r.BatchCount = rgrgResults.Count;
 
                if (rgExtra != null && rgExtra.Count > 0)
                    r.ExtraData = PackExtraData(rgExtra);
 
                if (rg.Count == 0)
                    entities.RawImageResults.Add(r);
 
                entities.SaveChanges();
 
                return r.ID;
            }
        }
 
        public List<Tuple<SimpleDatum, List<Result>>> GetRawImageResultBatch(int nBatchCount, byte[] rgResults)
        {
            if (nBatchCount <= 0)
                throw new Exception("The batch count must be 1 or greater!");
 
            return ResultDescriptor.GetResults(nBatchCount, rgResults);
        }
 
        public int PutRawImageResultExtraData(int nRawImageResultID, byte[] rgExtraData)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImageResult> rg = entities.RawImageResults.Where(p => p.ID == nRawImageResultID).ToList();
                if (rg.Count == 0)
                    return 0;
 
                rg[0].ExtraData = rgExtraData;
                entities.SaveChanges();
 
                return rg[0].ID;
            }
        }
 
        public List<RawImageResult> GetRawImageResults(int nSrcId = 0, bool bRequireExtraData = false, int nMax = -1)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = MyCaffe.db.image.EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImageResult> iQry = entities.RawImageResults.AsNoTracking().Where(p => p.SourceID == nSrcId).OrderBy(p => p.TimeStamp);
 
                if (bRequireExtraData)
                    iQry = iQry.Where(p => p.ExtraData != null);
 
                if (nMax > 0)
                    iQry = iQry.Take(nMax);
 
                return iQry.ToList();
            }
        }
 
        public static byte[] PackExtraData(List<Tuple<DateTime, int>> rg)
        {
            using (MemoryStream ms = new MemoryStream())
            using (BinaryWriter bw = new BinaryWriter(ms))
            {
                bw.Write(rg.Count);
 
                for (int i = 0; i < rg.Count; i++)
                {
                    bw.Write(rg[i].Item1.ToFileTimeUtc());
                    bw.Write(rg[i].Item2);
                }
 
                ms.Flush();
                return ms.ToArray();
            }
        }
 
        public static List<Tuple<DateTime, int>> UnpackExtraData(byte[] rg)
        {
            List<Tuple<DateTime, int>> rgExtra = new List<Tuple<DateTime, int>>();
 
            using (MemoryStream ms = new MemoryStream(rg))
            using (BinaryReader br = new BinaryReader(ms))
            {
                int nCount = br.ReadInt32();
 
                for (int i = 0; i < nCount; i++)
                {
                    long lTime = br.ReadInt64();
                    int nLabel = br.ReadInt32();
                    DateTime dt = DateTime.FromFileTimeUtc(lTime);
 
                    rgExtra.Add(new Tuple<DateTime, int>(dt, nLabel));
                }
            }
 
            return rgExtra;
        }
 
        #endregion
 
 
        //---------------------------------------------------------------------
        //  RawImage Parameters
        //---------------------------------------------------------------------
        #region RawImage Parameters
 
        public string GetRawImageParameter(int nRawImageID, string strName, string strDefault)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImageParameter> rgP = entities.RawImageParameters.AsNoTracking().Where(p => p.RawImageID == nRawImageID && p.Name == strName).ToList();
 
                if (rgP.Count == 0)
                    return strDefault;
 
                return rgP[0].TextValue;
            }
        }
 
        public int GetRawImageParameter(int nRawImageID, string strName, int nDefault)
        {
            string str = GetRawImageParameter(nRawImageID, strName, null);
 
            if (str == null)
                return nDefault;
 
            return int.Parse(str);
        }
 
        public double GetRawImageParameter(int nRawImageID, string strName, double dfDefault)
        {
            string str = GetRawImageParameter(nRawImageID, strName, null);
 
            if (str == null)
                return dfDefault;
 
            return BaseParameter.ParseDouble(str);
        }
 
        public bool GetRawImageParameter(int nRawImageID, string strName, bool bDefault)
        {
            string str = GetRawImageParameter(nRawImageID, strName, null);
 
            if (str == null)
                return bDefault;
 
            return bool.Parse(str);
        }
 
        public byte[] GetRawImageParameterData(int nRawImageID, string strName)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImageParameter> rgP = entities.RawImageParameters.AsNoTracking().Where(p => p.RawImageID == nRawImageID && p.Name == strName).ToList();
 
                if (rgP.Count == 0)
                    return null;
 
                return getRawImage(rgP[0].Value, null, null, entities);
            }
        }
 
        public RawImageParameter GetRawImageParameterEx(int nRawImageID, string strName)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImageParameter> rgP = entities.RawImageParameters.AsNoTracking().Where(p => p.RawImageID == nRawImageID && p.Name == strName).ToList();
 
                if (rgP.Count == 0)
                    return null;
 
                return rgP[0];
            }
        }
 
        public int SetRawImageParameter(int nRawImageID, string strName, string strValue, double? dfVal = null, byte[] rgData = null, bool bSave = true, bool bOnlyAddNew = false, DNNEntities entities = null)
        {
            if (entities == null)
                entities = m_entities;
 
            List<RawImageParameter> rgP = entities.RawImageParameters.Where(p => p.RawImageID == nRawImageID && p.Name == strName).ToList();
            RawImageParameter riP = null;
 
            if (rgP.Count == 0)
            {
                riP = new RawImageParameter();
                riP.RawImageID = nRawImageID;
                riP.SourceID = m_src.ID;
                riP.Name = strName;
            }
            else
            {
                if (bOnlyAddNew)
                    return rgP[0].ID;
 
                riP = rgP[0];
            }
 
            float fVal = 0;
            if (dfVal < -float.MaxValue)
                fVal = -float.MaxValue;
            else if (dfVal > float.MaxValue)
                fVal = float.MaxValue;
            else
                fVal = (float)dfVal;
 
            riP.TextValue = strValue;
            riP.NumericValue2 = fVal;
            riP.NumericValue = null;
            riP.Value = setImageByteData(rgData, "param_" + strName);
 
            if (rgP.Count == 0)
                entities.RawImageParameters.Add(riP);
 
            if (bSave)
                entities.SaveChanges();
 
            return riP.ID;
        }
 
        public int SetRawImageParameter(int nSrcId, int nRawImageID, string strName, string strValue, double? dfVal = null, byte[] rgData = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImageParameter> rgP = entities.RawImageParameters.Where(p => p.RawImageID == nRawImageID && p.Name == strName).ToList();
                RawImageParameter riP = null;
 
                if (rgP.Count == 0)
                {
                    riP = new RawImageParameter();
                    riP.RawImageID = nRawImageID;
                    riP.SourceID = nSrcId;
                    riP.Name = strName;
                }
                else
                {
                    riP = rgP[0];
                }
 
                riP.TextValue = strValue;
                riP.NumericValue2 = (float)dfVal;
                riP.Value = setImageByteData(rgData, "param_" + strName);
 
                if (rgP.Count == 0)
                    entities.RawImageParameters.Add(riP);
 
                entities.SaveChanges();
 
                return riP.ID;
            }
        }
 
        public int SetRawImageParameterAt(DateTime dt, string strName, string strValue, double? dfVal, byte[] rgData)
        {
            List<RawImage> rg = m_entities.RawImages.Where(p => p.SourceID == m_src.ID && p.TimeStamp == dt).ToList();
 
            if (rg.Count == 0)
                return 0;
 
            return SetRawImageParameter(m_src.ID, rg[0].ID, strName, strValue, dfVal, rgData);
        }
 
        public void DeleteRawImageParameters(int nSrcId)
        {
            string strCmd = "DELETE FROM RawImageParameters WHERE (SourceID = " + nSrcId.ToString() + ")";
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        public int GetRawImageParameterCount(string strName, int nSrcId = 0, string strType = "TEXT")
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                if (strType.ToLower() == "text")
                    return entities.RawImageParameters.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Name == strName && p.TextValue != null).Count();
 
                if (strType.ToLower() == "numeric")
                    return entities.RawImageParameters.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Name == strName && p.NumericValue != null).Count();
 
                if (strType.ToLower() == "value")
                    return entities.RawImageParameters.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Name == strName && p.Value != null).Count();
            }
 
            return 0;
        }
 
        public bool GetRawImageParameterExist(string strName, int nSrcId = 0, string strType = "TEXT")
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                if (strType.ToLower() == "text")
                {
                    int nCount = entities.RawImageParameters.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Name == strName && p.TextValue != null).Take(1).Count();
                    return (nCount > 0) ? true : false;
                }
 
                if (strType.ToLower() == "numeric")
                {
                    int nCount = entities.RawImageParameters.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Name == strName && p.NumericValue != null).Take(1).Count();
                    return (nCount > 0) ? true : false;
                }
 
                if (strType.ToLower() == "value")
                {
                    int nCount = entities.RawImageParameters.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Name == strName && p.Value != null).Take(1).Count();
                    return (nCount > 0) ? true : false;
                }
            }
 
            return false;
        }
 
        public List<string> GetRawImageDistinctParameterDescriptions(int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "SELECT DISTINCT [TextValue] FROM RawImageParameters WHERE SourceID = " + nSrcId.ToString();
                DbRawSqlQuery<string> res = entities.Database.SqlQuery<string>(strCmd);
                List<string> rgstr = res.ToList<string>();
 
                List<string> rgstr1 = new List<string>();
 
                foreach (string str in rgstr)
                {
                    if (str.Trim().Length > 0)
                        rgstr1.Add(str);
                }
 
                return rgstr1;
            }
        }
 
        #endregion
 
 
        //---------------------------------------------------------------------
        //  RawImage Groups
        //---------------------------------------------------------------------
        #region RawImage Groups
 
        public int AddRawImageGroup(Image img, int nIdx, DateTime dtStart, DateTime dtEnd, List<double> rgProperties)
        {
            RawImageGroup g = new RawImageGroup();
 
            g.RawData = ImageTools.ImageToByteArray(img);
            g.Idx = nIdx;
            g.StartDate = dtStart;
            g.EndDate = dtEnd;
 
            if (rgProperties != null)
            {
                if (rgProperties.Count > 0)
                    g.Property1 = (decimal)rgProperties[0];
 
                if (rgProperties.Count > 1)
                    g.Property2 = (decimal)rgProperties[1];
 
                if (rgProperties.Count > 2)
                    g.Property3 = (decimal)rgProperties[2];
 
                if (rgProperties.Count > 3)
                    g.Property4 = (decimal)rgProperties[3];
 
                if (rgProperties.Count > 4)
                    g.Property5 = (decimal)rgProperties[4];
            }
 
            return PutRawImageGroup(g);
        }
 
        public int PutRawImageGroup(RawImageGroup g)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImageGroup> rgG = null;
                
                if (g.ID != 0)
                    rgG = entities.RawImageGroups.Where(p => p.ID == g.ID).ToList();
 
                if (rgG != null && rgG.Count > 0)
                {
                    rgG[0].EndDate = g.EndDate;
                    rgG[0].Idx = g.Idx;
                    rgG[0].Property1 = g.Property1;
                    rgG[0].Property2 = g.Property2;
                    rgG[0].Property3 = g.Property3;
                    rgG[0].Property4 = g.Property4;
                    rgG[0].Property5 = g.Property5;
                    rgG[0].Rating = g.Rating;
                    rgG[0].RawData = g.RawData;
                    rgG[0].StartDate = g.StartDate;                    
                }
                else
                {
                    entities.RawImageGroups.Add(g);
                }
 
                entities.SaveChanges();
 
                return g.ID;
            }
        }
 
        public RawImageGroup FindRawImageGroup(int nIdx, DateTime dtStart, DateTime dtEnd)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<RawImageGroup> rgG = entities.RawImageGroups.AsNoTracking().Where(p => p.Idx == nIdx && p.StartDate == dtStart && p.EndDate == dtEnd).ToList();
 
                if (rgG.Count == 0)
                    return null;
 
                return rgG[0];
            }
        }
 
        public int FindRawImageGroupID(int nIdx, DateTime dtStart, DateTime dtEnd)
        {
            RawImageGroup g = FindRawImageGroup(nIdx, dtStart, dtEnd);
 
            if (g == null)
                return 0;
 
            return g.ID;
        }
 
        public void DeleteRawImageGroups()
        {
            string strCmd = "DELETE FROM RawImageGroups";
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.ExecuteSqlCommand(strCmd);
            }
        }
 
        #endregion
 
 
        //---------------------------------------------------------------------
        //  Sources
        //---------------------------------------------------------------------
        #region Sources
 
#pragma warning disable 1591
 
        public List<int> GetAllDataSourcesIDs() 
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                return entities.Sources.AsNoTracking().Select(p => p.ID).ToList();
            }
        }
 
#pragma warning restore 1591
 
        public void DeleteSourceData()
        {
            DeleteSourceData(m_src.ID);
        }
 
        public void UpdateSaveImagesToFile(bool bSaveToFile, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Source> rg = entities.Sources.Where(p => p.ID == nSrcId).ToList();
 
                if (rg.Count > 0)
                {
                    rg[0].SaveImagesToFile = bSaveToFile;
                    entities.SaveChanges();
                }
            }
 
            if (m_src != null)
                m_src.SaveImagesToFile = bSaveToFile;
        }
 
        public void UpdateSource(int nChannels, int nWidth, int nHeight, bool bDataIsReal, int nSrcId = 0)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Source> rg = entities.Sources.Where(p => p.ID == nSrcId).ToList();
 
                if (rg.Count > 0)
                {
                    rg[0].ImageChannels = nChannels;
                    rg[0].ImageWidth = nWidth; 
                    rg[0].ImageHeight = nHeight;
                    rg[0].ImageEncoded = bDataIsReal;
                    entities.SaveChanges();
                }
            }
 
            if (m_src != null)
            {
                m_src.ImageChannels = nChannels;
                m_src.ImageWidth = nWidth;
                m_src.ImageHeight = nHeight;
                m_src.ImageEncoded = bDataIsReal;
            }
        }
 
        public void UpdateSourceCounts(int nImageCount, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                int nSrcId = m_src.ID;
                List<Source> rg = entities.Sources.Where(p => p.ID == nSrcId).ToList();
 
                if (rg.Count > 0)
                {
                    rg[0].ImageCount = nImageCount;
                    entities.SaveChanges();
                }
            }
 
            m_src.ImageCount = nImageCount;
        }
 
        public void UpdateSourceCounts(ConnectInfo ci = null)
        {
            string strCmd = "SELECT COUNT(ID) FROM RawImages WHERE (SourceID = " + m_src.ID.ToString() + " AND Active=1)";
            DbRawSqlQuery<int> result = m_entities.Database.SqlQuery<int>(strCmd);
            List<int> rgResult = result.ToList();
            int nCount = 0;
 
            if (rgResult.Count > 0)
                nCount = rgResult[0];
 
            if (nCount == 0)
            {
                strCmd = "SELECT COUNT(ID) FROM RawValues WHERE (SourceID = " + m_src.ID.ToString() + " AND Active=1)";
                result = m_entities.Database.SqlQuery<int>(strCmd);
                rgResult = result.ToList();
                nCount = 0;
 
                if (rgResult.Count > 0)
                    nCount = rgResult[0];
            }
 
            UpdateSourceCounts(nCount, ci);
        }
 
        public int GetSourceID(string strName, ConnectInfo ci = null)
        {
            strName = convertWs(strName, '_');
            Source src = GetSource(strName, ci);
 
            if (src == null)
                return 0;
 
            return src.ID;
        }
 
        public string GetSourceName(int nID, ConnectInfo ci = null)
        {
            Source src = GetSource(nID);
 
            if (src == null)
                return null;
 
            return src.Name;
        }
 
        public Source GetSource(string strName, ConnectInfo ci = null)
        {
            strName = convertWs(strName, '_');
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Source> rgSrc = entities.Sources.AsNoTracking().Where(p => p.Name == strName).ToList();
 
                if (rgSrc.Count == 0)
                    return null;
 
                return rgSrc[0];
            }
        }
 
        public Source GetSource(int nID, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Source> rgSrc = entities.Sources.AsNoTracking().Where(p => p.ID == nID).ToList();
 
                if (rgSrc.Count == 0)
                    return null;
 
                return rgSrc[0];
            }
        }
 
        public int PutSource(Source src, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Source> rgSrc = null;
 
                if (src.ID != 0)
                    rgSrc = entities.Sources.Where(p => p.ID == src.ID).ToList();
                else
                    rgSrc = entities.Sources.Where(p => p.Name == src.Name).ToList();
 
                if (rgSrc != null && rgSrc.Count > 0)
                {
                    rgSrc[0].ImageChannels = src.ImageChannels;
                    rgSrc[0].ImageCount = src.ImageCount;
                    rgSrc[0].ImageEncoded = src.ImageEncoded;
                    rgSrc[0].ImageHeight = src.ImageHeight;
                    rgSrc[0].ImageWidth = src.ImageWidth;
                    rgSrc[0].SaveImagesToFile = src.SaveImagesToFile;
                    src.ID = rgSrc[0].ID;
                }
                else
                {
                    entities.Sources.Add(src);
                }
 
                entities.SaveChanges();
 
                return src.ID;
            }
        }
 
        public int AddSource(string strName, int nChannels, int nWidth, int nHeight, bool bDataIsReal, int nCopyOfSourceID = 0, bool bSaveImagesToFile = true, ConnectInfo ci = null)
        {
            Source src = new Source();
 
            src.Name = convertWs(strName, '_');
            src.ImageChannels = nChannels;
            src.ImageHeight = nHeight;
            src.ImageWidth = nWidth;
            src.ImageEncoded = bDataIsReal;
            src.ImageCount = 0;
            src.SaveImagesToFile = bSaveImagesToFile;
            src.CopyOfSourceID = nCopyOfSourceID;
 
            return PutSource(src, ci);
        }
 
        public virtual bool DeleteSource(int nSrcId = 0)
        {
            string strCmd;
 
            if (nSrcId == 0)
            {
                if (m_src == null)
                    return false;
 
                nSrcId = m_src.ID;
            }
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                strCmd = "DELETE FROM Sources WHERE (ID = " + nSrcId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE FROM SourceParameters WHERE (SourceID = " + nSrcId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE FROM Labels WHERE (SourceID = " + nSrcId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "IF OBJECT_ID (N'dbo.ValueItems', N'U') IS NOT NULL DELETE FROM ValueItems WHERE (SourceID = " + nSrcId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "IF OBJECT_ID (N'dbo.ValueStreams', N'U') IS NOT NULL DELETE FROM ValueStreams WHERE (SourceID = " + nSrcId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "IF OBJECT_ID (N'dbo.RawValues', N'U') IS NOT NULL DELETE FROM RawValues WHERE (SourceID = " + nSrcId.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
            }
 
            DeleteSourceData(nSrcId);
 
            return true;
        }
 
        public void DeleteSources(params string[] rgstrSrc)
        {
            List<string> rgstrSrc1 = new List<string>();
 
            foreach (string str in rgstrSrc)
            {
                rgstrSrc1.Add(convertWs(str, '_'));
            }
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                foreach (string str in rgstrSrc1)
                {                
                    List<Source> rgSrc = entities.Sources.Where(p => rgstrSrc.Contains(p.Name)).ToList();
 
                    foreach (Source src in rgSrc)
                    {
                        DeleteSource(src.ID);
                    }
                }
            }
        }
 
        public virtual bool DeleteSourceData(int nSrcId = 0)
        {
            if (nSrcId == 0)
            {
                if (m_src == null)
                    return false;
 
                nSrcId = m_src.ID;
            }
 
            DeleteRawImageMeans(nSrcId);
            DeleteRawImageResults(nSrcId);
            DeleteRawImageParameters(nSrcId);
            DeleteRawImages(nSrcId);
 
            return true;
        }
 
        public void DeleteSourceData(string strSrc)
        {
            DeleteSourceData(GetSourceID(convertWs(strSrc, '_')));
        }
 
        public Dictionary<string, string> GetSourceParameters(int nSrcId = 0, ConnectInfo ci = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<SourceParameter> rgP = entities.SourceParameters.AsNoTracking().Where(p => p.SourceID == nSrcId).ToList();
                Dictionary<string, string> rgPval = new Dictionary<string, string>();
 
                foreach (SourceParameter p in rgP)
                {
                    rgPval.Add(p.Name, p.Value);
                }
 
                return rgPval;
            }
        }
 
        public string GetSourceParameter(string strName, int nSrcId = 0)
        {
            strName = convertWs(strName, '_');
 
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<SourceParameter> rgP = entities.SourceParameters.AsNoTracking().Where(p => p.SourceID == nSrcId && p.Name == strName).ToList();
 
                if (rgP.Count == 0)
                    return null;
 
                return rgP[0].Value;
            }
        }
 
        public int GetSourceParameter(string strName, int nDefault, int nSrcId = 0)
        {
            strName = convertWs(strName, '_');
            string strVal = GetSourceParameter(strName, nSrcId);
 
            if (strVal == null)
                return nDefault;
 
            return int.Parse(strVal);
        }
 
        public double GetSourceParameter(string strName, double dfDefault, int nSrcId = 0)
        {
            strName = convertWs(strName, '_');
            string strVal = GetSourceParameter(strName, nSrcId);
 
            if (strVal == null)
                return dfDefault;
 
            return BaseParameter.ParseDouble(strVal);
        }
 
        public bool GetSourceParameter(string strName, bool bDefault, int nSrcId = 0)
        {
            strName = convertWs(strName, '_');
            string strVal = GetSourceParameter(strName, nSrcId);
 
            if (strVal == null)
                return bDefault;
 
            return bool.Parse(strVal);
        }
 
        public void SetSourceParameter(string strName, string strValue, int nSrcId = 0)
        {
            strName = convertWs(strName, '_');
 
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<SourceParameter> rgP = entities.SourceParameters.Where(p => p.SourceID == nSrcId && p.Name == strName).ToList();
                SourceParameter sP = null;
 
                if (rgP.Count == 0)
                {
                    sP = new SourceParameter();
                    sP.Name = strName;
                    sP.SourceID = nSrcId;
                }
                else
                {
                    sP = rgP[0];
                }
 
                sP.Value = strValue;
 
                if (rgP.Count == 0)
                    entities.SourceParameters.Add(sP);
 
                entities.SaveChanges();
            }
        }
 
        public DateTime GetFirstTimeStamp(int nSrcId = 0, string strDesc = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImage> iquery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId).OrderBy(p => p.TimeStamp);
                if (strDesc != null)
                    iquery = iquery.Where(p => p.Description == strDesc);
 
                List<RawImage> rgImages = iquery.Take(1).ToList();
                if (rgImages.Count == 0)
                    return DateTime.MinValue;
 
                return rgImages[0].TimeStamp.GetValueOrDefault();
            }
        }
 
        public DateTime GetLastTimeStamp(int nSrcId = 0, string strDesc = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImage> iquery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId).OrderByDescending(p => p.TimeStamp);
                if (strDesc != null)
                    iquery = iquery.Where(p => p.Description == strDesc);
 
                List<RawImage> rgImages = iquery.Take(1).ToList();
                if (rgImages.Count == 0)
                    return DateTime.MinValue;
 
                return rgImages[0].TimeStamp.GetValueOrDefault();
            }
        }
 
        public DateTime GetLastTimeStamp(DateTime dtStart, DateTime dtEnd, bool bEndInclusive, int nSrcId = 0, string strDesc = null)
        {
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImage> iquery;
                
                if (bEndInclusive)
                    iquery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId && p.TimeStamp >= dtStart && p.TimeStamp <= dtEnd).OrderByDescending(p => p.TimeStamp);
                else
                    iquery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId && p.TimeStamp >= dtStart && p.TimeStamp < dtEnd).OrderByDescending(p => p.TimeStamp);
 
                if (strDesc != null)
                    iquery = iquery.Where(p => p.Description == strDesc);
 
                List<RawImage> rgImages = iquery.Take(1).ToList();
                if (rgImages.Count == 0)
                    return DateTime.MinValue;
 
                return rgImages[0].TimeStamp.GetValueOrDefault();
            }
        }
 
        public DateTime GetLastTimeStamp(out int nIndex, int nSrcId = 0, string strDesc = null)
        {
            nIndex = -1;
 
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImage> iquery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId).OrderByDescending(p => p.TimeStamp);
                if (strDesc != null)
                    iquery = iquery.Where(p => p.Description == strDesc);
 
                List<RawImage> rgImages = iquery.Take(1).ToList();
                if (rgImages.Count == 0)
                    return DateTime.MinValue;
 
                nIndex = rgImages[0].Idx.GetValueOrDefault(-1);
 
                return rgImages[0].TimeStamp.GetValueOrDefault();
            }
        }
 
        public DateTime GetLastTimeStamp(DateTime dtStart, DateTime dtEnd, bool bEndInclusive, out int nIndex, int nSrcId = 0, string strDesc = null)
        {
            nIndex = -1;
 
            if (nSrcId == 0)
                nSrcId = m_src.ID;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                IQueryable<RawImage> iquery;
 
                if (bEndInclusive)
                    iquery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId && p.TimeStamp >= dtStart && p.TimeStamp <= dtEnd).OrderByDescending(p => p.TimeStamp);
                else
                    iquery = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcId && p.TimeStamp >= dtStart && p.TimeStamp < dtEnd).OrderByDescending(p => p.TimeStamp);
 
                if (strDesc != null)
                    iquery = iquery.Where(p => p.Description == strDesc);
 
                List<RawImage> rgImages = iquery.Take(1).ToList();
                if (rgImages.Count == 0)
                    return DateTime.MinValue;
 
                nIndex = rgImages[0].Idx.GetValueOrDefault(-1);
 
                return rgImages[0].TimeStamp.GetValueOrDefault();
            }
        }
 
        #endregion
 
 
        //---------------------------------------------------------------------
        //  Datasets
        //---------------------------------------------------------------------
        #region Datasets
 
        public string FindDatasetNameFromSourceName(string strTrainSrc, string strTestSrc)
        {
            int nTrainSrcID = 0;
            int nTestSrcID = 0;
 
            if (!String.IsNullOrEmpty(strTestSrc))
                nTestSrcID = GetSourceID(strTestSrc);
 
            if (!String.IsNullOrEmpty(strTrainSrc))
                nTrainSrcID = GetSourceID(strTrainSrc);
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = null;
 
                if (nTestSrcID != 0 && nTrainSrcID != 0)
                    rgDs = entities.Datasets.AsNoTracking().Where(p => p.TrainingSourceID == nTrainSrcID && p.TestingSourceID == nTestSrcID).ToList();
                else if (nTestSrcID != 0)
                    rgDs = entities.Datasets.AsNoTracking().Where(p => p.TestingSourceID == nTestSrcID).ToList();
                else if (nTrainSrcID != 0)
                    rgDs = entities.Datasets.AsNoTracking().Where(p => p.TrainingSourceID == nTrainSrcID).ToList();
 
                if (rgDs != null && rgDs.Count > 0)
                    return rgDs[0].Name;
            }
 
            return null;
        }
 
        public int FindDatasetFromSourceId(int nSourceId)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = entities.Datasets.AsNoTracking().Where(p => p.TrainingSourceID == nSourceId || p.TestingSourceID == nSourceId).ToList();
                if (rgDs.Count == 0)
                    return 0;
 
                return rgDs[0].ID;
            }
        }
 
        public int GetDatasetID(string strName, ConnectInfo ci = null)
        {
            strName = convertWs(strName, '_');
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Dataset> rgDs = entities.Datasets.AsNoTracking().Where(p => p.Name == strName).ToList();
 
                if (rgDs.Count == 0)
                    return 0;
 
                return rgDs[0].ID;
            }
        }
 
        public string GetDatasetName(int nID, ConnectInfo ci = null)
        {
            Dataset ds = GetDataset(nID, ci);
 
            if (ds == null)
                return null;
 
            return ds.Name;
        }
 
        public Dataset GetDataset(int nID, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                IQueryable<Dataset> iQuery = entities.Datasets.AsNoTracking().Where(p => p.ID == nID);
 
                if (iQuery.Count() == 0)
                    return null;
 
                List<Dataset> rgDs = iQuery.ToList();
 
                if (rgDs.Count == 0)
                    return null;
 
                return rgDs[0];
            }
        }
 
        public Dataset GetDataset(string strName)
        {
            strName = convertWs(strName, '_');
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = entities.Datasets.AsNoTracking().Where(p => p.Name == strName).ToList();
 
                if (rgDs.Count == 0)
                    return null;
 
                return rgDs[0];
            }
        }
 
        public Dataset GetDataset(string strTestingSrc, string strTrainingSrc)
        {
            int nTestingSrcId = GetSourceID(strTestingSrc);
            int nTrainingSrcId = GetSourceID(strTrainingSrc);
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = entities.Datasets.AsNoTracking().Where(p => p.TestingSourceID == nTestingSrcId && p.TrainingSourceID == nTrainingSrcId).ToList();
 
                if (rgDs.Count == 0)
                    return null;
 
                return rgDs[0];
            }
        }
 
        public int AddDataset(int nDsCreatorID, string strName, int nTestSrcId, int nTrainSrcId, int nDsGroupID = 0, int nModelGroupID = 0, ConnectInfo ci = null, bool bVerify = true)
        {
            strName = convertWs(strName, '_');
 
            Source srcTest = GetSource(nTestSrcId, ci);
            if (srcTest == null)
                throw new Exception("Could not find either the test source with ID = " + nTestSrcId.ToString() + "!");
 
            Source srcTrain = GetSource(nTrainSrcId, ci);
            if (srcTrain == null)
                throw new Exception("Could not find either the train source with ID = " + nTrainSrcId.ToString() + "!");
 
            if (bVerify)
            {
                if (srcTest.ImageChannels.GetValueOrDefault() != srcTrain.ImageChannels.GetValueOrDefault())
                    throw new Exception("The test and train sources have different image channels!");
 
                if (srcTest.ImageHeight.GetValueOrDefault() != srcTrain.ImageHeight.GetValueOrDefault())
                    throw new Exception("The test and train sources have different image heights!");
 
                if (srcTest.ImageWidth.GetValueOrDefault() != srcTrain.ImageWidth.GetValueOrDefault())
                    throw new Exception("The test and train sources have different image widths!");
 
                if (srcTest.ImageEncoded.GetValueOrDefault() != srcTrain.ImageEncoded.GetValueOrDefault())
                    throw new Exception("The test and train sources have different image encodings!");
            }
 
            Dataset ds = null;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Dataset> rgDs = entities.Datasets.Where(p => p.Name == strName && p.DatasetCreatorID == nDsCreatorID).ToList();
 
                if (rgDs.Count > 0)
                {
                    ds = rgDs[0];
                }
                else
                {
                    ds = new Dataset();
                    ds.Name = strName;
                    ds.DatasetCreatorID = nDsCreatorID;
                }
 
                double dfPct = 0;
                double dfTotal = srcTest.ImageCount.GetValueOrDefault() + srcTrain.ImageCount.GetValueOrDefault();
 
                if (dfTotal > 0)
                    dfPct = srcTest.ImageCount.GetValueOrDefault() / dfTotal;
 
                ds.ImageChannels = srcTrain.ImageChannels;
                ds.ImageHeight = srcTrain.ImageHeight;
                ds.ImageWidth = srcTrain.ImageWidth;
                ds.ImageEncoded = srcTrain.ImageEncoded;
                ds.DatasetGroupID = nDsGroupID;
                ds.ModelGroupID = nModelGroupID;
                ds.TestingTotal = srcTest.ImageCount;
                ds.TrainingTotal = srcTrain.ImageCount;
                ds.TestingPercent = (decimal)dfPct;
                ds.Relabeled = false;
                ds.TestingSourceID = srcTest.ID;
                ds.TrainingSourceID = srcTrain.ID;
 
                if (rgDs.Count == 0)
                    entities.Datasets.Add(ds);
 
                entities.SaveChanges();
            }
 
            UpdateLabelCounts(srcTest.ID, 0, ci);
            UpdateLabelCounts(srcTrain.ID, 0, ci);
 
            if (ds == null)
                return 0;
 
            return ds.ID;
        }
 
        public void UpdateDatasetDescription(int nDsId, string strDesc)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = entities.Datasets.Where(p => p.ID == nDsId).ToList();
 
                if (rgDs.Count == 0)
                    return;
 
                rgDs[0].Description = strDesc;
                entities.SaveChanges();
            }
        }
 
        public void UpdateDatasetCounts(int nDsId, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<Dataset> rgDs = entities.Datasets.Where(p => p.ID == nDsId).ToList();
 
                if (rgDs.Count == 0)
                    return;
 
                int nTestSrcId = rgDs[0].TestingSourceID.GetValueOrDefault();
                List<Source> rgSrcTest = entities.Sources.Where(p => p.ID == nTestSrcId).ToList();
 
                int nTrainSrcId = rgDs[0].TrainingSourceID.GetValueOrDefault();
                List<Source> rgSrcTrain = entities.Sources.Where(p => p.ID == nTrainSrcId).ToList();
 
                int nTestingTotal = rgSrcTest[0].ImageCount.GetValueOrDefault(0);
                int nTrainingTotal = rgSrcTrain[0].ImageCount.GetValueOrDefault(0);
 
                rgDs[0].TestingTotal = nTestingTotal;
                rgDs[0].TrainingTotal = nTrainingTotal;
                rgDs[0].TestingPercent = 0;
 
                if (nTrainingTotal + nTrainingTotal > 0)
                    rgDs[0].TestingPercent = (decimal)nTestingTotal / (decimal)(nTestingTotal + nTrainingTotal);
 
                entities.SaveChanges();
            }
        }
 
        public int PutDataset(Dataset ds)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = null;
                
                if (ds.ID != 0)
                    rgDs = entities.Datasets.Where(p => p.ID == ds.ID).ToList();
 
                if (rgDs != null && rgDs.Count > 0)
                {
                    rgDs[0].DatasetGroupID = ds.DatasetGroupID;
                    rgDs[0].Name = ds.Name;
                    rgDs[0].TestingSourceID = ds.TestingSourceID;
                    rgDs[0].TrainingSourceID = ds.TrainingSourceID;
                    rgDs[0].DatasetCreatorID = ds.DatasetCreatorID;
                    rgDs[0].ImageChannels = ds.ImageChannels;
                    rgDs[0].ImageEncoded = ds.ImageEncoded;
                    rgDs[0].ImageHeight = ds.ImageHeight;
                    rgDs[0].ImageWidth = ds.ImageWidth;
                    rgDs[0].ModelGroupID = ds.ModelGroupID;
                    rgDs[0].TestingPercent = ds.TestingPercent;
                    rgDs[0].TrainingTotal = ds.TrainingTotal;
                    rgDs[0].TestingTotal = ds.TestingTotal;
                }
                else
                {
                    entities.Datasets.Add(ds);
                }
 
                entities.SaveChanges();
            }
 
            return ds.ID;
        }
 
        public DatasetGroup GetDatasetGroup(int nGroupID, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<DatasetGroup> rgGroups = entities.DatasetGroups.AsNoTracking().Where(p => p.ID == nGroupID).ToList();
 
                if (rgGroups.Count == 0)
                    return null;
 
                return rgGroups[0];
            }
        }
 
        public string GetDatasetGroupName(int nGroupID)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<DatasetGroup> rgGroups = entities.DatasetGroups.AsNoTracking().Where(p => p.ID == nGroupID).ToList();
 
                if (rgGroups.Count == 0)
                    return null;
 
                return rgGroups[0].Name;
            }
        }
 
        public Dictionary<string, string> GetDatasetParameters(int nDsId, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<DatasetParameter> rgP = entities.DatasetParameters.AsNoTracking().Where(p => p.DatasetID == nDsId).ToList();
                Dictionary<string, string> rgDsP = new Dictionary<string, string>();
 
                foreach (DatasetParameter p in rgP)
                {
                    if (!rgDsP.ContainsKey(p.Name))
                        rgDsP.Add(p.Name, p.Value);
                }
 
                return rgDsP;
            }
        }
 
        public string GetDatasetParameter(int nDsId, string strName)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<DatasetParameter> rgP = entities.DatasetParameters.AsNoTracking().Where(p => p.DatasetID == nDsId && p.Name == strName).ToList();
 
                if (rgP.Count == 0)
                    return null;
 
                return rgP[0].Value;
            }
        }
 
        public int GetDatasetParameter(int nDsId, string strName, int nDefault)
        {
            string strVal = GetDatasetParameter(nDsId, strName);
 
            if (strVal == null)
                return nDefault;
 
            return int.Parse(strVal);
        }
 
        public double GetDatasetParameter(int nDsId, string strName, double dfDefault)
        {
            string strVal = GetDatasetParameter(nDsId, strName);
 
            if (strVal == null)
                return dfDefault;
 
            return BaseParameter.ParseDouble(strVal);
        }
 
        public bool GetDatasetParameter(int nDsId, string strName, bool bDefault)
        {
            string strVal = GetDatasetParameter(nDsId, strName);
 
            if (strVal == null)
                return bDefault;
 
            return bool.Parse(strVal);
        }
 
        public void SetDatasetParameter(int nDsId, string strName, string strValue)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<DatasetParameter> rgP = entities.DatasetParameters.Where(p => p.DatasetID == nDsId && p.Name == strName).ToList();
                DatasetParameter dsP = null;
 
                if (rgP.Count == 0)
                {
                    dsP = new DatasetParameter();
                    dsP.Name = strName;
                    dsP.DatasetID = nDsId;
                }
                else
                {
                    dsP = rgP[0];
                }
 
                dsP.Value = strValue;
 
                if (rgP.Count == 0)
                    entities.DatasetParameters.Add(dsP);
 
                entities.SaveChanges();
            }
        }
 
        public void SetDatasetParameters(int nDsId, Dictionary<string, string> rgP)
        {
            WaitForFileWriter();
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Configuration.AutoDetectChangesEnabled = false;
                entities.Configuration.ValidateOnSaveEnabled = false;
 
                foreach (KeyValuePair<string, string> kv in rgP)
                {
                    DatasetParameter dsp = new DatasetParameter();
                    dsp.Name = kv.Key;
                    dsp.Value = kv.Value;
                    dsp.DatasetID = nDsId;
                    entities.DatasetParameters.Add(dsp);
                }
 
                entities.SaveChanges();
            }
        }
 
        public virtual void DeleteDataset(string strDsName, bool bDeleteRelatedProjects, Log log, CancelEvent evtCancel)
        {
            Dataset ds = GetDataset(strDsName);
            if (ds == null)
                return;
 
            Source srcTraining = GetSource(ds.TrainingSourceID.GetValueOrDefault());
            Source srcTesting = GetSource(ds.TestingSourceID.GetValueOrDefault());
            string strCmd;
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                entities.Database.CommandTimeout = 180;
 
                strCmd = "DELETE RawImageParameters WHERE (SourceID = " + ds.TestingSourceID.GetValueOrDefault().ToString() + ") OR (SourceID = " + ds.TrainingSourceID.GetValueOrDefault().ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE Labels WHERE (SourceID = " + ds.TestingSourceID.GetValueOrDefault().ToString() + ") OR (SourceID = " + ds.TrainingSourceID.GetValueOrDefault().ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE RawImageMeans WHERE (SourceID = " + ds.TestingSourceID.GetValueOrDefault().ToString() + ") OR (SourceID = " + ds.TrainingSourceID.GetValueOrDefault().ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE RawImageResults WHERE (SourceID = " + ds.TestingSourceID.GetValueOrDefault().ToString() + ") OR (SourceID = " + ds.TrainingSourceID.GetValueOrDefault().ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE RawImages WHERE (SourceID = " + ds.TestingSourceID.GetValueOrDefault().ToString() + ") OR (SourceID = " + ds.TrainingSourceID.GetValueOrDefault().ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE SourceParameters WHERE (SourceID = " + ds.TestingSourceID.GetValueOrDefault().ToString() + ") OR (SourceID = " + ds.TrainingSourceID.GetValueOrDefault().ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE Sources WHERE (ID = " + ds.TestingSourceID.GetValueOrDefault().ToString() + ") OR (ID = " + ds.TrainingSourceID.GetValueOrDefault().ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE DatasetParameters WHERE (DatasetID = " + ds.ID.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                strCmd = "DELETE Datasets WHERE (ID = " + ds.ID.ToString() + ")";
                entities.Database.ExecuteSqlCommand(strCmd);
 
                DeleteFiles del = new DeleteFiles(log, evtCancel);
 
                if (srcTesting != null && !string.IsNullOrEmpty(srcTesting.Name))
                    del.DeleteDirectory(getImagePathBase(srcTesting.Name, entities));
 
                if (srcTraining != null && !string.IsNullOrEmpty(srcTraining.Name))
                    del.DeleteDirectory(getImagePathBase(srcTraining.Name, entities));
            }
        }
 
        public List<Dataset> GetAllDatasets(int nDatasetGroupID)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                if (nDatasetGroupID > 0)
                    return entities.Datasets.AsNoTracking().Where(p => p.DatasetGroupID == nDatasetGroupID).ToList();
 
                return entities.Datasets.ToList();
            }
        }
 
        public List<Dataset> GetAllDatasetsWithCreators(int nDatasetGroupID)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = new List<Dataset>();
                IQueryable<Dataset> iQuery = entities.Datasets.AsNoTracking().Where(p => p.DatasetCreatorID > 0);
 
                if (iQuery.Count() > 0)
                {
                    rgDs = iQuery.ToList();
 
                    if (nDatasetGroupID > 0)
                        return rgDs.Where(p => p.DatasetGroupID == nDatasetGroupID).ToList();
                }
 
                return rgDs;
            }
        }
 
        public List<Dataset> GetAllDatasetsWithCreator(int nDsCreatorID, bool? bRelabeled = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                if (bRelabeled.HasValue)
                {
                    bool bRelabeledValue = bRelabeled.Value;
                    return entities.Datasets.AsNoTracking().Where(p => p.DatasetCreatorID == nDsCreatorID && p.Relabeled == bRelabeled).ToList();
                }
                else
                {
                    return entities.Datasets.Where(p => p.DatasetCreatorID == nDsCreatorID).ToList();
                }
            }
        }
 
        public int GetDatasetGroupID(string strName)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<DatasetGroup> rgGroups = entities.DatasetGroups.AsNoTracking().Where(p => p.Name == strName).ToList();
 
                if (rgGroups.Count == 0)
                    return 0;
 
                return rgGroups[0].ID;
            }
        }
 
        public string GetDatasetCreatorName(int nDatasetCreatorID, ConnectInfo ci = null)
        {
            if (ci == null)
                ci = EntitiesConnection.GlobalDatabaseConnectInfo;
 
            if (ci != null && !string.IsNullOrEmpty(ci.Password))
                return "";
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<DatasetCreator> rgDsc = entities.DatasetCreators.AsNoTracking().Where(p => p.ID == nDatasetCreatorID).ToList();
 
                if (rgDsc.Count == 0)
                    return "";
 
                return rgDsc[0].Name;
            }
        }
 
        public int GetDatasetCreatorID(string strName)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<DatasetCreator> rgDsc = entities.DatasetCreators.AsNoTracking().Where(p => p.Name == strName).ToList();
 
                if (rgDsc.Count == 0)
                    return 0;
 
                return rgDsc[0].ID;
            }
        }
 
        public void ResetAllDatasetRelabelWithCreator(int nDsCreatorID)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = entities.Datasets.Where(p => p.DatasetCreatorID == nDsCreatorID && p.Relabeled == true).ToList();
 
                foreach (Dataset ds in rgDs)
                {
                    ds.Relabeled = false;
                }
 
                entities.SaveChanges();
            }
        }
 
        public void UpdateDatasetRelabel(int nDsID, bool bRelabel)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = entities.Datasets.Where(p => p.ID == nDsID).ToList();
 
                foreach (Dataset ds in rgDs)
                {
                    ds.Relabeled = bRelabel;
                }
 
                entities.SaveChanges();
            }
        }
 
        public DateTime GetDatasetMinimumTimestamp(int nDsID)
        {
            Dataset ds = GetDataset(nDsID);
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                int nSrcTestId = ds.TestingSourceID.GetValueOrDefault();
                int nSrcTrainId = ds.TrainingSourceID.GetValueOrDefault();
 
                List<RawImage> rgImg = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcTestId || p.SourceID == nSrcTrainId).OrderBy(p => p.TimeStamp).Take(1).ToList();
                if (rgImg.Count == 0)
                    return DateTime.MinValue;
 
                return rgImg[0].TimeStamp.GetValueOrDefault(DateTime.MinValue);
            }
        }
 
        public DateTime GetDatasetMaximumTimestamp(int nDsID)
        {
            Dataset ds = GetDataset(nDsID);
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                int nSrcTestId = ds.TestingSourceID.GetValueOrDefault();
                int nSrcTrainId = ds.TrainingSourceID.GetValueOrDefault();
 
                List<RawImage> rgImg = entities.RawImages.AsNoTracking().Where(p => p.SourceID == nSrcTestId || p.SourceID == nSrcTrainId).OrderByDescending(p => p.TimeStamp).Take(1).ToList();
                if (rgImg.Count == 0)
                    return DateTime.MinValue;
 
                return rgImg[0].TimeStamp.GetValueOrDefault(DateTime.MinValue);
            }
        }
 
        public void UpdateDatasetCounts(CancelEvent evtCancel, Log log, int nDatasetCreatorID, List<string> rgstrDs, string strParamNameForDescription)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Dataset> rgDs = entities.Datasets.Where(p => p.DatasetCreatorID == nDatasetCreatorID).ToList();
 
                foreach (Dataset ds in rgDs)
                {
                    bool bContainsDs = false;
                    int nIdx;
                    Stopwatch sw = new Stopwatch();
 
                    if (evtCancel.WaitOne(0))
                    {
                        log.WriteLine("Aborting dataset count update.");
                        return;
                    }
 
                    if (rgstrDs != null)
                    {
                        foreach (string str in rgstrDs)
                        {
                            if (ds.Name.Contains(str))
                            {
                                bContainsDs = true;
                                break;
                            }
                        }
 
                        if (!bContainsDs)
                            continue;
                    }
 
                    log.WriteLine("Updating '" + ds.Name + "'...");
 
                    int nSrcIdTraining = ds.TrainingSourceID.GetValueOrDefault();
                    if (nSrcIdTraining> 0)
                    {
                        List<Source> rgSrc = entities.Sources.Where(p => p.ID == nSrcIdTraining).ToList();
                        if (rgSrc.Count > 0)
                        {
                            Source src = rgSrc[0];
                            int nImageCount = entities.RawImages.Where(p => p.SourceID == nSrcIdTraining).Count();
 
                            if (src.ImageCount != nImageCount)
                                src.ImageCount = nImageCount;
 
                            if (ds.TrainingTotal != nImageCount)
                                ds.TrainingTotal = nImageCount;
 
                            UpdateLabelCounts(src.ID);
 
                            if (bContainsDs)
                            {
                                List<int> rgId = QueryAllRawImageIDs(src.ID);
 
                                nIdx = 0;
                                sw.Restart();
 
                                foreach (int nID in rgId)
                                {
                                    if (evtCancel.WaitOne(0))
                                        return;
 
                                    UpdateRawImageDescriptionFromParameter(nID, strParamNameForDescription);
                                    nIdx++;
 
                                    if (sw.Elapsed.TotalMilliseconds > 1000)
                                    {
                                        log.Progress = ((double)nIdx / (double)rgId.Count);
                                        sw.Restart();
                                    }
                                }
                            }
                        }
                    }
 
                    int nSrcIdTesting = ds.TestingSourceID.GetValueOrDefault();
                    if (nSrcIdTesting > 0)
                    {
                        List<Source> rgSrc = entities.Sources.Where(p => p.ID == nSrcIdTesting).ToList();
                        if (rgSrc.Count > 0)
                        {
                            Source src = rgSrc[0];
                            int nImageCount = entities.RawImages.Where(p => p.SourceID == nSrcIdTesting).Count();
 
                            if (src.ImageCount != nImageCount)
                                src.ImageCount = nImageCount;
 
                            if (ds.TestingTotal != nImageCount)
                                ds.TestingTotal = nImageCount;
 
                            UpdateLabelCounts(src.ID);
 
                            if (bContainsDs)
                            {
                                List<int> rgId = QueryAllRawImageIDs(src.ID);
 
                                nIdx = 0;
                                sw.Restart();
 
                                foreach (int nID in rgId)
                                {
                                    if (evtCancel.WaitOne(0))
                                        return;
 
                                    UpdateRawImageDescriptionFromParameter(nID, strParamNameForDescription);
                                    nIdx++;
 
                                    if (sw.Elapsed.TotalMilliseconds > 1000)
                                    {
                                        log.Progress = ((double)nIdx / (double)rgId.Count);
                                        sw.Restart();
                                    }
                                }
                            }
                        }
                    }
 
                    double dfTestingPct = (double)ds.TestingTotal / (double)(ds.TestingTotal + ds.TrainingTotal);
                    if (ds.TestingPercent != (decimal)dfTestingPct)
                        ds.TestingPercent = (decimal)dfTestingPct;
 
                    entities.SaveChanges();
                }
            }
        }
 
        public ModelGroup GetModelGroup(int nGroupID, ConnectInfo ci = null)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities(ci))
            {
                List<ModelGroup> rgGroups = entities.ModelGroups.AsNoTracking().Where(p => p.ID == nGroupID).ToList();
 
                if (rgGroups.Count == 0)
                    return null;
 
                return rgGroups[0];
            }
        }
 
        public string GetModelGroupName(int nGroupID)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<ModelGroup> rgGroups = entities.ModelGroups.AsNoTracking().Where(p => p.ID == nGroupID).ToList();
 
                if (rgGroups.Count == 0)
                    return null;
 
                return rgGroups[0].Name;
            }
        }
 
        public int GetModelGroupID(string strGroup)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<ModelGroup> rgGroup = entities.ModelGroups.AsNoTracking().Where(p => p.Name == strGroup).ToList();
                if (rgGroup.Count == 0)
                    return 0;
 
                return rgGroup[0].ID;
            }
        }
 
        public List<Dataset> GetAllDatasetsInModelGroup(int nModelGroupId)
        {
            using (DNNEntities entities = MyCaffe.db.image.EntitiesConnection.CreateEntities())
            {
                return entities.Datasets.AsNoTracking().Where(p => p.ModelGroupID == nModelGroupId).ToList();
            }
        }
 
        public void DeleteModelGroup(string strGroup, Log log, CancelEvent evtCancel)
        {
            int nModelGroupId = GetModelGroupID(strGroup);
            List<Dataset> rgDs = GetAllDatasetsInModelGroup(nModelGroupId);
 
            foreach (Dataset ds in rgDs)
            {
                DeleteDataset(ds.Name, false, log, evtCancel);
            }
 
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                string strCmd = "DELETE FROM ModelGroups WHERE ([Name] = '" + strGroup + "')";
                entities.Database.ExecuteSqlCommand(strCmd);
            }
 
        }
 
        #endregion
    }
 
#pragma warning disable 1591
 
    class FileDataCollection 
    {
        List<Tuple<string, byte[]>> m_rgData = new List<Tuple<string, byte[]>>();
        object m_syncObj = new object();
 
        public FileDataCollection()
        { 
        }
 
        public int Count
        {
            get { return m_rgData.Count; }
        }
 
        public void Add(string strFile, byte[] rgData)
        {
            lock (m_syncObj)
            {
                m_rgData.Add(new Tuple<string, byte[]>(strFile, rgData));
            }
        }
 
        public Tuple<string, byte[]> Remove()
        {
            lock (m_syncObj)
            {
                if (m_rgData.Count == 0)
                    return null;
 
                Tuple<string, byte[]> item = m_rgData[0];
                m_rgData.RemoveAt(0);
                return item;
            }
        }
    }
 
    class FileWriter : IDisposable 
    {
        FileDataCollection m_rgData = new FileDataCollection();
        Task[] m_rgFileWriterThreads = null;
        ManualResetEvent m_evtCancelFileWriterThread = new ManualResetEvent(false);
        ManualResetEvent m_evtFileWriterDone = new ManualResetEvent(false);
        ManualResetEvent m_evtFileWriterEmpty = new ManualResetEvent(true);
        int m_nPeriod = 1000;
        int m_nThreads = 50;
 
        public FileWriter(int nThreads = 50, int nPeriod = 1000)
        {
            m_nPeriod = nPeriod;
            m_nThreads = nThreads;
        }
 
        public void Dispose()
        {
            Cancel();
            WaitUntilCompletion();
 
            if (m_rgFileWriterThreads != null)
            {
                foreach (Task t in m_rgFileWriterThreads)
                {
                    t.Dispose();
                }
            }
        }
 
        public void Cancel()
        {
            m_evtCancelFileWriterThread.Set();
        }
 
        public void Add(string strPath, byte[] rgData)
        {
            m_evtFileWriterEmpty.Reset();
            m_rgData.Add(strPath, rgData);
 
            if (m_rgFileWriterThreads == null)
            {
                m_evtCancelFileWriterThread.Reset();
                m_evtFileWriterDone.Reset();
                m_rgFileWriterThreads = new Task[m_nThreads];
 
                for (int i = 0; i < m_nThreads; i++)
                {
                    m_rgFileWriterThreads[i] = Task.Factory.StartNew(new Action(fileWriterThread));
                }
            }
        }
 
        public bool WaitUntilCompletion(int nWait = int.MaxValue)
        {
            if (m_rgFileWriterThreads == null)
                return true;
 
            if (m_evtFileWriterEmpty.WaitOne(nWait))
                return true;
 
            return false;
        }
 
        private void fileWriterThread()
        {
            try
            {
                int nWait = m_nPeriod;
 
                while (!m_evtCancelFileWriterThread.WaitOne(nWait))
                {
                    Tuple<string, byte[]> item = m_rgData.Remove();
                    if (item != null)
                    {
                        File.WriteAllBytes(item.Item1, item.Item2);
                        Thread.Sleep(0);
                        nWait = 5;
                    }
                    else
                    {
                        m_evtFileWriterEmpty.Set();
                        nWait = m_nPeriod;
                    }
                }
            }
            finally
            {
                m_evtFileWriterDone.Set();
            }
        }
    }
 
#pragma warning restore 1591
 
    public class ParameterData
    {
        string m_strName;
        string m_strValue = null;
        double? m_dfValue = null;
        byte[] m_rgValue = null;
        int m_nImageID = 0;
        bool m_bOnlyAddNew = false;
        int m_nSrcId = 0;
 
        public ParameterData(string strName, string strValue, double? dfVal = null, byte[] rgData = null, int nImageID = 0, bool bOnlyAddNew = false, int nSrcId = 0)
        {
            m_strName = strName;
            m_strValue = strValue;
            m_dfValue = dfVal;
            m_rgValue = rgData;
            m_nImageID = nImageID;
            m_bOnlyAddNew = bOnlyAddNew;
            m_nSrcId = nSrcId;
        }
 
        public ParameterData(string strName, int nImageID)
        {
            m_strName = strName;
            m_nImageID = nImageID;
        }
 
        public string Name
        {
            get { return m_strName; }
        }
 
        public int SourceID
        {
            get { return m_nSrcId; }
            set { m_nSrcId = value; }
        }
 
        public bool OnlyAddNew
        {
            get { return m_bOnlyAddNew; }
        }
 
        public string Value
        {
            get { return m_strValue; }
        }
 
        public double? NumericValue
        {
            get { return m_dfValue; }
        }
 
        public byte[] Data
        {
            get { return m_rgValue; }
        }
 
        public int ImageID
        {
            get { return m_nImageID; }
            set { m_nImageID = value; }
        }
    }
 
    public class DbItem
    {
        object m_tag;
 
        public DbItem()
        {
        }
 
        public DbItem Clone()
        {
            DbItem item = new DbItem();
            item.id = id;
            item.virtualid = virtualid;
            item.index = index;
            item.label = label;
            item.boost = boost;
            item.time = time;
            item.desc = desc;
            item.originalsrcid = originalsrcid;
            item.active = active;
            return item;
        }
 
        public object Tag
        {
            get { return m_tag; }
            set { m_tag = value; }
        }
 
        public int ID
        {
            get { return id; }
        }
 
        public int id { get; set; } 
 
        public int VirtualID
        {
            get { return virtualid.GetValueOrDefault(); }
        }
 
        public int? virtualid { get; set; }
 
        public int Index
        {
            get { return index.GetValueOrDefault(); }
        }
 
        public int? index { get;  set; } 
 
        public int Label
        {
            get { return label.GetValueOrDefault(); }
        }
 
        public int? label { get;  set; } 
 
        public int Boost
        {
            get { return boost.GetValueOrDefault(); }
        }
 
        public int? boost { get; set; } 
 
        public DateTime Time
        {
            get { return time.GetValueOrDefault(); }
        }
 
        public DateTime? time { get; set; } 
 
        public string Desc
        {
            get { return desc; }
        }
 
        public string desc { get; set; }
 
        public int? OriginalSourceID
        {
            get { return originalsrcid; }
        }
 
        public int? originalsrcid { get; set; }
 
        public bool Active
        {
            get { return active.GetValueOrDefault(false); }
        }
 
        public bool? active { get; set; }
 
        public override string ToString()
        {
            return "Active=" + Active.ToString() + "; ID=" + ID.ToString() + "; Idx=" + Index.ToString() + "; Label=" + Label.ToString() + "; Time=" + Time.ToString() + "; Desc=" + Desc + "; Boost=" + Boost.ToString();
        }
    }
}