VocabularyCharacter.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
 
namespace MyCaffe.layers.gpt
{
    public class VocabularyCharacter : IVocabulary
    {
        bool m_bEnablePad = true;
        Random m_random;
        Dictionary<char, int> m_rgVocabKeyToIdx = new Dictionary<char, int>();
        Dictionary<int, char> m_rgVocabIdxToKey = new Dictionary<int, char>();
        bool m_bAddBos;
        bool m_bAddEos;
 
        public VocabularyCharacter(Random random, bool bAddBos, bool bAddEos, bool bEnablePad)
        {
            m_random = random;
            m_bAddBos = bAddBos;
            m_bAddEos = bAddEos;
            m_bEnablePad = bEnablePad;
            
            if (bAddBos)
                m_rgVocabKeyToIdx.Add(BOS, 1);
 
            if (bAddEos)
                m_rgVocabKeyToIdx.Add(EOS, 2);
        }
 
        public int Count
        {
            get { return m_rgVocabKeyToIdx.Count + ((m_bEnablePad) ? 1 : 0); }
        }
 
        public void Add(char ch)
        {
            if (!m_rgVocabKeyToIdx.ContainsKey(ch))
                m_rgVocabKeyToIdx.Add(ch, 1);
        }
 
        public void Add(string str)
        {
            foreach (char ch in str)
            {
                Add(ch);
            }
        }
 
        public int Build()
        {
            List<char> rgKeys = m_rgVocabKeyToIdx.Keys.ToList();
            rgKeys.Sort();
 
            m_rgVocabKeyToIdx.Clear();
 
            int nPadOffset = (m_bEnablePad) ? 1 : 0;
            
            // index 0 reserved for pad.
            for (int i = 0; i < rgKeys.Count; i++)
            {
                m_rgVocabKeyToIdx.Add(rgKeys[i], i + nPadOffset);
                m_rgVocabIdxToKey.Add(i + nPadOffset, rgKeys[i]);
            }
 
            return Count;
        }
 
        public int BuildFromString(string strData)
        {
            foreach (char ch in strData)
            {
                Add(ch);
            }
 
            return Build();
        }
 
        public char BOS
        {
            get { return (char)1; }
        }
 
        public char EOS
        {
            get { return (char)2; }
        }
 
        public int[] CreateTarget(int[] rgSrc)
        {
            List<int> rgTrg = new List<int>(rgSrc);
 
            rgTrg.RemoveAt(0);
            rgTrg.Add(EOS);
 
            return rgTrg.ToArray();
        }
 
        public List<int> Tokenize(string str1, bool bMustExist = true)
        {
            if (str1.Length != 1)
                throw new Exception("The character must be a single character!");
 
            List<int> rgTokens = new List<int>();
            char ch = str1[0];
 
            if (!m_rgVocabKeyToIdx.ContainsKey(ch))
            {
                if (bMustExist)
                    throw new Exception("The character '" + ch.ToString() + " is not in the vocabulary!");
                else
                    rgTokens.Add(m_random.Next(Count));
            }
 
            rgTokens.Add(m_rgVocabKeyToIdx[ch]);
            return rgTokens;
        }
 
        public int[] Tokenize(string str, bool bAddBos, bool bAddEos)
        {
            List<int> rgTokens = new List<int>();
 
            foreach (char ch in str)
            {                
                rgTokens.AddRange(Tokenize(ch.ToString()));
            }
 
            if (bAddBos)
                rgTokens.Insert(0, BOS);
 
            if (bAddEos)
                rgTokens.Add(EOS);
 
            return rgTokens.ToArray();
        }
 
        public string Detokenize(int nIdxToken, bool bIgnoreBos, bool bIgnoreEos)
        {
            string str = "";
 
            if (m_bAddBos && nIdxToken == BOS)
            {
                if (!bIgnoreBos)
                    str += "<BOS>";
            }
 
            else if (m_bAddEos && nIdxToken == EOS)
            {
                if (!bIgnoreEos)
                    str += "<EOS>";
            }
 
            else
            {
                if (m_rgVocabIdxToKey.ContainsKey(nIdxToken))
                    str += m_rgVocabIdxToKey[nIdxToken];
                else if (nIdxToken == 0)
                    str += "";
                else
                    throw new Exception("The token '" + nIdxToken.ToString() + "' is not in the vocabulary!");
            }
            
            return str;
        }
 
        public string Detokenize(float[] rgf, bool bIgnoreBos, bool bIgnoreEos)
        {
            string str = "";
 
            foreach (float f in rgf)
            {
                string str1 = Detokenize((int)f, bIgnoreBos, bIgnoreEos);
 
                if (!string.IsNullOrEmpty(str1))
                {
                    char ch = str1[0];
 
                    if (ch == EOS)
                        break;
 
                    if (ch != 0 && ch != BOS && ch != EOS)
                        str += ch;
                }
            }
 
            return str;
        }
    }
}