/// <summary>
/// Token read from input stream and placed in parse tree
/// </summary>
public class Token
{
#region private members
//valid symbols
private static char[] _s = { 'd', '+', '-', '(', ')', 'x', '*', ',' };
//type of this token
private TokenType _t;
//if this tokens type is Number it is the numbers value
//if this tokens type is Symbol, it is the index of the symbol within _s
private int _v;
//index of this token within the expression
private int _n;
//left hand arguement of symbol
private Token _l;
//right hand arguement of symbol
private Token _r;
//random number generator, added to allow
//seeding of rolls for deterministic behavior
private Random _roll;
#endregion
#region public properties
/// <summary>
/// Left hand arguement of this symbol.
/// </summary>
public Token L
{
get { return _l; }
set { _l = value; }
}
/// <summary>
/// Right hand arguement of symbol.
/// </summary>
public Token R
{
get { return _r; }
set { _r = value; }
}
/// <summary>
/// Index of this token within the expression.
/// </summary>
public int I
{
get { return _n; }
}
/// <summary>
/// Type of this token.
/// </summary>
public TokenType T
{
get { return _t; }
}
/// <summary>
/// Char representing this token. For symbols it is the tokens symbol.
/// Other tokens are invalid
/// </summary>
public char CharValue
{
get
{
switch (_t)
{
case TokenType.Symbol:
return _s[_v];
default:
return '_';
}
}
}
/// <summary>
/// The value generated by this token and its child nodes within the parse tree.
/// Valid tokens include the symbols '+', '-', 'd', 'x', '(' and numbers.
/// Invalid, end, '*' and ',' tokens return 0.
/// </summary>
public int IntValue
{
get
{
if (_t == TokenType.Number)
return _v;
else if (_t == TokenType.Symbol)
{
int count, sum, rng, val;
switch (CharValue)
{
case '(':
return _l.IntValue;
case 'd':
sum = 0;
rng = 0;
val = 0;
count = _l.IntValue;
if (_roll == null)
_roll = new Random();
for (int i = 0; i < count; i++)
{
rng = _r.IntValue + 1;
if (rng < 1)
break;
val = _roll.Next(1, rng);
sum += val;
}
return sum;
case 'x':
val = 0;
sum = 0;
rng = 0;
count = _l.IntValue;
for (int i = 0; i < count; i++)
{
rng = _r.IntValue + 1;
if (rng < 2)
break;
else if (rng == 2)
sum += 1; //do not explode when min and max are 1
else
{
do
{
if (_roll == null)
_roll = new Random();
val = _roll.Next(1, rng);
sum += val;
}
while (val == rng - 1);
}
}
return sum;
case '+':
return _l.IntValue + _r.IntValue;
case '-':
return _l.IntValue - _r.IntValue;
default:
return 0;
}
}
else
return 0;
}
}
#endregion
#region constructors
/// <summary>
/// Creates an end token.
/// </summary>
/// <param name="ndx">Index for this token within the expression.</param>
public Token(int ndx)
{
_n = ndx;
_t = TokenType.End;
}
/// <summary>
/// Creats a numeric token.
/// </summary>
/// <param name="number">value of this number</param>
/// <param name="ndx">Index for this token within the expression.</param>
public Token(int number, int ndx)
{
_n = ndx;
_t = TokenType.Number;
_v = number;
}
//create a symbol token
/// <summary>
/// Creates a symbol token if its symbol is valid,
/// otherwise creates an invalid token.
/// </summary>
/// <param name="symbol">char value of symbol</param>
/// <param name="ndx">Index for this token within the expression.</param>
public Token(char symbol, int ndx, Random rnd)
{
_roll = rnd;
_n = ndx;
if (_s.Contains(symbol))
{
_t = TokenType.Symbol;
_v = Array.IndexOf(_s, symbol);
}
else
_t = TokenType.Invalid;
}
#endregion
#region public methods
/// <summary>
/// The list of values generated by this token and its child nodes within the parse tree.
/// The symbols '+', '-', 'd', 'x', '(' and numbers return a list with a single element.
/// The symbols '*' and ',' return a list with a number of elements equal to its left hand arguement.
/// Other tokens are invalid.
/// </summary>
/// <param name="limit">Limit placed on number of values returned for performance reasons.</param>
public List<int> IntValues(int limit)
{
if (_t == TokenType.Number)
return new List<int> { _v };
else if (_t == TokenType.Symbol)
{
List<int> lst = new List<int>();
switch (CharValue)
{
case '(':
case 'd':
case 'x':
case '+':
case '-':
lst.Add(IntValue);
return lst;
case ',':
lst.AddRange(_l.IntValues(limit));
lst.AddRange(_r.IntValues(limit - lst.Count));
return lst;
case '*':
int count = _l.IntValue;
if (count > limit)
count = limit;
for (int i = 0; i < count; i++)
lst.Add(_r.IntValue);
return lst;
default:
return null;
}
}
else
return null;
}
/// <summary>
/// The list of values generated by this token and its child nodes within the parse tree.
/// The symbols '+', '-', 'd', 'x', '(' and numbers return a list with a single element.
/// The symbols '*' and ',' return a list with a number of elements equal to its left hand arguement.
/// Other tokens are invalid.
/// </summary>
public List<int> IntValues()
{
if (_t == TokenType.Number)
return new List<int> { _v };
else if (_t == TokenType.Symbol)
{
List<int> lst = new List<int>();
switch (CharValue)
{
case '(':
case 'd':
case 'x':
case '+':
case '-':
lst.Add(IntValue);
return lst;
case ',':
lst.AddRange(_l.IntValues());
lst.AddRange(_r.IntValues());
return lst;
case '*':
int count = _l.IntValue;
for (int i = 0; i < count; i++)
lst.Add(_r.IntValue);
return lst;
default:
return null;
}
}
else
return null;
}
/// <summary>
/// Matches symbol against this token and returns true if this token is that symbol.
/// </summary>
/// <param name="c">Symbol to match.</param>
public bool Match(char c)
{
if (_t == TokenType.Symbol)
if (CharValue == c)
return true;
return false;
}
/// <summary>
/// Matches type against this token and returns true if this token is that type.
/// </summary>
/// <param name="t">Type to match against.</param>
public bool Match(TokenType t)
{
if (_t == t)
return true;
return false;
}
/// <summary>
/// Returns a string representing this token and all its children within the parse tree.
/// </summary>
public override string ToString()
{
StringBuilder sb = new StringBuilder();
switch (_t)
{
case TokenType.Number:
sb.Append(_v);
break;
case TokenType.Symbol:
if (CharValue == '(')
{
sb.Append('(');
sb.Append(_l.ToString());
sb.Append(')');
}
else
{
if (_l != null)
sb.Append(_l.ToString());
sb.Append(_s[_v]);
if (_r != null)
sb.Append(_r.ToString());
}
break;
case TokenType.End:
case TokenType.Invalid:
default:
return String.Empty;
}
return string.Empty;
}
#endregion
#region private methods
/// <summary>
/// Intended to improve efficiency of generating lists of values.
/// reference to list is passed in so that lists do not need to be
/// created and garbage collected for each element in the list.
/// </summary>
/// <param name="limit">Limit placed on number of values returned for performance reasons.</param>
private void IntValues(int limit, List<int> lst)
{
if (_t == TokenType.Number)
lst.Add(_v);
else if (_t == TokenType.Symbol)
{
switch (CharValue)
{
case '(':
case 'd':
case 'x':
case '+':
case '-':
if (limit > 0)
lst.Add(IntValue);
break;
case ',':
if (limit > 0)
if (_l.Match(',') || _l.Match('*'))
_l.IntValues(limit, lst);
else
lst.Add(_l.IntValue);
if (limit - lst.Count > 0)
if (_r.Match(',') || _r.Match('*'))
_r.IntValues(limit - lst.Count, lst);
else
lst.Add(_l.IntValue);
break;
case '*':
int count = _l.IntValue;
if (count > limit)
count = limit;
for (int i = 0; i < count; i++)
lst.Add(_r.IntValue);
break;
default:
break;
}
}
}
/// <summary>
/// Intended to improve efficiency of generating lists of values.
/// reference to list is passed in so that lists do not need to be
/// created and garbage collected for each element in the list.
/// </summary>
private void IntValues(List<int> lst)
{
if (_t == TokenType.Number)
lst.Add(_v);
else if (_t == TokenType.Symbol)
{
switch (CharValue)
{
case '(':
case 'd':
case 'x':
case '+':
case '-':
lst.Add(IntValue);
return;
case ',':
if (_l.Match(',') || _l.Match('*'))
_l.IntValues(lst);
else
lst.Add(_l.IntValue);
if (_r.Match(',') || _r.Match('*'))
_r.IntValues(lst);
else
lst.Add(_l.IntValue);
return;
case '*':
int count = _l.IntValue;
for (int i = 0; i < count; i++)
lst.Add(_r.IntValue);
return;
default:
return;
}
}
else
return;
}
#endregion
}
/// <summary>
/// This tokenizer has made with few simple assumptions.
/// All whitespace is ignored.
/// Symbols may be no longer than 1 character.
/// Numeric values are integer only.
/// </summary>
public class Tokenizer
{
#region private members
//expression to tokenize
private string _xpr;
//lookahead "pointer"
private int _lh;
//next token to return
private Token _next;
//random number generator, added to allow
//seeding of rolls for deterministic behavior
//passed to tokens
private Random _roll;
#endregion
#region public properties
/// <summary>
/// Position of lookahead pointer within expression.
/// </summary>
public int LA
{
get { return _lh; }
}
/// <summary>
/// Expression to be split into tokens.
/// </summary>
public string Expression
{
get { return _xpr; }
set { _xpr = value; _lh = -1; LookAhead(); }
}
#endregion
#region constructors
/// <summary>
/// Creates a Tokenizer for the input expression.
/// </summary>
public Tokenizer(string input, Random rnd)
{
_roll = rnd;
Expression = input;
}
#endregion
#region public methods
/// <summary>
/// Loads next token and returns true if it is valid.
/// Returns false if token is invalid or the end of expression has been reached.
/// </summary>
/// <param name="next">Token to be loaded.</param>
/// <returns></returns>
public bool LoadNextToken(ref Token next)
{
if (_next.T == TokenType.End || _next.T == TokenType.Invalid)
{
next = _next;
return false;
}
else
{
next = _next;
return true;
}
}
/// <summary>
/// Gets the next token from the expression and returns it.
/// </summary>
public Token GetNextToken()
{
return _next;
}
/// <summary>
/// Increments lookahead pointer within the exression and prepares the next token to be retrieved.
/// </summary>
public void LookAhead()
{
_lh++; //it is the next token
if (_xpr == null)
{
_next = new Token(_lh); //a null expression starts at its end
return;
}
while (_lh < _xpr.Length && char.IsWhiteSpace(_xpr[_lh]))
_lh++; //ignore whitespace
if (_lh >= _xpr.Length)
_next = new Token(_lh); //reached end of expression
else if (char.IsNumber(_xpr[_lh]))
{ //extract numeric value from expression
int val = 0;
while (_lh < _xpr.Length && char.IsNumber(_xpr[_lh]))
{ //accumulate value
val *= 10;
val += _xpr[_lh++] - '0';
}
_next = new Token(val, --_lh); //lookahead is ahead of itself, needs decrimenting
}
else //this must be a symbol
_next = new Token(_xpr[_lh], _lh, _roll); //let token determine if it is valid or invalid
}
#endregion
}
/// <summary>
/// Types of tokens.
/// </summary>
public enum TokenType
{
Symbol,
Number,
Invalid,
End
}
On a mildly related note: The only copy of code for my web app that I have? is not the code that is running on the site. So I have no idea what is broken, just a pile of heavilly modified code that I have not looked at in a year.
Author
Topic: if self.isCoder(): post() #Programming Thread (Read 903615 times)