package tokenizer

import org.scalatest.FunSuite
import tokenizer.Tokenizer._

//@RunWith(classOf[JUnitRunner])
class TokenizerTest extends FunSuite {
  
  def getTokenMap(s: String) = {
    val tokens = getAllTokens(s)
    val counts: Counts = scala.collection.mutable.Map()
    for (token <- tokens) {
      counts(token._1) = 1 + counts.getOrElse(token._1, 0)
    }
    counts
  }
  
  test("testing Tokenizer") {
      assert(8 === getTokenMap("12 345 99l 999L 0 077 0xabCD 0xabL")(IntegerType))
      assert(6 === getTokenMap("1.23 45. .6 7.8e10 9E+5 9999e-10")(FloatingPointType))
      assert(3 === getTokenMap(" 'a' '\\t' '\u21A0'")(CharacterType))
      assert(4 === getTokenMap(""" "a b"  "ab\ncd"  "ab\\ncd"  "ab\"cd" """ )(StringType))
      assert(2 === getTokenMap(" \"\"\"a b c\"\"\"  \"\"\"c\"d\"e\"\"\" ")(StringType))
      assert(1 === getTokenMap(" \"\"\"c\nd\ne\"\"\" ")(StringType))
      assert(7 === getTokenMap("id1 Id2 +++ count_= ## fiddle_de_diddle >=")(IdType))
      assert(12 === getTokenMap(" do if while _ <% >: # @ => \u21D2 \u2190 yield")(KeywordType))
      assert(2 === getTokenMap(" `this is an id` `so is this`")(IdType))
      assert(2 === getTokenMap(" 'abc 'r7 ")(SymbolType))
      assert(2 === getTokenMap(" //comment \n and // another comment\n")(CommentType))
      assert(2 === getTokenMap(" /* This is\n a multi-line \n comment*/ /*and here is another*/")(CommentType))     
      assert(7 === getTokenMap("Here are the 9 delimiters: ()[]{};.,")(DelimiterType))
        // Period and comma should be delimiters, but are counted as ids. I don't know why.
  }
  
}