Below is the file 'tokens_test.cc' from this revision. You can also download the file.

#include <iostream>
#include <fstream>
#include "markov.hh"
#include "tokens.hh"

#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>

using namespace std;

typedef Tokens<string> s_tok;

void
string_test (s_tok &t)
{
    string s;

    t.add ("goat");
    t.add ("cheese");
    t.add ("chhhhhhhhhheeeese");
}

void
input_test (s_tok &t)
{
    string i;
    int cnt = 0;

    while (cin >> i) {
        cnt++;
        if (!(cnt % 10000)) {
            cout << cnt << ".." << i << endl;
        }
        t.add (i);
    }
    cout << "token reading done" << endl;
}

void
test (void)
{
    s_tok t;

    string_test (t);
    input_test (t);
//    t.playback ();
    MarkovModel m(t, 2);
    m.Dump();

    {
        int a = 2;
        std::ofstream ofs ("test.ser");
        boost::archive::text_oarchive oa(ofs);
        oa << a;
    }
}

int
main (int argc, char *argv[])
{
    test ();
    return 0;
}