Below is the file 'tokens.hh' from this revision. You can also download the file.

#ifndef _TOKENS_HH
#define _TOKENS_HH

#include <ext/hash_map>
#include <vector>
#include <iostream>

typedef unsigned long int token_t;

namespace __gnu_cxx {
    template <>
    struct hash<std::string>
    {
      size_t
      operator()(std::string __s) const
      { return __gnu_cxx::hash<char *>()(__s.c_str ()); }
    };
}

template<class T, class H = __gnu_cxx::hash<T> >
class Tokens {
    private:
        token_t last_id;
        std::vector<token_t> stash;
        __gnu_cxx::hash_map<T, token_t, H> token_to_id;
        __gnu_cxx::hash_map<token_t, T, __gnu_cxx::hash<token_t> > id_to_token;

    public:
        Tokens(void) {
            last_id = 0;
        }

        token_t lookup(const T &query) {
            return token_to_id[query];
        }

        token_t add(const T &add) {
            token_t tid;

            tid = token_to_id[add];
            if (tid == 0) {
                tid = token_to_id[add] = ++last_id;
                id_to_token[tid] = add;
            }
            stash.push_back (tid);
            return tid;
        }

        void playback (void) {
            for (std::vector<token_t>::iterator i=stash.begin(); i!=stash.end(); ++i) {
                std::cout << *i << " : " << id_to_token[*i] << std::endl;
            }
        }
};

#endif