Below is the file 'tokens.hh' from this revision. You can also download the file.

#ifndef _TOKENS_HH
#define _TOKENS_HH

#include <ext/hash_map>
#include <vector>
#include <iostream>

#define __EXT__ __gnu_cxx

typedef unsigned long int token_t;

namespace __EXT__ {
    template <>
    struct hash<std::string>
    {
      size_t
      operator()(std::string __s) const
      { return __EXT__::hash<char *>()(__s.c_str ()); }
    };
}

class Stash {
protected:
    std::vector<token_t> stash;

public:
    const std::vector<token_t>& get_stash() const {
        return stash;
    }
};

template<class T, class H = __EXT__::hash<T> >
class Tokens : public Stash {
private:
    token_t last_id;
    __EXT__::hash_map<T, token_t, H> token_to_id;
    __EXT__::hash_map<token_t, T, __EXT__::hash<token_t> > id_to_token;

public:
    Tokens(void) {
        last_id = 0;
    }

    token_t lookup(const T &query) {
        return token_to_id[query];
    }

    token_t add(const T &add) {
        token_t tid;

        tid = token_to_id[add];
        if (tid == 0) {
            tid = token_to_id[add] = ++last_id;
            id_to_token[tid] = add;
        }
        stash.push_back (tid);
        return tid;
    }

    void playback (void) {
        for (std::vector<token_t>::iterator i=stash.begin(); i!=stash.end(); ++i) {
            std::cout << *i << " : " << id_to_token[*i] << std::endl;
        }
    }
};

#endif