Tokenize strings in C++

You might have wondered at least once how to simply tokenize a string in C++. Here is a nice and easy way to do it using string streams, inserting tokens into a vector:

#include <string>
#include <vector>
#include <sstream>
#include <iterator>
#include <algorithm>

using namespace std;

void tokenize_string( string words, vector<string>& tokens )
{
   tokens.clear();
   istringstream iss(words);
   copy( 
      istream_iterator<string>(iss), 
      istream_iterator<string>(), 
      back_inserter<vector<string> >(tokens) 
   );
}

Now, say you want to put the tokens into a set instead.. how would you do it? Here it is with a few changes:

#include <set>
#include <string>
#include <vector>
#include <sstream>
#include <iterator>
#include <algorithm>

using namespace std;

struct set_inserter : public iterator< input_iterator_tag, string >
{
   string word;
   set<string>* container;

   set_inserter( set<string>& c ): container(&c) {}
   inline reference operator* () { return word; }
   set_inserter& operator++ () { container->insert(word); return *this; }
};



void tokenize_string( string words, set<string>& tokens )
{
   tokens.clear();
   istringstream iss(words);
   copy( 
      istream_iterator<string>(iss), 
      istream_iterator<string>(), 
      set_inserter(tokens) 
   );
}

What did I do? I replaced the back inserter with my own structure, which behaves virtually as an interator, even though it's not. You can tweak this structure as you want to perform the treatment that you want, provided that you overload the indirection and the left-increment operators as I did :) That's all!