Sign in to follow this  
Shael

Task Scheduler

Recommended Posts

Heya,

I'm working on a generic task scheduler that I can use in my engine to do all sorts of asynchronous jobs such as resource loading, and continuous jobs such as running update loops. I've got a basic working system so far but would like to get some constructive criticism on the design and help on a couple of things.

Current problems I'm aware of and would like to fix but need help on:

- Duplicate code in the Task classes. Would like to have Task as base class and TaskNotify as derived but kept getting template errors I couldn't decipher.
- I'd like to somehow make the callbacks more robust and allow them to accept the return value of the task. I couldn't quite figure out how to do this.



The scheduler and task classes below make heavy use of boost threads/futures/packages and threadpool.

TaskScheduler
[source lang=cpp]

#ifndef _TASK_SCHEDULER_H_
#define _TASK_SCHEDULER_H_

#include "boost/threadpool.hpp"
#include "boost/thread.hpp"
#include "boost/utility/result_of.hpp"
#include "boost/shared_ptr.hpp"

#include "Task.h"

class TaskScheduler : private boost::noncopyable
{
public:

TaskScheduler ( uint32 n = boost::thread::hardware_concurrency() ) : m_ThreadPool(n)
{

}

template <typename T>
typename T::Future Submit(T& t)
{
boost::threadpool::schedule(m_ThreadPool, boost::bind(&T::operator(), t));
return t.future();
}

template<typename F>
typename Task<typename boost::result_of<F()>::type>::Future Submit (const F& f)
{
Task<typename boost::result_of<F()>::type> t(f);
return Submit(t);
}

template<typename F, typename C>
typename TaskNotify<typename boost::result_of<F()>::type, typename boost::result_of<C()>::type>::Future Submit (const F& f, const C& c)
{
TaskNotify<typename boost::result_of<F()>::type, typename boost::result_of<C()>::type> t(f,c);
return Submit(t);
}

private:

boost::threadpool::pool m_ThreadPool;
};

#endif[/source]

Tasks
[source lang=cpp]


#ifndef _TASK_H_
#define _TASK_H_

template <typename R>
class Task
{
public:

typedef typename boost::unique_future<R> Future;

protected:

typedef boost::packaged_task<R> PackagedTask;
typedef boost::shared_ptr<PackagedTask> PackagedTaskPtr;

public:

template <typename F>
explicit Task(const F& f) : m_Task(new PackagedTask(f))
{
}

void operator()()
{
(*m_Task)();
}

Future future()
{
return m_Task->get_future();
}

protected:

PackagedTaskPtr m_Task;
};


template <typename R, typename Callback = void>
class TaskNotify
{
public:

typedef typename boost::unique_future<R> Future;

protected:

typedef boost::packaged_task<R> PackagedTask;
typedef boost::shared_ptr<PackagedTask> PackagedTaskPtr;

typedef boost::packaged_task<Callback> PackagedCallback;
typedef boost::shared_ptr<PackagedCallback> PackagedCallbackPtr;

public:

template <typename F, typename C>
explicit TaskNotify(const F& f, const C& c) : m_Task(new PackagedTask(f)), m_Callback(new PackagedCallback©)
{
}

void operator()()
{
(*m_Task)();
(*m_Callback)();
}

Future future()
{
return m_Task->get_future();
}

protected:

PackagedCallbackPtr m_Callback;
PackagedTaskPtr m_Task;
};

#endif
[/source]

Main/Tests
[source lang=cpp]

#include "stdafx.h"
#include <iostream>
#include <conio.h>
typedef unsigned int uint32;
#include "TaskScheduler.h"

static boost::mutex mutex;
void func(int i)
{
boost::mutex::scoped_lock lock(mutex);
std::cout << "i = " << i << std::endl;
}

int fib(int x)
{
if (x == 0)
return 0;

if (x == 1)
return 1;

return fib(x-1)+fib(x-2);
}

void fib1Done()
{
std::cout << "Fib 1 Done" << std::endl;
}

void fib2Done()
{
std::cout << "Fib 2 Done" << std::endl;
}

int _tmain(int argc, _TCHAR* argv[])
{
TaskScheduler s;

// Simple test of ordered tasks
Task<void> task(boost::bind(&func, 1));
Task<void>::Future a = s.Submit(task);
Task<void>::Future b = s.Submit(boost::bind(&func, 2));
Task<void>::Future c = s.Submit(boost::bind(&func, 3));

a.wait();
b.wait();
c.wait();

// Callback test
TaskNotify<int> fibTask(boost::bind(&fib, 40), boost::bind(&fib1Done));
TaskNotify<int>::Future fibby1 = s.Submit(fibTask);

TaskNotify<int>::Future fibby2 = s.Submit(boost::bind(&fib, 30), boost::bind(&fib2Done));

_getch();
return 0;
}
[/source]

Share this post


Link to post
Share on other sites
Here is my approach:

[list]
[*] boost::asio::io_service for the thread pool instead of the (unofficial) boost::threadpool
[*] boost::function for the callback type, just do callback(future.get()) and your issue #2 is solved.
[*] Single templated T::future_type submit(T&) for both task and task_notify (or any class defining future_type and operator() really). Your compiler errors probably due to not passing template argument as boost::ref(t). (boost::unique_future<> not copy-constructable).
[/list]

[i]task [/i]and [i]task_notify[/i]

[source lang=cpp]
template <typename R>
class task {
public:
typedef boost::packaged_task<R> task_type;
typedef boost::shared_ptr<task_type> task_ptr;
typedef boost::unique_future<R> future_type;

template <typename F>
task(F f)
: ptask(boost::make_shared<task_type>(f)),
future(ptask->get_future())
{}

void operator()()
{ (*ptask)(); }

future_type& get_future()
{ return future; }

R get()
{ return future.get(); }

bool ready() const
{ return future.is_ready(); }

private:
task_ptr ptask;
future_type future;
};

template <typename R>
class task_notify : public task<R> {
public:
typedef boost::function<void (R)> callback_type;

template <typename F>
task_notify(F f, callback_type cb)
: task(f), callback(cb)
{}

void operator()()
{
task<R>::operator()();
callback(get());
}

private:
callback_type callback;
};
[/source]

[i]task_scheduler[/i]

[source lang=cpp]
class task_scheduler {
public:
task_scheduler()
: work(io_service)
{
std::size_t ncores = boost::thread::hardware_concurrency();
for (std::size_t i = 0; i < ncores; ++i) {
threads.create_thread(boost::bind(&task_scheduler::run, this));
}
}

~task_scheduler()
{
io_service.stop();
threads.join_all();
}

template <typename T>
typename T::future_type& submit(T& t)
{
io_service.post(boost::bind(&T::operator(), boost::ref(t)));
return t.get_future();
}

private:
void run() {
io_service.run();
}

boost::asio::io_service io_service;
boost::asio::io_service::work work;
boost::thread_group threads;
};
[/source]

[i]Main/Tests[/i]

[source lang=cpp]

#if _WIN32
#pragma comment (linker, "/ENTRY:mainCRTStartup")
#pragma comment (linker, "/SUBSYSTEM:CONSOLE")
#endif

#include <boost/asio.hpp>
#include <boost/thread.hpp>
#include <boost/make_shared.hpp>

#include <iostream>

// include: task
// include: task_notify
// include: task_scheduler

int fib(int x)
{
if(x == 0) return 0;
if(x == 1) return 1;
return fib(x-1)+fib(x-2);
}

void fib30_done(int result) {
std::cout << "fib30_done, fib(30) = " << result << std::endl;
}

int main(int argc, char *argv[])
{
task_scheduler ts;

task<int> t1(boost::bind(&fib, 10));
task<int> t2(boost::bind(&fib, 20));
task_notify<int> t3(boost::bind(&fib, 30), boost::bind(&fib30_done, _1));

ts.submit(t1);
ts.submit(t2);
ts.submit(t3);

std::cout << "fib(10) = " << t1.get() << std::endl;
std::cout << "fib(20) = " << t2.get() << std::endl;
std::cout << "fib(30) = " << t3.get() << std::endl;

getchar();
return 0;
}
[/source]

Share this post


Link to post
Share on other sites
Oh nice! That's pretty elegant and the kind of where I wanted to head :)

I had thought about boost::asio but wasn't sure if it was overkill or the right tool for the job but it seems to fit in nicely.

Thanks!

Share this post


Link to post
Share on other sites
You're welcome.

Although my code should be considered "proof-of-concept" as I omitted proper copy semantics and what not. Also my task<> and task_notify<> objects are not allowed to go out of scope. I suspect that with your shared_ptr<packaged_task> that the idea was so task objects could go out of scope and still complete asynchronously through the packaged_task?

You inspired me to try a similar design in my engine. I wonder if its best to pass around the task scheduler to all objects that wish to create tasks, or just have the task scheduler static in the task objects. The latter approach sure is tempting, just put a task on the stack and forget about it (unless you need the return value, of course). Another idea is to just have a global 'schedule_task()' function that takes a function object and returns a future. No need to worry about the task scheduler or task objects. On the other hand, I want to avoid globals, but a task scheduler sure seems like a good candidate as you don't want to have more than one task scheduler (compare to Intel TBB).
Thoughts?

Share this post


Link to post
Share on other sites
I haven't actually tested the case of the Task objects going out of scope yet, but my plan was to allow them to just be put on the stack and forgot about and if you needed the return value then you would use a task notifier. The idea of the task notify object was mostly to fit in with resource loading. I wanted to be able to ask the resource manager for a resource and have it return a dummy resource straight away, and then have it push a file load task into the scheduler and when the task was done it would use the callback to post the file data back to a resource loader which would set up the dummy resource with real data.

So I think the task objects definitely need to be able to go out of scope but still have something sharing them until the task is finished and then it can be destroyed.

I was recently thinking about how the scheduler should be managed and was thinking that there is probably only a few major places within the engine that would need to be able to create tasks and so I thought it'd be fine just passing a shared_ptr around. So far I can see only the engine core and resource manager requiring it. I did also think about whether it should be a Singleton and just have global access to it but I've been trying to sway away from Singletons as it seems to be shunned upon these days.

Share this post


Link to post
Share on other sites
[quote name='void0' timestamp='1318120264' post='4870645']
Here is my approach:

[list][*] boost::asio::io_service for the thread pool instead of the (unofficial) boost::threadpool[*] boost::function for the callback type, just do callback(future.get()) and your issue #2 is solved.[*] Single templated T::future_type submit(T&) for both task and task_notify (or any class defining future_type and operator() really). Your compiler errors probably due to not passing template argument as boost::ref(t). (boost::unique_future<> not copy-constructable).[/list][/quote]

Hi folks,

I agree with using ASIO in comparison to the original system. Given that ASIO is written against a generally very well scaling system on a per OS level (IOCompletions for Win32, KQueue bsd/osx, EPoll linux etc) it is a very good and well debugged starting point. I would suggest avoiding the futures system for "small" tasks and prefer the ASIO strands in this case, the performance seemed measurably higher with pretty much the same usable behavior.

The down side to ASIO for threading is that lots of anything "small" will suck in terms of OS overhead. Unfortunately all the variations of ASIO are, for all intents and purposes, central queues which means adding/removing tasks from a central location which means you hit Ahmdahl's law in a very big hurry if you run too much through it too fast. As far as the suggested usage goes, it should be completely fine, just don't push too much further as tempting as it may be. I use ASIO in systems which have "NOTHING" to do with IO just because it is a great way to create an event driven framework for any number of things. Unfortunately though, it is very tempting to push game related items through it and that will be a mistake. Been there, done that, kernel times go apeshit and performance drops off very quickly. Just don't go there. :)

Share this post


Link to post
Share on other sites
[quote name='AllEightUp' timestamp='1318213354' post='4870931']
/../ Unfortunately though, it is very tempting to push game related items through it and that will be a mistake. Been there, done that, kernel times go apeshit and performance drops off very quickly. Just don't go there. :)
[/quote]

Agreed. Use it for long running or computationally heavy tasks (like resource loading). Even not considering performance, using it for game related items is a mistake as those tasks would block waiting for the long running tasks to finish (only 2 tasks at once on a dual-core, for example). For game related items I'd probably use coroutines or similar to create an event driven system that runs from the main thread.

Share this post


Link to post
Share on other sites
I added some code which allows you to create a task and forget about it by using shared pointers. It's technically not a "create on stack and forget about" but I don't really see any other way of doing it.

In the task scheduler I added this function:
[source lang=cpp]

template <typename T>
typename T::future_type& submit(boost::shared_ptr<T>& t)
{
io_service.post(boost::bind(&T::operator(), t));
return t->get_future();
}
[/source]

The test is then simply:
[source lang=cpp]


void outofscopetest(task_scheduler& ts)
{
boost::shared_ptr<task_notify<int> > fibTask(new task_notify<int>(boost::bind(&fib, 35), boost::bind(&fibDone, _1)));
ts.submit(fibTask);
}
[/source]

In terms of design I'm not sure whether the other [i]submit[/i] function should be removed and all tasks should be added as shared pointers or both should be kept to allow for stack and heap allocated tasks. It might be best to force shared pointers only to save confusion and user error.

Thoughts?

Share this post


Link to post
Share on other sites
phantom is right, there should be no need for shared_ptr for the task itself. It is however required for packaged_task to be a shared_ptr because io_service completion handlers must be copyable.

I made the following changes to my code:

[list=1]
[*] task objects now allowed to go out of scope
[*] task_sheduler::submit receives copy of functor and posts copy to io_service::post
[*] Removed task_notify class. Reason for this is I got compilation errors ([color="#FF0000"]*[/color]) and couldn't quite figure out why. I think the end result is better anyway.
[*] Callback is now second constructor to task<>. boost::function::empty used to decide if we should callback.
[/list]

[color="#FF0000"]*[/color] error C2664: 'void boost::detail::future_object<T>::mark_finished_with_result(const int &)' : cannot convert parameter 1 from 'void' to 'const int &' c:\boost\include\boost-1_45\boost\thread\future.hpp 1224



DISCLAIMER: Beware of bugs in the code below. Still proof of concept :)

Full source:

[source lang=cpp]
#if _WIN32
#pragma comment (linker, "/ENTRY:mainCRTStartup")
#pragma comment (linker, "/SUBSYSTEM:CONSOLE")
#endif

#include <boost/asio.hpp>
#include <boost/thread.hpp>
#include <boost/make_shared.hpp>

#include <iostream>

template <typename R>
class task {
public:
typedef boost::packaged_task<R> task_type;
typedef boost::shared_ptr<task_type> task_ptr;
typedef boost::shared_future<R> future_type;
typedef boost::function<void (R)> callback_type;

template <typename F>
task(F f)
: ptask(boost::make_shared<task_type>(f)),
future(ptask->get_future())
{}

template <typename F>
task(F f, callback_type cb)
: ptask(boost::make_shared<task_type>(f)),
future(ptask->get_future()),
callback(cb)
{}

void operator()()
{
(*ptask)();
if(!callback.empty()) callback(get());
}

future_type get_future()
{ return future; }

R get()
{ return future.get(); }

bool ready() const
{ return future.is_ready(); }

private:
task_ptr ptask;
future_type future;
callback_type callback;
};

class task_scheduler {
public:
task_scheduler()
: work(io_service)
{
std::size_t ncores = boost::thread::hardware_concurrency();
for (std::size_t i = 0; i < ncores; ++i) {
threads.create_thread(boost::bind(&boost::asio::io_service::run,
boost::ref(io_service)));
}
}

~task_scheduler()
{
io_service.stop();
threads.join_all();
}

template <typename T>
typename T::future_type submit(T t)
{
io_service.post(t);
return t.get_future();
}

private:
boost::asio::io_service io_service;
boost::asio::io_service::work work;
boost::thread_group threads;
};

int fib(int x)
{
if(x == 0) return 0;
if(x == 1) return 1;
return fib(x-1)+fib(x-2);
}

void async_fib_done(int x, int result) {
std::cout << "fib("<<x<<") = " << result << std::endl;
}

task<int>::future_type async_fib(task_scheduler& ts, int x) {
task<int> t(boost::bind(&fib, x));
return ts.submit(t);
}
task<int>::future_type async_fib_notify(task_scheduler& ts, int x) {
task<int> t(boost::bind(&fib, x), boost::bind(&async_fib_done, x, _1));
return ts.submit(t);
}

int main(int argc, char *argv[])
{
task_scheduler ts;

task<int>::future_type f1 = async_fib(ts, 10);
task<int>::future_type f2 = async_fib_notify(ts, 30);

std::cout << "fib(10) = " << f1.get() << std::endl;
// .. fib(30) printed from callback ..

getchar();
return 0;
}
[/source]

Share this post


Link to post
Share on other sites
[quote name='nfries88' timestamp='1318357271' post='4871526']
I can't see why you're holding on to a copy of io_service::work in your task_scheduler class.
[/quote]

So io_service::run won't exit when it runs out of tasks to process. I've never used Boost.Asio before so maybe it is not needed.

[b]Edit[/b]: In fact, if io_service::work goes out of scope, then io_service::work_finished() is called which will terminate the worker thread when it runs out of tasks.

See impl/io_service.hpp:

[source]
inline io_service::work::work(boost::asio::io_service& io_service)
: io_service_(io_service)
{
io_service_.impl_.work_started();
}

inline io_service::work::work(const work& other)
: io_service_(other.io_service_)
{
io_service_.impl_.work_started();
}

inline io_service::work::~work()
{
io_service_.impl_.work_finished();
}
[/source]

Or am I wrong?

Share this post


Link to post
Share on other sites
I've used boost.asio three times and never had to touch the work class. The documentation is a little misleading in its wording; what the work class does is prevents the run function from returning when you call stop IIRC.

But then, I've only used asio for the asio part; not the task part (well, I did wind up using the task part, but you know what I mean). So maybe it's different when you're not listening for a connection.

Share this post


Link to post
Share on other sites
[quote name='nfries88' timestamp='1318360947' post='4871557']
I've used boost.asio three times and never had to touch the work class. The documentation is a little misleading in its wording; what the work class does is prevents the run function from returning when you call stop IIRC.

But then, I've only used asio for the asio part; not the task part (well, I did wind up using the task part, but you know what I mean). So maybe it's different when you're not listening for a connection.
[/quote]

I think you are wrong. Maybe your io_service actually had always "work" to do (periodic timer?). Here is the documentation for io_service::work that I found in the io_service.hpp header file:

[quote] * @par Stopping the io_service from running out of work
*
* Some applications may need to prevent an io_service object's run() call from
* returning when there is no more work to do. For example, the io_service may
* be being run in a background thread that is launched prior to the
* application's asynchronous operations. The run() call may be kept running by
* creating an object of type boost::asio::io_service::work:
*
* @code boost::asio::io_service io_service;
* boost::asio::io_service::work work(io_service);
* ... @endcode
*
* To effect a shutdown, the application will then need to call the io_service
* object's stop() member function. This will cause the io_service run() call
* to return as soon as possible, abandoning unfinished operations and without
* permitting ready handlers to be dispatched.[/quote]

[b]Edit:[/b] I read your answer too quickly. Yes, listening on a connection is different. This qualifies as "work" and will prevent io_service::run from exiting. In my case with a thread pool and tasks posted through io_service::post(), I must keep the io_service busy when there are no more tasks to run.

Share this post


Link to post
Share on other sites
On the topic of io_service and work, should we be calling io_service::stop? Does it kill the threads abruptly or does it let them finish properly? The doco mentions this:

[quote]

To effect a shutdown, the application will then need to call the io_service object's stop() member function. This will cause the io_service run() call to return as soon as possible, abandoning unfinished operations and without permitting ready handlers to be dispatched.

Alternatively, if the application requires that all operations and handlers be allowed to finish normally, the work object may be explicitly destroyed.
[font="sans-serif"][size="2"][source lang=cpp][/size][/font]
[font="sans-serif"][size="2"]
boost::asio::io_service io_service;
auto_ptr<boost::asio::io_service::work> work(
new boost::asio::io_service::work(io_service));
...
work.reset(); // Allow run() to exit.
[/source][/size][/font]
[/quote]

As for your recent changes void0, that is definitely much nicer than using the shared pointer. I did however notice a problem with combining the Task and TaskNotify classes and that is you can't have a task that returns void. This would be a problem with TaskNotify too though if you tried using void as return type.

The compiler will complain at the point of calling the callback with:

[quote]
error C2064: term does not evaluate to a function taking 1 arguments
[/quote]

I'm not really a boost guru so I'm not sure how or if we can get around this. Maybe there's some other magic to use with boost::function and templates heh.

Share this post


Link to post
Share on other sites
[quote name='void0' timestamp='1318361433' post='4871561']
[quote name='nfries88' timestamp='1318360947' post='4871557']
I've used boost.asio three times and never had to touch the work class. The documentation is a little misleading in its wording; what the work class does is prevents the run function from returning when you call stop IIRC.

But then, I've only used asio for the asio part; not the task part (well, I did wind up using the task part, but you know what I mean). So maybe it's different when you're not listening for a connection.
[/quote]

I think you are wrong. Maybe your io_service actually had always "work" to do (periodic timer?). Here is the documentation for io_service::work that I found in the io_service.hpp header file:

[quote] * @par Stopping the io_service from running out of work
*
* Some applications may need to prevent an io_service object's run() call from
* returning when there is no more work to do. For example, the io_service may
* be being run in a background thread that is launched prior to the
* application's asynchronous operations. The run() call may be kept running by
* creating an object of type boost::asio::io_service::work:
*
* @code boost::asio::io_service io_service;
* boost::asio::io_service::work work(io_service);
* ... @endcode
*
* To effect a shutdown, the application will then need to call the io_service
* object's stop() member function. This will cause the io_service run() call
* to return as soon as possible, abandoning unfinished operations and without
* permitting ready handlers to be dispatched.[/quote]

[b]Edit:[/b] I read your answer too quickly. Yes, listening on a connection is different. This qualifies as "work" and will prevent io_service::run from exiting. In my case with a thread pool and tasks posted through io_service::post(), I must keep the io_service busy when there are no more tasks to run.
[/quote]

I never run into this problem because I always keep a "heartbeat" timer going. It pretty much doesn't matter what I'm doing, I want some form of "yes, I'm still working" sort of status printf'd, logged, sent to sockets, whatever. Given boost asio, the best "yes, I'm still alive" method is for the "main" function to maintain a deadline timer at all times. If you start more threads to work on the queue, each should maintain it's own heartbeat timer to make sure it hasn't hung up for some reason.

Without this, it is easy to end up with a hung thread doing who knows what using a full core because it is trashed but leaving the other threads working and everything is happy. As more threads crash you "may" notice the problem fairly late and by which time your server is severely hosed.

Share this post


Link to post
Share on other sites
[quote name='Shael' timestamp='1318376549' post='4871653']
As for your recent changes void0, that is definitely much nicer than using the shared pointer. I did however notice a problem with combining the Task and TaskNotify classes and that is you can't have a task that returns void. This would be a problem with TaskNotify too though if you tried using void as return type.

The compiler will complain at the point of calling the callback with:

[quote]
error C2064: term does not evaluate to a function taking 1 arguments
[/quote]

I'm not really a boost guru so I'm not sure how or if we can get around this. Maybe there's some other magic to use with boost::function and templates heh.
[/quote]

You can get around this with full template specialization. If you can stomach the duplicated code, that is. I couldn't quite figure out how to put the common code into a base class as I get compile errors trying to call a templated base class constructor through a templated sub class constructor (Is it even possible?). You see, I'm no template/boost guru either, but it gets the job done:

[source lang=cpp]
template <typename R>
class task_base {
public:
typedef boost::packaged_task<R> task_type;
typedef boost::shared_ptr<task_type> task_ptr;
typedef boost::shared_future<R> future_type;
typedef boost::function<void (R)> callback_type;
};

template <typename R>
class task : public task_base<R> {
public:
template <typename F>
task(F f)
: ptask(boost::make_shared<task_type>(f)),
future(ptask->get_future())
{}

template <typename F>
task(F f, callback_type cb)
: ptask(boost::make_shared<task_type>(f)),
future(ptask->get_future()),
callback(cb)
{}

void operator()()
{
(*ptask)();
if(!callback.empty()) callback(get());
}

future_type get_future()
{ return future; }

R get()
{ return future.get(); }

bool ready() const
{ return future.is_ready(); }

private:
task_ptr ptask;
future_type future;
callback_type callback;
};

template <>
class task<void> : public task_base<void> {
public:
template <typename F>
task(F f)
: ptask(boost::make_shared<task_type>(f)),
future(ptask->get_future())
{}

template <typename F>
task(F f, callback_type cb)
: ptask(boost::make_shared<task_type>(f)),
future(ptask->get_future()),
callback(cb)
{}

void operator()()
{
(*ptask)();
if(!callback.empty()) callback();
}

// This one really does not make sense, force a compile error?
//void get() {}

future_type get_future()
{ return future; }

bool ready() const
{ return future.is_ready(); }

private:
task_ptr ptask;
future_type future;
callback_type callback;
};
[/source]

Share this post


Link to post
Share on other sites

Create an account or sign in to comment

You need to be a member in order to leave a comment

Create an account

Sign up for a new account in our community. It's easy!

Register a new account

Sign in

Already have an account? Sign in here.

Sign In Now

Sign in to follow this