[tornado] Asynchronous botocore

AWS services and boto project are great things, but that we can't use them asynchronously in tornado is a big disadvantage.

This article inspired me to search for a way to use boto asynchronously:
http://blog.joshhaas.com/2011/06/marrying-boto-to-tornado-greenlets-bring-them-together/

I found that it is easy to rewrite code in botocore and use tornado AsyncHTTPClien.
The idea is to rewrite botocore.operation.call that it would use tornado.httpclient.AsyncHTTPClient. After some experiments I got a simple wrapper for botocore that allows to use it in tornado asynchronously.

Repository:
https://github.com/nanvel/tornado-botocore
PyPI:
https://pypi.python.org/pypi/tornado-botocore

Installation:
pip install tornado-botocore

Usage example:
from tornado.ioloop import IOLoop
from tornado_botocore import Botocore


def on_response(response):
    http_response, response_data = response
    print response_data


if __name__ == '__main__':
    ec2 = Botocore(
        service='ec2', operation='DescribeInstances',
        region_name='us-east-1')
    ec2.call(callback=on_response)
    IOLoop.instance().start()

Get country code/name by IP address

Using tornado and maxmind geoip2 database:
pip install tornado
pip install geoip2

Use self.country to get client country code:
import os.path

from geoip2.database import Reader as GeoIP2Reader
from tornado import options
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, Application


class BaseHandler(RequestHandler):

    # download it from maxmind site: http://dev.maxmind.com/geoip/geoip2/geolite2/
    GEOIP_BINARY = 'bin/GeoLite2-Country.mmdb'

    @property
    def country(self):
        try:
            ip = self.request.remote_ip
            geoip = getattr(self.application, '_geoip', None)
            if not geoip:
                geoip_file = os.path.join(
                    os.path.dirname(os.path.realpath(__file__)),
                    self.GEOIP_BINARY)
                geoip = GeoIP2Reader(geoip_file)
                self.application._geoip = geoip
            return geoip.country(ip).country.iso_code
        except Exception as e:
            return None


class CountryHandler(BaseHandler):

    def get(self):
        self.write(str(self.country))


if __name__ == '__main__':
    options.parse_command_line()
    application = Application(
        handlers=[
            (r'/', CountryHandler),
        ],
        debug=True,
    )
    application.listen(8000)
    IOLoop.instance().start()

[tornado] Async handlers, code refactor

Let's start with sync implementation:

import json

from tornado import options
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, Application
from tornado.httpclient import HTTPClient, HTTPError


class FBHandler(RequestHandler):

    FB_GRAPH_ME_URL = 'https://graph.facebook.com/me?fields=id&access_token={fb_token}'

    def get(self):
        fb_id = self.get_argument('fb_id')
        fb_token = self.get_argument('fb_token')
        url = self.FB_GRAPH_ME_URL.format(fb_token=fb_token)
        http_client = HTTPClient()
        try:
            response = http_client.fetch(request=url, method='GET')
            if json.loads(response.body).get('id') == fb_id:
                self.write('Ok')
                return
        except HTTPError:
            pass
        self.write('Fail')


if __name__ == '__main__':
    options.parse_command_line()
    application = Application(
        handlers=[
            (r'/', FBHandler),
        ],
        debug=True,
    )
    application.listen(8000)
    IOLoop.instance().start()

Make it async:

import json

from tornado import options
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, Application, asynchronous
from tornado.httpclient import AsyncHTTPClient, HTTPError


class FBHandler(RequestHandler):

    FB_GRAPH_ME_URL = 'https://graph.facebook.com/me?fields=id&access_token={fb_token}'

    @asynchronous
    def get(self):
        self.fb_id = self.get_argument('fb_id')
        fb_token = self.get_argument('fb_token')
        url = self.FB_GRAPH_ME_URL.format(fb_token=fb_token)
        http_client = AsyncHTTPClient()
        try:
            response = http_client.fetch(
                request=url, method='GET', callback=self.on_fetch)
        except HTTPError:
            self.write('Fail')

    def on_fetch(self, response):
        if json.loads(response.body).get('id') == self.fb_id:
            self.write('Ok')
        else:
            self.write('Fail')
        self.finish()


...

Add gen sugar:

import json

from tornado import options,
gen
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, Application
from tornado.httpclient import AsyncHTTPClient, HTTPError


class FBHandler(RequestHandler):

    FB_GRAPH_ME_URL = 'https://graph.facebook.com/me?fields=id&access_token={fb_token}'

    @gen.coroutine
    def get(self):
        fb_id = self.get_argument('fb_id')
        fb_token = self.get_argument('fb_token')
        url = self.FB_GRAPH_ME_URL.format(fb_token=fb_token)
        http_client = AsyncHTTPClient()
        try:
            response = yield http_client.fetch(request=url, method='GET')
            if json.loads(response.body).get('id') == fb_id:
                self.write('Ok')
                return
            self.write('Fail')
        except HTTPError:
            pass
        self.write('Fail')


...

Hold handlers as simple as possible:

import json

from tornado import options, gen
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, Application
from tornado.httpclient import AsyncHTTPClient, HTTPError


class FBHandler(RequestHandler):

    @gen.coroutine
    def get(self):
        try:
            fb_id = self.get_argument('fb_id')
            fb_token = self.get_argument('fb_token')
            yield self.validate(fb_id=fb_id, fb_token=fb_token)
            self.write('Ok')
        except Exception:
            self.write('Fail')

    @gen.coroutine
    def validate(self, fb_id, fb_token):
        FB_GRAPH_ME_URL = 'https://graph.facebook.com/me?fields=id&access_token={fb_token}'

        url = FB_GRAPH_ME_URL.format(fb_token=fb_token)
        http_client = AsyncHTTPClient()
        response = yield http_client.fetch(request=url, method='GET')
        assert json.loads(response.body).get('id') == fb_id
        raise gen.Return(True)


...

[Django] Save model instance into json dict

I need to save a dict with numbers, text and django model instances. And I don't know which model instances may be present in the dict.
data = {
    'count': 10,
    'title': 'Example title',
    'user': request.user,
}

or

data = {
    'content': 'Example content',
    'site': Site.objects.get_current(),
}

If I try to dump dicts above, I'll get TypeError Exception:
>>> import json
>>> json.dumps(data)
TypeError: <User: exampleuser> is not JSON serializable

Possible solution:
from django.contrib.contenttypes.models import ContentType
from django.db.models import Model


def encode(data):
    new_data = dict(data)
    for node, value in data.iteritems():
        if isinstance(value, Model):
            node_type = ContentType.objects.get_for_model(value.__class__)
            new_data[node] = {
                'app_label': node_type.app_label,
                'model': node_type.model,
                'id': value.id}
    return new_data

def decode(data):
    new_data = dict(data)
    for node, value in data.iteritems():
        if not isinstance(value, dict):
            continue
        if 'app_label' in value and 'model' in value and 'id' in value:
            user_type = ContentType.objects.get(
                app_label=value['app_label'],
                model=value['model'])
            new_data[node] = user_type.get_object_for_this_type(id=value['id'])
    return new_data

>>> data
{'count': 10, 'user': <User: exampleuser>, 'title': 'Example title'}
>>> json.dumps(data)
TypeError: <User: exampleuser> is not JSON serializable
>>> encoded = json.dumps(encode(data))
>>> encoded
'{"count": 10, "user": {"model": "user", "id": 1, "app_label": "auth"}, "title": "Example title"}'
>>> decode(json.loads(encoded))
{u'count': 10, u'user': <User: exampleuser>, u'title': u'Example title'}