Efficient actions counter using tornado IOLoop.add_timeout
Redis used as temporary storage and sqlite3 as persistent storage for counters.
pip install tornado==4.1 pip install redis
Create sqlite table:
(.env)nanvel-air:tornado_counter nanvel$ sqlite3 actions.sqlite3 SQLite version 3.8.5 2014-08-15 22:37:57 Enter ".help" for usage hints. sqlite> CREATE TABLE counters(action TEXT PRIMARY KEY, count INTEGER DEFAULT 0);
import datetime import logging import os import sqlite3 import redis from tornado import web, ioloop, gen, options logger = logging.getLogger(__name__) class ActionHandler(web.RequestHandler): ALLOWED_ACTIONS = ['open_browser', 'open_new_tab', 'enter_search_term', 'scroll', 'click_on_link'] REDIS_COUNTER_KEY = 'actions:counter:{action}' REDIS_TASK_KEY = 'actions:task:{action}' UPDATE_PERIOD = 30 # seconds def inc_count(self, action): key = self.REDIS_COUNTER_KEY.format(action=action) counter_key = self.REDIS_COUNTER_KEY.format(action=action) task_key = self.REDIS_TASK_KEY.format(action=action) count, delted_counter, deleted_task = self.application.redis.pipeline( ).get(counter_key).delete(counter_key).delete(task_key).execute() if not count: return with sqlite3.connect(self.application.db_path) as connection: cursor = connection.cursor() result = cursor.executescript("INSERT OR REPLACE INTO counters(action, count) VALUES ('{action}', COALESCE((SELECT count + 1 FROM counters WHERE action = '{action}'), 1));".format( count=count, action=action)) logger.warning('Saved to sqlite3.') def post(self): action = self.get_body_argument('action') if action in self.ALLOWED_ACTIONS: counter_key = self.REDIS_COUNTER_KEY.format(action=action) task_key = self.REDIS_TASK_KEY.format(action=action) count, task_exists = self.application.redis.pipeline().incr(counter_key, 1).get(task_key).execute() if not task_exists: ioloop.IOLoop.instance().add_timeout( deadline=datetime.timedelta(seconds=self.UPDATE_PERIOD), callback=self.inc_count, action=action) self.application.redis.setex(task_key, self.UPDATE_PERIOD * 2, 1) logger.warning('Incremented.') self.write('Ok') class CounterApplication(web.Application): def __init__(self, *args, **kwargs): handlers = [(r'/action', ActionHandler)] kwargs['debug'] = True self.db_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'counters.sqlite3') self.redis = redis.StrictRedis(host='localhost') super(CounterApplication, self).__init__(handlers, *args, **kwargs) if __name__ == "__main__": options.parse_command_line() CounterApplication().listen(5000) ioloop.IOLoop.instance().start() # curl --data "action=open_browser" http://localhost:5000/action # [W 150308 20:27:00 app:46] Incremented. # [I 150308 20:27:00 web:1825] 200 POST /action (::1) 7.90ms # [W 150308 20:27:01 app:46] Incremented. # [I 150308 20:27:01 web:1825] 200 POST /action (::1) 1.74ms # [W 150308 20:27:02 app:46] Incremented. # [I 150308 20:27:02 web:1825] 200 POST /action (::1) 1.81ms # [W 150308 20:27:05 app:32] Saved to sqlite3. # [W 150308 20:27:10 app:46] Incremented. # [I 150308 20:27:10 web:1825] 200 POST /action (::1) 1.56ms # [W 150308 20:27:11 app:46] Incremented. # [I 150308 20:27:11 web:1825] 200 POST /action (::1) 1.29ms # [W 150308 20:27:12 app:46] Incremented. # [I 150308 20:27:12 web:1825] 200 POST /action (::1) 1.38ms # [W 150308 20:27:12 app:46] Incremented. # [I 150308 20:27:12 web:1825] 200 POST /action (::1) 1.38ms # [W 150308 20:27:13 app:46] Incremented. # [I 150308 20:27:13 web:1825] 200 POST /action (::1) 1.29ms # [W 150308 20:27:14 app:46] Incremented. # [I 150308 20:27:14 web:1825] 200 POST /action (::1) 1.72ms # [W 150308 20:27:15 app:32] Saved to sqlite3.
(.env)nanvel-air:tornado_counter nanvel$ sqlite3 counters.sqlite3 SQLite version 3.8.5 2014-08-15 22:37:57 Enter ".help" for usage hints. sqlite> select * from counters; open_new_tab|3 open_browser|21
Licensed under CC BY-SA 3.0