Skip to content
Snippets Groups Projects
Commit ae15ce81 authored by John Gerlock's avatar John Gerlock
Browse files

Add error handling and statds support

parent bb576767
No related branches found
No related tags found
No related merge requests found
...@@ -55,4 +55,4 @@ local.db ...@@ -55,4 +55,4 @@ local.db
envfile envfile
/deploy /deploy
.venv/ .venv/
\ No newline at end of file
# EditorConfig is awesome: http://EditorConfig.org
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
# 4 space indentation
[*.py]
indent_style = space
indent_size = 4
# isort config
force_single_line=True
known_third_party=aiocache,aiohttp,funcy,pygtrie,ujson,sanic,statsd,websockets
# Tab indentation (no size specified)
[Makefile]
indent_style = tab
# Indentation override for all JS under lib directory
[lib/**.js]
indent_style = space
indent_size = 2
# Matches the exact files either package.json or .travis.yml
[{package.json,.travis.yml}]
indent_style = space
indent_size = 2
- repo: git://github.com/pre-commit/pre-commit-hooks
sha: v0.9.1
hooks:
- id: check-ast
- id: autopep8-wrapper
- id: trailing-whitespace
- id: check-byte-order-marker
- id: check-executables-have-shebangs
- id: check-json
- id: check-yaml
- id: check-symlinks
- id: check-case-conflict
- id: debug-statements
- id: check-merge-conflict
- id: detect-aws-credentials
- id: detect-private-key
- id: end-of-file-fixer
- id: fix-encoding-pragma
- id: check-added-large-files
- repo: git://github.com/FalconSocial/pre-commit-python-sorter
sha: 1.0.4
hooks:
- id: python-import-sorter
args: ['--silent-overwrite']
- repo: git://github.com/pre-commit/mirrors-yapf
sha: v0.16.2
hooks:
- id: yapf
- repo: git://github.com/Lucas-C/pre-commit-hooks
sha: v1.0.1
hooks:
- id: remove-tabs
exclude: Makefile
- repo: local
hooks:
- id: pylint
name: pylint
entry: pipenv run pylint
language: system
types: [python]
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
# Python code to execute, usually for sys.path manipulation such as # Python code to execute, usually for sys.path manipulation such as
# pygtk.require(). # pygtk.require().
#init-hook= init-hook='import sys; pth="{}/lib/python3.6".format(os.environ.get("VIRTUAL_ENV")); sys.path.append(pth)'
# Add files or directories to the blacklist. They should be base names, not # Add files or directories to the blacklist. They should be base names, not
# paths. # paths.
......
FROM phusion/baseimage:0.9.19 FROM phusion/baseimage:0.9.19
ENV LOG_LEVEL INFO ENV LOG_LEVEL INFO
ENV LANG en_US.UTF-8 ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8 ENV LC_ALL en_US.UTF-8
ENV PIPENV_VENV_IN_PROJECT 1
ENV APP_ROOT /app ENV APP_ROOT /app
ENV APP_CMD ${APP_ROOT}/jussi/serve.py ENV APP_CMD ${APP_ROOT}/jussi/serve.py
ENV PIPENV_VENV_IN_PROJECT 1
# all nginx env vars must also be changed in service/nginx/nginx.conf # all nginx env vars must also be changed in service/nginx/nginx.conf
ENV NGINX_SERVER_PORT 8080 ENV NGINX_SERVER_PORT 8080
ENV JUSSI_SERVER_HOST 0.0.0.0 ENV JUSSI_SERVER_HOST 0.0.0.0
ENV JUSSI_SERVER_PORT 9000 ENV JUSSI_SERVER_PORT 9000
ENV JUSSI_STEEMD_WS_URL wss://steemd.steemitdev.com ENV JUSSI_STEEMD_WS_URL wss://steemd.steemitdev.com
...@@ -19,8 +15,6 @@ ENV JUSSI_SBDS_HTTP_URL https://sbds.steemitdev.com ...@@ -19,8 +15,6 @@ ENV JUSSI_SBDS_HTTP_URL https://sbds.steemitdev.com
ENV JUSSI_REDIS_PORT 6379 ENV JUSSI_REDIS_PORT 6379
ENV ENVIRONMENT PROD ENV ENVIRONMENT PROD
RUN \ RUN \
apt-get update && \ apt-get update && \
apt-get install -y \ apt-get install -y \
...@@ -40,18 +34,15 @@ RUN \ ...@@ -40,18 +34,15 @@ RUN \
nginx \ nginx \
wget wget
RUN wget -q https://www.scalyr.com/scalyr-repo/stable/latest/scalyr-repo-bootstrap_1.2.1_all.deb # add scalyr agent
RUN dpkg -r scalyr-repo scalyr-repo-bootstrap # Remove any previous repository definitions, if any. RUN wget -q https://www.scalyr.com/scalyr-repo/stable/latest/scalyr-repo-bootstrap_1.2.1_all.deb && \
RUN dpkg -i ./scalyr-repo-bootstrap_1.2.1_all.deb dpkg -r scalyr-repo scalyr-repo-bootstrap && \ # Remove any previous repository definitions, if any.
dpkg -i ./scalyr-repo-bootstrap_1.2.1_all.deb && \
#RUN rm scalyr-repo-bootstrap_1.2.1_all.deb
RUN \
apt-get update && \ apt-get update && \
apt-get install -y \ apt-get install -y \
scalyr-repo \ scalyr-repo \
scalyr-agent-2 scalyr-agent-2 && \
rm scalyr-repo-bootstrap_1.2.1_all.deb
RUN \ RUN \
mkdir -p /var/lib/nginx/body && \ mkdir -p /var/lib/nginx/body && \
...@@ -78,8 +69,8 @@ WORKDIR /app ...@@ -78,8 +69,8 @@ WORKDIR /app
RUN \ RUN \
pip3 install --upgrade pip && \ pip3 install --upgrade pip && \
pip3 install pipenv && \ pip3 install pipenv && \
pipenv install --three && \ pipenv install --three && \
pipenv run python3 setup.py install && \ pipenv run python3 setup.py install && \
apt-get remove -y \ apt-get remove -y \
build-essential \ build-essential \
libffi-dev \ libffi-dev \
...@@ -97,4 +88,3 @@ RUN \ ...@@ -97,4 +88,3 @@ RUN \
EXPOSE ${NGINX_SERVER_PORT} EXPOSE ${NGINX_SERVER_PORT}
MIT License MIT License
Copyright (c) 2017 Copyright (c) 2017
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal
......
...@@ -24,7 +24,7 @@ run: ...@@ -24,7 +24,7 @@ run:
docker run $(PROJECT_DOCKER_RUN_ARGS) $(PROJECT_DOCKER_TAG) docker run $(PROJECT_DOCKER_RUN_ARGS) $(PROJECT_DOCKER_TAG)
run-local: run-local:
pipenv run python3 jussi/serve.py pipenv run python3 jussi/serve.py --server_workers=1
test: test:
pipenv run py.test tests pipenv run py.test tests
...@@ -34,4 +34,7 @@ lint: ...@@ -34,4 +34,7 @@ lint:
fmt: fmt:
pipenv run yapf --recursive --in-place --style pep8 $(PROJECT_NAME) pipenv run yapf --recursive --in-place --style pep8 $(PROJECT_NAME)
pipenv run autopep8 --recursive --in-placedo $(PROJECT_NAME) pipenv run autopep8 --recursive --in-placedo $(PROJECT_NAME)
\ No newline at end of file
pre-commit:
pipenv run pre-commit run --all-files
...@@ -14,9 +14,11 @@ sphinxcontrib-programoutput = "*" ...@@ -14,9 +14,11 @@ sphinxcontrib-programoutput = "*"
sphinxcontrib-restbuilder = "*" sphinxcontrib-restbuilder = "*"
yapf = "*" yapf = "*"
autopep8 = "*" autopep8 = "*"
Sphinx = "*" pre_commit = "*"
certifi = "*" certifi = "*"
urllib3 = "*" urllib3 = "*"
Sphinx = "*"
pylint = "*"
[packages] [packages]
ujson = "*" ujson = "*"
...@@ -27,12 +29,12 @@ cchardet = "*" ...@@ -27,12 +29,12 @@ cchardet = "*"
uvloop = "*" uvloop = "*"
statsd = "*" statsd = "*"
aiodns = "*" aiodns = "*"
Sanic = "*"
aiocache = "*" aiocache = "*"
pygtrie = "*" pygtrie = "*"
funcy = "*" funcy = "*"
pipenv = "*" pipenv = "*"
click = "*" click = "*"
Sanic = "*"
[requires] [requires]
python_version = "3.5" python_version = "3.6"
...@@ -9,9 +9,9 @@ POST / HTTP/1.1 ...@@ -9,9 +9,9 @@ POST / HTTP/1.1
Content-Type: application/json Content-Type: application/json
{ {
"method": "sbds.count_operations", "method": "sbds.count_operations",
"params": {"operation":"account_creates"}, "params": {"operation":"account_creates"},
"jsonrpc": "2.0", "jsonrpc": "2.0",
"id": 1 "id": 1
} }
``` ```
...@@ -24,9 +24,9 @@ POST / HTTP/1.1 ...@@ -24,9 +24,9 @@ POST / HTTP/1.1
Content-Type: application/json Content-Type: application/json
{ {
"method": "get_block", "method": "get_block",
"params": [1], "params": [1],
"jsonrpc": "2.0", "jsonrpc": "2.0",
"id": 1 "id": 1
} }
``` ```
...@@ -37,9 +37,9 @@ POST / HTTP/1.1 ...@@ -37,9 +37,9 @@ POST / HTTP/1.1
Content-Type: application/json Content-Type: application/json
{ {
"method": "steemd.get_block", "method": "steemd.get_block",
"params": [1], "params": [1],
"jsonrpc": "2.0", "jsonrpc": "2.0",
"id": 1 "id": 1
} }
``` ```
# -*- coding: utf-8 -*-
__all__ = [ __all__ = [
"__title__", "__title__",
"__version__", "__version__",
"__summary", "__summary__",
"__uri__", "__uri__",
"__author__", "__author__",
"__email__", "__email__",
......
# -*- coding: utf-8 -*-
# coding=utf-8 # -*- coding: utf-8 -*-
import asyncio import asyncio
import hashlib import hashlib
import logging import logging
import aiocache
import aiocache.plugins
import ujson import ujson
from jussi.serializers import CompressionSerializer
logger = logging.getLogger('sanic') logger = logging.getLogger('sanic')
async def setup_caches(app, loop):
logger.info('before_server_start -> setup_cache')
args = app.config.args
# only use redis if we can really talk to it
caches_config = {
'default': {
'cache': aiocache.SimpleMemoryCache,
'serializer': {
'class': CompressionSerializer
},
'plugins': [
{'class': aiocache.plugins.HitMissRatioPlugin},
{'class': aiocache.plugins.TimingPlugin}
]
},
'redis': {
'cache': aiocache.RedisCache,
'endpoint': args.redis_host,
'port': args.redis_port,
'timeout': 3,
'serializer': {
'class': CompressionSerializer
},
'plugins': [
{'class': aiocache.plugins.HitMissRatioPlugin},
{'class': aiocache.plugins.TimingPlugin}
]
}
}
aiocache.caches.set_config(caches_config)
try:
cache = aiocache.caches.get('redis')
await cache.set('test', b'testval')
val = await cache.get('test')
logger.debug('before_server_start -> setup_cache val=%s', val)
assert val == b'testval'
except Exception as e:
logger.exception(e)
logger.error(
'Unable to use redis (was a setting not defined?), using in-memory cache instead...'
)
del caches_config['redis']
aiocache.caches.set_config(caches_config)
return aiocache.caches
def jsonrpc_cache_key(single_jsonrpc_request): def jsonrpc_cache_key(single_jsonrpc_request):
if isinstance(single_jsonrpc_request.get('params'), dict): if isinstance(single_jsonrpc_request.get('params'), dict):
# the params dict should already be sorted, so no need to sort again # the params dict should already be sorted, so no need to sort again
...@@ -20,16 +72,19 @@ def jsonrpc_cache_key(single_jsonrpc_request): ...@@ -20,16 +72,19 @@ def jsonrpc_cache_key(single_jsonrpc_request):
).encode()).hexdigest()) ).encode()).hexdigest())
async def cache_get(app, jussi_attrs): async def cache_get(request, jussi_attrs):
cache = app.config.cache caches = request.app.config.caches
logger.debug('%s.get(%s)', cache, jussi_attrs.key) logger.debug('cache.get(%s)', jussi_attrs.key)
response = await cache.get(jussi_attrs.key)
if response: # happy eyeballs approach supports use of multiple caches, eg, SimpleMemoryCache and RedisCache
logger.debug(logger.debug('cache --> %s', response)) for result in asyncio.as_completed([cache.get(jussi_attrs.key) for cache in caches]):
return response response = await result
if response:
logger.debug(logger.debug('cache --> %s', response))
return response
async def cache_set(app, value, jussi_attrs): async def cache_set(request, value, jussi_attrs):
# ttl of -1 means don't cache # ttl of -1 means don't cache
ttl = jussi_attrs.ttl ttl = jussi_attrs.ttl
...@@ -38,12 +93,16 @@ async def cache_set(app, value, jussi_attrs): ...@@ -38,12 +93,16 @@ async def cache_set(app, value, jussi_attrs):
return return
elif ttl == 0: elif ttl == 0:
ttl = None ttl = None
cache = app.config.cache caches = request.app.config.caches
logger.debug('%s.set(%s, %s, ttl=%s)', cache, jussi_attrs.key, value, ttl) logger.debug('cache.set(%s, %s, ttl=%s)', jussi_attrs.key, value, ttl)
asyncio.ensure_future(cache.set(jussi_attrs.key, value, ttl=ttl)) for cache in caches:
# avoid using too much memory, especially beause there may be os.cpu_count() instances running
if isinstance(cache, aiocache.SimpleMemoryCache) and ttl >= 0:
ttl = 60
asyncio.ensure_future(cache.set(jussi_attrs.key, value, ttl=ttl))
async def cache_json_response(app, value, jussi_attrs): async def cache_json_response(request, value, jussi_attrs):
"""Don't cache error responses """Don't cache error responses
Args: Args:
...@@ -71,4 +130,4 @@ async def cache_json_response(app, value, jussi_attrs): ...@@ -71,4 +130,4 @@ async def cache_json_response(app, value, jussi_attrs):
parsed['error'], jussi_attrs.upstream_url) parsed['error'], jussi_attrs.upstream_url)
return return
else: else:
asyncio.ensure_future(cache_set(app, value, jussi_attrs)) asyncio.ensure_future(cache_set(request, value, jussi_attrs))
# coding=utf-8 # -*- coding: utf-8 -*-
import sys import sys
from sanic.defaultFilter import DefaultFilter from sanic.defaultFilter import DefaultFilter
LOGGING = { LOGGING = {
......
# coding=utf-8 # -*- coding: utf-8 -*-
import time
import logging import logging
import time
from sanic import response from sanic import response
from sanic.exceptions import InvalidUsage from sanic.exceptions import InvalidUsage
from .cache import cache_get from .cache import cache_get
from .timers import init_timers
from .timers import log_timers
from .utils import async_exclude_methods from .utils import async_exclude_methods
from .utils import jussi_attrs from .utils import jussi_attrs
from .utils import replace_jsonrpc_id
from .utils import sort_request from .utils import sort_request
from .timers import init_timers
from .timers import log_timers
logger = logging.getLogger('sanic') logger = logging.getLogger('sanic')
...@@ -20,7 +19,6 @@ async def start_stats(request): ...@@ -20,7 +19,6 @@ async def start_stats(request):
request['timers'] = init_timers(start_time=time.time()) request['timers'] = init_timers(start_time=time.time())
request['statsd'] = request.app.config.statsd_client.pipeline() request['statsd'] = request.app.config.statsd_client.pipeline()
@async_exclude_methods(exclude_http_methods=('GET', )) @async_exclude_methods(exclude_http_methods=('GET', ))
async def add_jussi_attrs(request): async def add_jussi_attrs(request):
# request.json handles json parse errors, this handles empty json # request.json handles json parse errors, this handles empty json
...@@ -38,7 +36,7 @@ async def caching_middleware(request): ...@@ -38,7 +36,7 @@ async def caching_middleware(request):
return return
jussi_attrs = request['jussi'] jussi_attrs = request['jussi']
with request['timers']['caching_middleware']: with request['timers']['caching_middleware']:
cached_response = await cache_get(request.app, jussi_attrs) cached_response = await cache_get(request, jussi_attrs)
if cached_response: if cached_response:
return response.raw( return response.raw(
...@@ -47,28 +45,27 @@ async def caching_middleware(request): ...@@ -47,28 +45,27 @@ async def caching_middleware(request):
headers={'x-jussi-cache-hit': jussi_attrs.key}) headers={'x-jussi-cache-hit': jussi_attrs.key})
# pylint: disable=unused-argument
async def finalize_timers(request, response): async def finalize_timers(request, response):
end = time.time() if request.get('timers'):
if not request.get('timers'): end = time.time()
logger.info('skipped finalizing timers, no timers to finalize') logger.info('skipped finalizing timers, no timers to finalize')
return return
request['timers']['total_jussi_elapsed'] request['timers']['total_jussi_elapsed'].end()
for name, timer in request['timers'].items(): for timer in request['timers'].values():
timer.end(end) timer.end(end)
log_timers(request.get('timers'), logger.debug) log_timers(request.get('timers'), logger.debug)
async def log_stats(request, response): async def log_stats(request, response):
if not request.get('timers'): if request.get('timers'):
logger.info('skipped logging timers, no timers to log') logger.info('skipped logging timers, no timers to log')
return log_timers(request.get('timers'), logger.info)
log_timers(request.get('timers'), logger.info) try:
pipe = request['statsd']
try: logger.debug(pipe._stats) # pylint: disable=protected-access
pipe = request['statsd'] for name, timer in request['timers'].items():
logger.info(pipe._stats) pipe.timing(name, timer.elapsed)
for name, timer in request['timers'].items(): pipe.send()
pipe.timing(name, timer.elapsed) except Exception as e:
pipe.send() logger.warning('Failed to send stats to statsd: %s', e)
except Exception as e:
logger.warning('Failed to send stats to statsd: %s', e)
# coding=utf-8 # -*- coding: utf-8 -*-
import zlib import zlib
import ujson
import ujson
from aiocache.serializers import StringSerializer from aiocache.serializers import StringSerializer
......
jussi/serve.py 100755 → 100644
#! /usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import argparse import argparse
import asyncio import asyncio
import functools import datetime
import logging import logging
import os import os
import ujson
import datetime
import aiocache
import aiocache.plugins
import aiohttp import aiohttp
import pygtrie
import funcy import funcy
import pygtrie
import ujson
import websockets import websockets
from statsd import StatsClient
from sanic import Sanic from sanic import Sanic
from sanic import response from sanic import response
from sanic.exceptions import SanicException from sanic.exceptions import SanicException
from statsd import StatsClient
from jussi.cache import cache_get from jussi.cache import cache_get
from jussi.cache import cache_json_response from jussi.cache import cache_json_response
from jussi.serializers import CompressionSerializer from jussi.cache import setup_caches
from jussi.logging_config import LOGGING from jussi.logging_config import LOGGING
from jussi.middlewares import add_jussi_attrs from jussi.middlewares import add_jussi_attrs
from jussi.middlewares import caching_middleware from jussi.middlewares import caching_middleware
from jussi.middlewares import start_stats
from jussi.middlewares import finalize_timers from jussi.middlewares import finalize_timers
from jussi.middlewares import log_stats from jussi.middlewares import log_stats
from jussi.utils import websocket_conn from jussi.middlewares import start_stats
from jussi.utils import return_bytes from jussi.utils import return_bytes
from jussi.utils import websocket_conn
# init logging # init logging
LOG_LEVEL = getattr(logging, os.environ.get('LOG_LEVEL', 'INFO')) LOG_LEVEL = getattr(logging, os.environ.get('LOG_LEVEL', 'INFO'))
...@@ -52,6 +48,8 @@ METHOD_CACHE_SETTINGS = (('get_block', 'steemd_websocket_url', ...@@ -52,6 +48,8 @@ METHOD_CACHE_SETTINGS = (('get_block', 'steemd_websocket_url',
('get_global_dynamic_properties', ('get_global_dynamic_properties',
'steemd_websocket_url', 1)) 'steemd_websocket_url', 1))
# pylint: disable=unused-argument
@funcy.log_calls(logger.debug) @funcy.log_calls(logger.debug)
@return_bytes @return_bytes
...@@ -88,7 +86,7 @@ async def dispatch_single(sanic_http_request, jsonrpc_request, jrpc_req_index): ...@@ -88,7 +86,7 @@ async def dispatch_single(sanic_http_request, jsonrpc_request, jrpc_req_index):
# return cached response if possible # return cached response if possible
with sanic_http_request['timers'][prefix('dispatch_single_caching_check')]: with sanic_http_request['timers'][prefix('dispatch_single_caching_check')]:
response = await cache_get(app, jussi_attrs) response = await cache_get(sanic_http_request, jussi_attrs)
if response: if response:
return response return response
...@@ -101,7 +99,7 @@ async def dispatch_single(sanic_http_request, jsonrpc_request, jrpc_req_index): ...@@ -101,7 +99,7 @@ async def dispatch_single(sanic_http_request, jsonrpc_request, jrpc_req_index):
jsonrpc_request) jsonrpc_request)
asyncio.ensure_future( asyncio.ensure_future(
cache_json_response(app, bytes_response, jussi_attrs=jussi_attrs)) cache_json_response(sanic_http_request, bytes_response, jussi_attrs=jussi_attrs))
return bytes_response return bytes_response
...@@ -122,7 +120,7 @@ async def dispatch_batch(sanic_http_request, jsonrpc_requests): ...@@ -122,7 +120,7 @@ async def dispatch_batch(sanic_http_request, jsonrpc_requests):
@app.post('/') @app.post('/')
async def handle(sanic_http_request): async def handle_jsonrpc(sanic_http_request):
app = sanic_http_request.app app = sanic_http_request.app
# retreive parsed jsonrpc_requests after request middleware processing # retreive parsed jsonrpc_requests after request middleware processing
...@@ -147,7 +145,7 @@ async def handle(sanic_http_request): ...@@ -147,7 +145,7 @@ async def handle(sanic_http_request):
# health check routes # health check routes
@app.get('/') @app.get('/')
async def handle(sanic_http_request): async def handle_root_health(sanic_http_request):
return response.json({ return response.json({
'status': 'OK', 'status': 'OK',
'datetime': datetime.datetime.utcnow().isoformat() 'datetime': datetime.datetime.utcnow().isoformat()
...@@ -155,7 +153,7 @@ async def handle(sanic_http_request): ...@@ -155,7 +153,7 @@ async def handle(sanic_http_request):
@app.get('/health') @app.get('/health')
async def handle(sanic_http_request): async def handle_health(sanic_http_request):
return response.json({ return response.json({
'status': 'OK', 'status': 'OK',
'datetime': datetime.datetime.utcnow().isoformat() 'datetime': datetime.datetime.utcnow().isoformat()
...@@ -164,7 +162,7 @@ async def handle(sanic_http_request): ...@@ -164,7 +162,7 @@ async def handle(sanic_http_request):
@app.exception(SanicException) @app.exception(SanicException)
def handle_errors(request, exception): def handle_errors(request, exception):
"""all errors return HTTP 502 """handles all errors
Args: Args:
request: request:
...@@ -173,14 +171,14 @@ def handle_errors(request, exception): ...@@ -173,14 +171,14 @@ def handle_errors(request, exception):
Returns: Returns:
""" """
logger.exception('%s caused %s', request, exception)
status_code = getattr(exception, 'status_code', 502) status_code = getattr(exception, 'status_code', 502)
message = str(exception) or 'Gateway Error' message = str(exception) or 'Gateway Error'
logger.error('%s-%s', request, exception) return response.json(status=status_code, body=
return response.json(
{ {
'error': message, 'error': 'Gateway Error',
'status_code': status_code 'status_code': status_code
}, status=status_code) })
# register listeners # register listeners
...@@ -210,53 +208,27 @@ def setup_middlewares(app, loop): ...@@ -210,53 +208,27 @@ def setup_middlewares(app, loop):
app.request_middleware.append(add_jussi_attrs) app.request_middleware.append(add_jussi_attrs)
app.request_middleware.append(caching_middleware) app.request_middleware.append(caching_middleware)
app.response_middleware.append(finalize_timers) #app.response_middleware.append(finalize_timers)
app.response_middleware.append(log_stats) #app.response_middleware.append(log_stats)
@app.listener('before_server_start') @app.listener('before_server_start')
async def setup_cache(app, loop): async def setup_cache(app, loop):
logger.info('before_server_start -> setup_cache') logger.info('before_server_start -> setup_cache')
args = app.config.args args = app.config.args
# only use redis if we can really talk to it caches = await setup_caches(app, loop)
logger.info('before_server_start -> setup_cache redis_host:%s', for cache_alias in caches.get_config().keys():
args.redis_host) logger.info('before_server_start -> setup_cache caches=%s', cache_alias)
logger.info('before_server_start -> setup_cache redis_port:%s', active_caches = [caches.get(alias) for alias in caches.get_config().keys()]
args.redis_port)
try:
if not args.redis_host:
raise ValueError('no redis host specified')
default_cache = aiocache.RedisCache(
serializer=CompressionSerializer(),
endpoint=args.redis_host,
port=args.redis_port,
plugins=[
aiocache.plugins.HitMissRatioPlugin(),
aiocache.plugins.TimingPlugin()
])
await default_cache.set('test', b'testval')
val = await default_cache.get('test')
logger.debug('before_server_start -> setup_cache val=%s', val)
assert val == b'testval'
except Exception as e:
logger.exception(e)
logger.error(
'Unable to use redis (was a setting not defined?), using in-memory cache instead...'
)
default_cache = aiocache.SimpleMemoryCache(
serializer=CompressionSerializer(),
plugins=[
aiocache.plugins.HitMissRatioPlugin(),
aiocache.plugins.TimingPlugin()
])
logger.info('before_server_start -> setup_cache cache=%s', default_cache)
cache_config = dict() cache_config = dict()
cache_config['default_cache_ttl'] = DEFAULT_CACHE_TTL cache_config['default_cache_ttl'] = DEFAULT_CACHE_TTL
cache_config['no_cache_ttl'] = NO_CACHE_TTL cache_config['no_cache_ttl'] = NO_CACHE_TTL
cache_config['no_cache_expire_ttl'] = NO_CACHE_EXPIRE_TTL cache_config['no_cache_expire_ttl'] = NO_CACHE_EXPIRE_TTL
app.config.cache_config = cache_config app.config.cache_config = cache_config
app.config.cache = default_cache app.config.caches = active_caches
@app.listener('before_server_start') @app.listener('before_server_start')
......
# coding=utf-8 # -*- coding: utf-8 -*-
import time import time
from collections import defaultdict from collections import defaultdict
...@@ -6,13 +6,13 @@ from collections import defaultdict ...@@ -6,13 +6,13 @@ from collections import defaultdict
class Timer(object): class Timer(object):
__slots__ = ('_start_time', '_end_time') __slots__ = ('_start_time', '_end_time')
def __init__(self, start=None, end=None): def __init__(self, start_time=None, end=None):
self._start_time = start or time.time() self._start_time = start_time or time.time()
self._end_time = end self._end_time = end
def start(self, start=None): def start(self, start_time=None):
if start: if start_time:
self._start_time = start self._start_time = start_time
def restart(self): def restart(self):
self._start_time = time.time() self._start_time = time.time()
...@@ -34,7 +34,7 @@ class Timer(object): ...@@ -34,7 +34,7 @@ class Timer(object):
return str(self.elapsed) return str(self.elapsed)
def __repr__(self): def __repr__(self):
return self.elapsed return 'Timer(start_time=%s end_time=%s elapsed=%s)' % (self._start_time, self._end_time, self.elapsed)
def __enter__(self, start=None): def __enter__(self, start=None):
if start: if start:
...@@ -46,11 +46,11 @@ class Timer(object): ...@@ -46,11 +46,11 @@ class Timer(object):
def init_timers(start_time=None): def init_timers(start_time=None):
return defaultdict(lambda: Timer(start=start_time)) return defaultdict(lambda: Timer(start_time=start_time))
def end_timers(timers, end=None): def end_timers(timers, end=None):
for name, timer in timers.items(): for timer in timers.values():
if end: if end:
timer.end(end) timer.end(end)
return timers return timers
...@@ -63,6 +63,6 @@ def display_timers(timers, end=None): ...@@ -63,6 +63,6 @@ def display_timers(timers, end=None):
print('%s %s' % (name, timer)) print('%s %s' % (name, timer))
def log_timers(timers, log_func, end=None): def log_timers(timers, log_func):
for name, timer in timers.items(): for name, timer in timers.items():
log_func('%s %s' % (name, timer)) log_func('%s %s' % (name, timer))
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import functools import functools
import time
import logging import logging
import websockets import time
from collections import OrderedDict from collections import OrderedDict
from collections import namedtuple from collections import namedtuple
import websockets
from sanic.exceptions import InvalidUsage from sanic.exceptions import InvalidUsage
from jussi.cache import jsonrpc_cache_key from jussi.cache import jsonrpc_cache_key
...@@ -87,9 +87,8 @@ def async_exclude_methods(middleware_func=None, exclude_http_methods=None): ...@@ -87,9 +87,8 @@ def async_exclude_methods(middleware_func=None, exclude_http_methods=None):
@functools.wraps(middleware_func) @functools.wraps(middleware_func)
async def f(request): async def f(request):
if request.method in exclude_http_methods: if request.method in exclude_http_methods:
return request return
else: return await middleware_func(request)
return await middleware_func(request)
return f return f
......
...@@ -15,4 +15,4 @@ exec 2>&1 \ ...@@ -15,4 +15,4 @@ exec 2>&1 \
--sbds_url "${JUSSI_SBDS_HTTP_URL}" \ --sbds_url "${JUSSI_SBDS_HTTP_URL}" \
--redis_host "${JUSSI_REDIS_HOST:-''}" \ --redis_host "${JUSSI_REDIS_HOST:-''}" \
--redis_port "${JUSSI_REDIS_PORT:-6379}" \ --redis_port "${JUSSI_REDIS_PORT:-6379}" \
2>&1 2>&1
\ No newline at end of file
...@@ -68,4 +68,4 @@ upstream jussi_upstream { ...@@ -68,4 +68,4 @@ upstream jussi_upstream {
proxy_pass http://jussi_upstream; proxy_pass http://jussi_upstream;
} }
} }
} }
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment