Unverified Commit 273b9dec authored by roadscape's avatar roadscape Committed by GitHub

Merge branch 'master' into route-getstate-witnesses

parents c1dc7a58 431bd3cb
......@@ -54,3 +54,8 @@ perf/
upstream_traffic.json
Untitled.ipynb
.pytest_cache
*.ipynb
.ipynb_checkpoints
build
prof/
local/
......@@ -13,9 +13,6 @@ trim_trailing_whitespace = true
indent_style = space
indent_size = 4
# isort config
force_single_line=True
from_first=False
# Tab indentation (no size specified)
[Makefile]
......
......@@ -124,3 +124,8 @@ api_initial_schema.json
api_signatures.json
Untitled.ipynb
.pytest_cache
*.ipynb
.ipynb_checkpoints
build
prof/
local/
[settings]
line_length=120
#force_to_top=file1.py,file2.py
#skip=file3.py,file4.py
#known_future_library=future,pies
#known_standard_library=std,std2
known_third_party=ujson,cytoolz,aredis
#known_first_party=mylib1,mylib2
indent=' '
length_sort=False
#forced_separate=django.contrib,django.utils
default_section=FIRSTPARTY
force_single_line=True
from_first=False
repos:
- repo: git://github.com/pre-commit/pre-commit-hooks
sha: v0.9.5
rev: v0.9.5
hooks:
- id: check-ast
- id: trailing-whitespace
......@@ -24,15 +25,15 @@
# hooks:
# - id: dockerfile_lint
- repo: git://github.com/Lucas-C/pre-commit-hooks
sha: v1.1.4
rev: v1.1.5
hooks:
- id: remove-tabs
exclude: Makefile
- repo: git://github.com/detailyang/pre-commit-shell
sha: 1b26bf757e6d5b9bfed2566339c6464f44779678
hooks:
- id: shell-lint
files: run$
#- repo: git://github.com/detailyang/pre-commit-shell
# rev: 1.0.2
# hooks:
# - id: shell-lint
# files: run$
- repo: local
hooks:
- id: check-bash-syntax
......
......@@ -7,11 +7,11 @@
"upstreams": [
{
"name": "steemd",
"translate_to_appbase": false,
"translate_to_appbase": true,
"urls": [
[
"steemd",
"wss://steemd.steemitstage.com"
"wss://steemd-appbase.steemit.com"
]
],
"ttls": [
......@@ -145,6 +145,10 @@
[
"appbase.condenser_api.broadcast_transaction_synchronous",
0
],
[
"appbase.condenser_api.get_ops_in_block.params=[2889020,false]",
20
]
]
}
......
......@@ -20,6 +20,7 @@ ENV APP_ROOT /app
ENV APP_CMD jussi.serve
ENV JUSSI_SERVER_HOST 0.0.0.0
ENV JUSSI_SERVER_PORT 9000
ENV JUSSI_MONITOR_PORT 7777
# all nginx env vars must also be changed in service/nginx/nginx.conf
ENV NGINX_SERVER_PORT 8080
......@@ -53,13 +54,13 @@ RUN \
RUN \
wget https://www.python.org/ftp/python/3.6.2/Python-3.6.2.tar.xz && \
tar xvf Python-3.6.2.tar.xz && \
cd Python-3.6.2/ && \
wget https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tar.xz && \
tar xvf Python-3.6.5.tar.xz && \
cd Python-3.6.5/ && \
./configure && \
make altinstall && \
cd .. && \
rm -rf Python-3.6.2.tar.xz Python-3.6.2/
rm -rf Python-3.6.5.tar.xz Python-3.6.5/
# nginx
RUN \
......@@ -115,3 +116,4 @@ RUN chown -R www-data . && \
RUN pipenv run pytest
EXPOSE ${NGINX_SERVER_PORT}
EXPOSE ${JUSSI_MONITOR_PORT}
......@@ -3,7 +3,7 @@ ROOT_DIR := $(shell pwd)
PROJECT_NAME := $(notdir $(ROOT_DIR))
PROJECT_DOCKER_TAG := steemit/$(PROJECT_NAME)
PROJECT_DOCKER_RUN_ARGS := -p8080:8080 --env-file .env -v $(shell pwd)/ALT_config.json:/app/ALT_config.json
PROJECT_DOCKER_RUN_ARGS := -p8080:8080 -p7777:7777 --env-file .env -v $(shell pwd)/DEV_config.json:/app/DEV_config.json
PIPENV_VENV_IN_PROJECT := 1
export PIPENV_VENV_IN_PROJECT
......@@ -82,7 +82,7 @@ remove-unused-imports: ## remove unused imports from python files
.PHONY: sort-imports
sort-imports: ## sorts python imports using isort with settings from .editorconfig
pipenv run isort --verbose --recursive --atomic --settings-path .editorconfig --virtual-env .venv $(PROJECT_NAME)
pipenv run isort --verbose --recursive --atomic --settings-path .isort.cfg --virtual-env .venv $(PROJECT_NAME)
.PHONY: pipenv-check
pipenv-check:
......@@ -134,24 +134,27 @@ mypy: ## run mypy type checking on python files
.PHONY: test-local-steemd-calls
test-local-steemd-calls:
pipenv run pytest -vv tests/test_responses.py::test_response_results_type --jussiurl http://localhost:9000
pipenv run pytest -vv tests/test_responses.py::test_steemd_responses --jussiurl http://localhost:9000
.PHONY: test-local-appbase-calls
test-local-appbase-calls:
pipenv run pytest -vv tests/test_responses.py::test_appbase_responses --jussiurl http://localhost:9000
.PHONY: test-local-appbase-translation-calls
test-local-appbase-translation-calls:
pipenv run pytest -vv tests/test_responses.py::test_appbase_translation_responses --jussiurl http://localhost:9000
.PHONY: test-live-dev-steemd-calls
test-live-dev-steemd-calls:
pipenv run pytest -vv tests/test_responses.py::test_response_results_type --jussiurl https://api.steemitdev.com
.PHONY: test-live-staging-steemd-calls
test-live-staging-steemd-calls:
pipenv run pytest -vv tests/test_responses.py::test_response_results_type --jussiurl https://api.steemitstage.com
.PHONY: test-live-dev-appbase-calls
test-live-dev-appbase-calls:
pipenv run pytest -vv tests/test_responses.py::test_appbase_responses --jussiurl https://api.steemitdev.com
.PHONY: test-live-staging-appbase-calls
test-live-staging-appbase-calls:
pipenv run pytest -vv tests/test_responses.py::test_appbase_responses --jussiurl https://api.steemitstage.com
.PHONY: test-live-prod-steemd-calls
test-live-prod-steemd-calls:
pipenv run pytest --maxfail=1 tests/test_responses.py::test_response_results_type --jussiurl https://api.steemit.com
.PHONY: test-live-prod-appbase-calls
test-live-prod-appbase-calls:
pipenv run pytest --maxfail=1 tests/test_responses.py::test_appbase_responses --jussiurl https://api.steemit.com
./perf:
......
......@@ -3,67 +3,61 @@ url = "https://pypi.python.org/simple"
verify_ssl = true
[dev-packages]
sanic = "*"
"autopep8" = "*"
"gprof2dot" = "*"
"pep8" = "*"
"pyprof2calltree" = "*"
"urllib3" = "*"
asynctest = "*"
autoflake = "*"
"autopep8" = "*"
awscli = "*"
certifi = "*"
crayons = "*"
"gprof2dot" = "*"
cython = "*"
ipython = "*"
isort = "*"
jsonschema = "*"
jupyter = "*"
line-profiler = "*"
mypy = "*"
"pep8" = "*"
pre-commit = "*"
progress = "*"
pydevd = "*"
pylint = "*"
"pyprof2calltree" = "*"
pytest = "*"
pytest-asyncio = "*"
pytest-console-scripts = "*"
pytest-cov = "*"
pytest-docker = "*"
pytest-mock = "*"
pytest = "==3.10.0"
pytest-asyncio = "==0.9.0"
pytest-console-scripts = "==0.1.7"
pytest-cov = "==2.6.0"
pytest-docker = "==0.6.1"
pytest-mock = "==1.10.0"
pytest-profiling = "*"
pytest-pylint = "*"
pytest-sanic = "*"
pytest-timeout = "*"
python-rapidjson = "*"
requests = "*"
"urllib3" = "*"
yapf = "*"
progress = "*"
jsonschema = "==2.6.0"
isort = "*"
pydevd = "*"
line-profiler = "*"
structlog = "*"
python-rapidjson = "*"
jupyter = "*"
funcy = "*"
vprof = "*"
[packages]
ujson = "*"
aiohttp = "*"
toolz = "*"
uvloop = "*"
statsd = "*"
aiodns = "*"
pygtrie = "*"
funcy = "*"
pipenv = "*"
aiohttp = "*"
aredis = "*"
cchardet = "*"
configargparse = "*"
cytoolz = "*"
janus = "*"
hiredis = "*"
pipenv = "*"
pygtrie = "*"
python-json-logger = "*"
cchardet = "*"
async-timeout = "*"
progress = "*"
pytest = "*"
aiocache = {extras = ["redis"]}
websockets = "*"
python-rapidjson = "*"
sanic = "*"
msgpack = "*"
configargparse = "*"
structlog = "*"
python-rapidjson = "*"
line-profiler = "*"
ujson = "*"
uvloop = "*"
websockets = "*"
async-timeout = "*"
[requires]
python_version = "3.6"
This diff is collapsed.
echo Build started on `date`
export IMAGE_TAG=`git ls-remote --heads origin | grep $(git rev-parse HEAD) | cut -d / -f 3`
if [ "$IMAGE_TAG" = "master" ] ; then export IMAGE_TAG=latest ; fi
export REPO_PATH=`git rev-parse --show-toplevel`
export REPO_NAME=`basename $REPO_PATH`
export IMAGE_REPO_NAME="steemit/$REPO_NAME"
export SOURCE_COMMIT=`git rev-parse HEAD`
echo Building branch $IMAGE_TAG from $IMAGE_REPO_NAME
docker build . -t $IMAGE_REPO_NAME:$IMAGE_TAG --build-arg SOURCE_COMMIT="${SOURCE_COMMIT}" --build-arg DOCKER_TAG="${IMAGE_TAG}"
wrk.method = "POST"
wrk.headers["Content-Type"] = "application/json"
function request()
body = '[{"id":1000,"jsonrpc":"2.0","method":"get_block","params":[' .. math.random(1,20000000) .. ']},{"id":2000,"jsonrpc":"2.0","method":"get_block","params":[' .. math.random(1,20000000) .. ']},{"id":3000,"jsonrpc":"2.0","method":"get_block","params":[' .. math.random(1,20000000) .. ']},{"id":4000,"jsonrpc":"2.0","method":"get_block","params":[' .. math.random(1,20000000) .. ']},{"id":5000,"jsonrpc":"2.0","method":"get_block","params":[' .. math.random(1,20000000) .. ']}]'
return wrk.format(nil,nil,nil,body)
end
......@@ -6,7 +6,7 @@
"speed_upload":%{speed_upload},
"time_appconnect":%{time_appconnect},
"time_connect":%{time_connect},
"time_nameloolup":%{time_namelookup},
"time_namelookup":%{time_namelookup},
"time_pretransfer":%{time_pretransfer},
"time_redirect":%{time_redirect},
"time_starttransfer":%{time_starttransfer},
......
......@@ -4,6 +4,7 @@
import json
import os
import sys
import random
count = int(sys.argv[1])
batch_size = int(sys.argv[2])
......@@ -25,7 +26,7 @@ def chunkify(iterable, chunksize=10000):
yield chunk
requests = [dict(id=i, jsonrpc='2.0', method='get_block', params=[i])
requests = [dict(id=i, jsonrpc='2.0', method='get_block', params=[random.randint(1, 20_000_000)])
for i in range(count)]
if batch_size > 1:
requests = list(chunkify(requests, batch_size))
......
blk_num = math.random(1, 20000000)
wrk.method = "POST"
wrk.body = '{"id":1,"jsonrpc":"2.0","method":"get_block","params":[1000]}'
wrk.body = '{"id":1,"jsonrpc":"2.0","method":"condenser_api.get_block","params":[1000]}'
wrk.headers["Content-Type"] = "application/json"
wrk.method = "POST"
wrk.headers["Content-Type"] = "application/json"
function request()
body = '{"id":1,"jsonrpc":"2.0","method":"condenser_api.get_block","params":['.. math.random(1,20000000) .. ']}'
return wrk.format(nil, nil, nil, body)
end
wrk.method = "GET"
wrk.headers["Content-Type"] = "application/json"
{"jsonrpc":"2.0", "method":"jussi.health", "id":1}
# -*- coding: utf-8 -*-
from sanic import Sanic
from sanic.response import json
import ujson
app = Sanic()
resp = {
"id": 1,
"jsonrpc": "2.0",
"result": {
"block_id": "000003e8b922f4906a45af8e99d86b3511acd7a5",
"extensions": [],
"previous": "000003e7c4fd3221cf407efcf7c1730e2ca54b05",
"signing_key": "STM8GC13uCZbP44HzMLV6zPZGwVQ8Nt4Kji8PapsPiNq1BK153XTX",
"timestamp": "2016-03-24T16:55:30",
"transaction_ids": [],
"transaction_merkle_root": "0000000000000000000000000000000000000000",
"transactions": [],
"witness": "initminer",
"witness_signature": "207f15578cac20ac0e8af1ebb8f463106b8849577e21cca9fc60da146d1d95df88072dedc6ffb7f7f44a9185bbf9bf8139a5b4285c9f423843720296a44d428856"
}
}
resp_json = ujson.dumps(resp).encode('utf8')
@app.route("/hello")
async def test(request):
return json({"hello": "world"})
@app.websocket('/')
async def feed(request, ws):
while True:
await ws.send(resp_json)
if __name__ == "__main__":
app.run(host="0.0.0.0", port=80)
# -*- coding: utf-8 -*-
from distutils.core import setup
from Cython.Build import cythonize
setup(
ext_modules=cythonize("jussi/ws/pool3.pyx")
)
FROM python:3.6-alpine
RUN mkdir /app
RUN apk add --no-cache --virtual .build-deps \
bzip2-dev \
coreutils \
dpkg-dev dpkg \
expat-dev \
gcc \
gdbm-dev \
libc-dev \
libffi-dev \
libnsl-dev \
libressl \
libressl-dev \
libtirpc-dev \
linux-headers \
make \
ncurses-dev \
pax-utils \
readline-dev \
sqlite-dev \
tcl-dev \
tk \
tk-dev \
xz-dev \
zlib-dev
RUN pip install uvloop
RUN apk del .build-deps
COPY udpserver.py /app/udpserver.py
EXPOSE 8125/udp
WORKDIR /app
CMD ["/app/udpserver.py"]
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import asyncio
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
class EchoServerProtocol:
def connection_made(self, transport):
self.transport = transport
def datagram_received(self, data, addr):
message = data.decode()
[print(line) for line in message.split('\n')]
loop = asyncio.get_event_loop()
print("Starting UDP server")
# One protocol instance will be created to serve all client requests
listen = loop.create_datagram_endpoint(
EchoServerProtocol, local_addr=('0.0.0.0', 8125))
transport, protocol = loop.run_until_complete(listen)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
transport.close()
loop.close()
# coding=utf-8
'''
from cpython.dict cimport PyDict_GetItem, PyDict_SetItem
from cpython.exc cimport PyErr_Clear, PyErr_GivenExceptionMatches, PyErr_Occurred
from cpython.list cimport PyList_Append, PyList_GET_ITEM, PyList_GET_SIZE
from cpython.object cimport PyObject_RichCompareBool, Py_NE
from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF
from cpython.sequence cimport PySequence_Check
from cpython.set cimport PySet_Add, PySet_Contains
from cpython.tuple cimport PyTuple_GET_ITEM, PyTuple_GetSlice, PyTuple_New, PyTuple_SET_ITEM
from typing import List,Tuple
from time import perf_counter
from libc.stdio cimport sprintf
'''
from cpython.object cimport Py_SIZE
from time import perf_counter
from cpython cimport array
import array
from cpython.list cimport PyList_Append
cdef class Timings:
cdef bytes prefix
cdef list names
cdef array.array timings
def __cinit__(self, bytes prefix):
self.prefix = prefix
self.timings = array.array('d',[])
self.names = ['created']
cpdef record(self, name:str):
cdef double now = perf_counter()
self.timings.append(now)
PyList_Append(self.names, name)
cdef array.array[double] calculate_elapsed(self, array.array[double] timings):
cdef double time1, time2, elapsed
cdef Py_ssize_t len_timings = len(timings)
cdef int i
cdef array.array[double] results = array.copy(timings)
for i in range(1,len_timings):
time1 = timings.data.as_doubles[i-1]
time2 = timings.data.as_doubles[i]
elapsed = ((time2 - time1) * 1000)
results[i-1] = elapsed
return results
cpdef list stats(self):
cdef array.array[double] elapsed = self.calculate_elapsed(self.timings)
prefix = self.prefix.decode()
return [f'{prefix}.{name}:{stat:0.6f}|ms' for name,stat in zip(self.names,elapsed)]
# -*- coding: utf-8 -*-
from time import perf_counter
class TimingsPy:
def __init__(self, prefix: bytes):
self.prefix = prefix
self.timings = []
self.names = ['created']
def record(self, name: str):
self.timings.append(perf_counter())
self.names.append(name)
def calculate_elapsed(self, timings):
results = []
for i in range(1, len(timings)):
time1 = timings[i - 1]
time2 = timings[i]
elapsed = ((time2 - time1) * 1000)
results.append(elapsed)
return results
def stats(self):
elapsed = self.calculate_elapsed(self.timings)
prefix = self.prefix.decode()
return [f'{prefix}.{name}:{stat:0.6f}|ms' for name, stat in zip(self.names, elapsed)]
# -*- coding: utf-8 -*-
import asyncio
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
class EchoServerProtocol:
data_len = 0
def connection_made(self, transport):
self.transport = transport
def datagram_received(self, data, addr):
self.data_len += len(data)
message = data.decode()
[print(line) for line in message.split('\n')]
def connection_lost(self, *args, **kwargs):
print(f'data received: {self.data_len}')
loop = asyncio.get_event_loop()
print("Starting UDP server")
# One protocol instance will be created to serve all client requests
listen = loop.create_datagram_endpoint(
EchoServerProtocol, local_addr=('127.0.0.1', 8125))
transport, protocol = loop.run_until_complete(listen)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
transport.close()
loop.close()
......@@ -5,23 +5,24 @@ services:
image: "steemit/jussi:latest"
ports:
- "8080:8080"
- "7777:7777"
environment:
JUSSI_REDIS_HOST: redis1
JUSSI_REDIS_PORT: 6379
JUSSI_REDIS_READ_REPLICA_HOSTS: redis2:6379
LOG_LEVEL: DEBUG
links:
- redis1
- redis2
JUSSI_REDIS_URL: redis://redis1:6379
JUSSI_REDIS_READ_REPLICA_URLS: redis://redis2:6379
JUSSI_STATSD_URL: statsd://statsd:8125
env_file: .env
volumes:
- /Users/muyloco/Dev/steemit/jussi/DEV_config.json:/app/DEV_config.json
redis1:
restart: "no"
image: "redis:3.2"
ports:
- "6380:6379"
redis2:
restart: "no"
image: "redis:3.2"
ports:
- "6379:6379"
statsd:
restart: "no"
image: "statsd"
ports:
- "8125:8125/udp"
# -*- coding: utf-8 -*-
import asyncio
from collections import deque
from random import random
from typing import List
from typing import Tuple
import structlog
# pylint: disable=no-name-in-module
from cytoolz import sliding_window
# pylint: enable=no-name-in-module
logger = structlog.get_logger('stats')
__all__ = ['AsyncStatsClient']
class DatagramClientProtocol:
def __init__(self):
self.transport = None
def connection_made(self, transport):
self.transport = transport
# pylint: disable=no-self-use
def error_received(self, exc):
logger.debug('error received:', e=exc)
# pylint: disable=unused-argument
def connection_lost(self, exc):
logger.info("socket closed, stopping the event loop")
loop = asyncio.get_event_loop()
loop.stop()