diff --git a/alembic.ini b/alembic.ini
new file mode 100644
index 0000000000000000000000000000000000000000..1e43ca64c68a9e4948ee5611c934d34298a3416e
--- /dev/null
+++ b/alembic.ini
@@ -0,0 +1,85 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions. When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = postgresql://hive:aMom3ieBieh9Ish0huiY2eGhreiThu7j@localhost/hive
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts. See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks=black
+# black.type=console_scripts
+# black.entrypoint=black
+# black.options=-l 79
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/alembic/README b/alembic/README
new file mode 100644
index 0000000000000000000000000000000000000000..98e4f9c44effe479ed38c66ba922e7bcc672916f
--- /dev/null
+++ b/alembic/README
@@ -0,0 +1 @@
+Generic single-database configuration.
\ No newline at end of file
diff --git a/alembic/env.py b/alembic/env.py
new file mode 100644
index 0000000000000000000000000000000000000000..70518a2eef734a8fffcd787cfa397309469f8e76
--- /dev/null
+++ b/alembic/env.py
@@ -0,0 +1,77 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+ """Run migrations in 'offline' mode.
+
+ This configures the context with just a URL
+ and not an Engine, though an Engine is acceptable
+ here as well. By skipping the Engine creation
+ we don't even need a DBAPI to be available.
+
+ Calls to context.execute() here emit the given string to the
+ script output.
+
+ """
+ url = config.get_main_option("sqlalchemy.url")
+ context.configure(
+ url=url,
+ target_metadata=target_metadata,
+ literal_binds=True,
+ dialect_opts={"paramstyle": "named"},
+ )
+
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def run_migrations_online():
+ """Run migrations in 'online' mode.
+
+ In this scenario we need to create an Engine
+ and associate a connection with the context.
+
+ """
+ connectable = engine_from_config(
+ config.get_section(config.config_ini_section),
+ prefix="sqlalchemy.",
+ poolclass=pool.NullPool,
+ )
+
+ with connectable.connect() as connection:
+ context.configure(
+ connection=connection, target_metadata=target_metadata
+ )
+
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/alembic/script.py.mako b/alembic/script.py.mako
new file mode 100644
index 0000000000000000000000000000000000000000..2c0156303a8df3ffdc9de87765bf801bf6bea4a5
--- /dev/null
+++ b/alembic/script.py.mako
@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+ ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+ ${downgrades if downgrades else "pass"}
diff --git a/alembic/versions/bf9df12d4f01_jsalyers_update_some_indexes.py b/alembic/versions/bf9df12d4f01_jsalyers_update_some_indexes.py
new file mode 100644
index 0000000000000000000000000000000000000000..d9623345bbbbc9a33fb20f0ecacb4dda532f3607
--- /dev/null
+++ b/alembic/versions/bf9df12d4f01_jsalyers_update_some_indexes.py
@@ -0,0 +1,26 @@
+"""jsalyers-update-some-indexes
+
+Revision ID: bf9df12d4f01
+Revises:
+Create Date: 2020-05-04 15:54:28.863707
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'bf9df12d4f01'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.create_index('hive_posts_cache_author_permlink_idx', 'hive_posts_cache', ['author', 'permlink'])
+ op.create_index('hive_posts_cache_post_id_author_permlink_idx', 'hive_posts_cache', ['post_id', 'author', 'permlink'])
+
+
+def downgrade():
+ op.drop_index('hive_posts_cache_author_permlink_idx')
+ op.drop_index('hive_posts_cache_post_id_author_permlink_idx')
diff --git a/build.sh b/build.sh
new file mode 100755
index 0000000000000000000000000000000000000000..1b2bfebd9bef7206cd286d21699f7eec72b7c42b
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,9 @@
+echo Build started on `date`
+export IMAGE_TAG=`git ls-remote --heads origin | grep $(git rev-parse HEAD) | cut -d / -f 3`
+if [ "$IMAGE_TAG" = "master" ] ; then export IMAGE_TAG=latest ; fi
+export REPO_PATH=`git rev-parse --show-toplevel`
+export REPO_NAME=`basename $REPO_PATH`
+export IMAGE_REPO_NAME="hive/$REPO_NAME"
+export SOURCE_COMMIT=`git rev-parse HEAD`
+echo Building branch $IMAGE_TAG from $IMAGE_REPO_NAME
+docker build . -t $IMAGE_REPO_NAME:$IMAGE_TAG --build-arg SOURCE_COMMIT="${SOURCE_COMMIT}" --build-arg DOCKER_TAG="${IMAGE_TAG}"
diff --git a/hive/server/condenser_api/cursor.py b/hive/server/condenser_api/cursor.py
index 2e3b42ba437f74b92b74d46987daad292bb51928..1b3fa5caa7c3aab810a0b715c30372fe19481e0f 100644
--- a/hive/server/condenser_api/cursor.py
+++ b/hive/server/condenser_api/cursor.py
@@ -166,28 +166,30 @@ async def pids_by_query(db, sort, start_author, start_permlink, limit, tag):
where.append("post_id IN (%s)" % sql)
start_id = None
- if start_permlink:
- start_id = await _get_post_id(db, start_author, start_permlink)
- if not start_id:
- return []
-
- sql = "%s <= (SELECT %s FROM %s WHERE post_id = :start_id)"
+ if start_permlink and start_author:
+ sql = "%s <= (SELECT %s FROM %s WHERE post_id = (SELECT post_id FROM hive_posts_cache WHERE author = :start_author AND permlink= :start_permlink))"
where.append(sql % (field, field, table))
- sql = ("SELECT post_id FROM %s WHERE %s ORDER BY %s DESC LIMIT :limit"
- % (table, ' AND '.join(where), field))
+ columns = ['hive_posts_cache.post_id', 'hive_posts_cache.author', 'hive_posts_cache.permlink',
+ 'hive_posts_cache.title', 'hive_posts_cache.body', 'hive_posts_cache.category',
+ 'hive_posts_cache.depth', 'hive_posts_cache.promoted',
+ 'hive_posts_cache.payout', 'hive_posts_cache.payout_at', 'hive_posts_cache.is_paidout',
+ 'hive_posts_cache.children', 'hive_posts_cache.votes', 'hive_posts_cache.created_at',
+ 'hive_posts_cache.updated_at', 'hive_posts_cache.rshares', 'hive_posts_cache.raw_json',
+ 'hive_posts_cache.json']
+ sql = ("SELECT %s FROM %s WHERE %s ORDER BY %s DESC LIMIT :limit"
+ % (', '.join(columns), table, ' AND '.join(where), field))
- return await db.query_col(sql, tag=tag, start_id=start_id, limit=limit)
+ #return await db.query_col(sql, tag=tag, start_id=start_id, limit=limit)
+ return [sql, tag, start_id, limit]
async def pids_by_blog(db, account: str, start_author: str = '',
start_permlink: str = '', limit: int = 20):
"""Get a list of post_ids for an author's blog."""
- account_id = await _get_account_id(db, account)
-
seek = ''
start_id = None
- if start_permlink:
+ if start_permlink and start_author:
start_id = await _get_post_id(db, start_author, start_permlink)
if not start_id:
return []
diff --git a/hive/server/condenser_api/methods.py b/hive/server/condenser_api/methods.py
index fe0a5bb9cf730659f0430aa8c888eca5e98795f2..fd76750eedb690028821863169506b8d21b7b548 100644
--- a/hive/server/condenser_api/methods.py
+++ b/hive/server/condenser_api/methods.py
@@ -4,6 +4,7 @@ from functools import wraps
import hive.server.condenser_api.cursor as cursor
from hive.server.condenser_api.objects import load_posts, load_posts_reblogs, resultset_to_posts
+from hive.server.condenser_api.objects import _mute_votes, _condenser_post_object
from hive.server.common.helpers import (
ApiError,
return_error_info,
@@ -13,6 +14,7 @@ from hive.server.common.helpers import (
valid_offset,
valid_limit,
valid_follow_type)
+from hive.server.common.mutes import Mutes
# pylint: disable=too-many-arguments,line-too-long,too-many-lines
@@ -101,12 +103,34 @@ async def get_content(context, author: str, permlink: str):
db = context['db']
valid_account(author)
valid_permlink(permlink)
- post_id = await cursor.get_post_id(db, author, permlink)
- if not post_id:
- return {'id': 0, 'author': '', 'permlink': ''}
- posts = await load_posts(db, [post_id])
- assert posts, 'post was not found in cache'
- return posts[0]
+
+ sql = """
+ ---get_content
+ SELECT hive_posts_cache.post_id, hive_posts_cache.author, hive_posts_cache.permlink,
+ hive_posts_cache.title, hive_posts_cache.body, hive_posts_cache.category, hive_posts_cache.depth,
+ hive_posts_cache.promoted, hive_posts_cache.payout, hive_posts_cache.payout_at,
+ hive_posts_cache.is_paidout, hive_posts_cache.children, hive_posts_cache.votes,
+ hive_posts_cache.created_at, hive_posts_cache.updated_at, hive_posts_cache.rshares,
+ hive_posts_cache.raw_json, hive_posts_cache.json, hive_accounts.reputation AS author_rep
+ FROM hive_posts_cache JOIN hive_accounts ON (hive_posts_cache.author = hive_accounts.name)
+ JOIN hive_posts ON (hive_posts_cache.post_id = hive_posts.id)
+ WHERE hive_posts_cache.author = :author AND hive_posts_cache.permlink = :permlink AND NOT hive_posts.is_deleted
+ """
+
+ result = await db.query_all(sql, author=author, permlink=permlink)
+ result = dict(result[0])
+ post = _condenser_post_object(result, 0)
+ post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
+
+ assert post, 'post was not found in cache'
+ return post
+
+ #post_id = await cursor.get_post_id(db, author, permlink)
+ #if not post_id:
+ # return {'id': 0, 'author': '', 'permlink': ''}
+ #posts = await load_posts(db, [post_id])
+ #assert posts, 'post was not found in cache'
+ #return posts[0]
@return_error_info
@@ -116,7 +140,9 @@ async def get_content_replies(context, author: str, permlink: str):
valid_account(author)
valid_permlink(permlink)
- sql = """SELECT post_id, author, permlink, title, body, category, depth,
+ sql = """
+ --get_content_replies
+ SELECT post_id, author, permlink, title, body, category, depth,
promoted, payout, payout_at, is_paidout, children, votes,
created_at, updated_at, rshares, raw_json, json
FROM hive_posts_cache WHERE post_id IN (
@@ -156,6 +182,37 @@ def nested_query_compat(function):
return function(*args, **kwargs)
return wrapper
+@return_error_info
+@nested_query_compat
+async def get_discussions_by(discussion_type, context, start_author: str = '',
+ start_permlink: str = '', limit: int = 20,
+ tag: str = None, truncate_body: int = 0,
+ filter_tags: list = None):
+ """ Common implementation for get_discussions_by calls """
+ assert not filter_tags, 'filter tags not supported'
+ assert discussion_type in ['trending', 'hot', 'created', 'promoted',
+ 'payout', 'payout_comments'], 'invalid discussion type'
+
+ db = context['db']
+ query_information = await cursor.pids_by_query(
+ db,
+ discussion_type,
+ valid_account(start_author, allow_empty=True),
+ valid_permlink(start_permlink, allow_empty=True),
+ valid_limit(limit, 100),
+ valid_tag(tag, allow_empty=True))
+
+ assert len(query_information) == 4, 'generated query is malformed, aborting'
+ sql = query_information[0]
+ sql = "---get_discussions_by_" + discussion_type + "\r\n" + sql
+ sql_tag = query_information[1]
+ sql_start_id = query_information[2]
+ sql_limit = query_information[3]
+
+ result = await db.query_all(sql, tag=sql_tag, start_id=sql_start_id, limit=sql_limit, start_author=start_author, start_permlink=start_permlink)
+ posts = await resultset_to_posts(db=db, resultset=result, truncate_body=0)
+ return posts
+
@return_error_info
@nested_query_compat
@@ -164,14 +221,9 @@ async def get_discussions_by_trending(context, start_author: str = '', start_per
truncate_body: int = 0, filter_tags: list = None):
"""Query posts, sorted by trending score."""
assert not filter_tags, 'filter_tags not supported'
- ids = await cursor.pids_by_query(
- context['db'],
- 'trending',
- valid_account(start_author, allow_empty=True),
- valid_permlink(start_permlink, allow_empty=True),
- valid_limit(limit, 100),
- valid_tag(tag, allow_empty=True))
- return await load_posts(context['db'], ids, truncate_body=truncate_body)
+ results = await get_discussions_by('trending', context, start_author, start_permlink,
+ limit, tag, truncate_body, filter_tags)
+ return results
@return_error_info
@@ -181,14 +233,9 @@ async def get_discussions_by_hot(context, start_author: str = '', start_permlink
truncate_body: int = 0, filter_tags: list = None):
"""Query posts, sorted by hot score."""
assert not filter_tags, 'filter_tags not supported'
- ids = await cursor.pids_by_query(
- context['db'],
- 'hot',
- valid_account(start_author, allow_empty=True),
- valid_permlink(start_permlink, allow_empty=True),
- valid_limit(limit, 100),
- valid_tag(tag, allow_empty=True))
- return await load_posts(context['db'], ids, truncate_body=truncate_body)
+ results = await get_discussions_by('hot', context, start_author, start_permlink,
+ limit, tag, truncate_body, filter_tags)
+ return results
@return_error_info
@@ -198,14 +245,9 @@ async def get_discussions_by_promoted(context, start_author: str = '', start_per
truncate_body: int = 0, filter_tags: list = None):
"""Query posts, sorted by promoted amount."""
assert not filter_tags, 'filter_tags not supported'
- ids = await cursor.pids_by_query(
- context['db'],
- 'promoted',
- valid_account(start_author, allow_empty=True),
- valid_permlink(start_permlink, allow_empty=True),
- valid_limit(limit, 100),
- valid_tag(tag, allow_empty=True))
- return await load_posts(context['db'], ids, truncate_body=truncate_body)
+ results = await get_discussions_by('promoted', context, start_author, start_permlink,
+ limit, tag, truncate_body, filter_tags)
+ return results
@return_error_info
@@ -215,14 +257,9 @@ async def get_discussions_by_created(context, start_author: str = '', start_perm
truncate_body: int = 0, filter_tags: list = None):
"""Query posts, sorted by creation date."""
assert not filter_tags, 'filter_tags not supported'
- ids = await cursor.pids_by_query(
- context['db'],
- 'created',
- valid_account(start_author, allow_empty=True),
- valid_permlink(start_permlink, allow_empty=True),
- valid_limit(limit, 100),
- valid_tag(tag, allow_empty=True))
- return await load_posts(context['db'], ids, truncate_body=truncate_body)
+ results = await get_discussions_by('created', context, start_author, start_permlink,
+ limit, tag, truncate_body, filter_tags)
+ return results
@return_error_info
@@ -233,14 +270,46 @@ async def get_discussions_by_blog(context, tag: str = None, start_author: str =
"""Retrieve account's blog posts, including reblogs."""
assert tag, '`tag` cannot be blank'
assert not filter_tags, 'filter_tags not supported'
- ids = await cursor.pids_by_blog(
- context['db'],
- valid_account(tag),
- valid_account(start_author, allow_empty=True),
- valid_permlink(start_permlink, allow_empty=True),
- valid_limit(limit, 100))
- return await load_posts(context['db'], ids, truncate_body=truncate_body)
+ valid_account(tag)
+ valid_account(start_author, allow_empty=True)
+ valid_permlink(start_permlink, allow_empty=True)
+ valid_limit(limit, 100)
+
+ sql = """
+ ---get_discussions_by_blog
+ SELECT hive_posts_cache.post_id, hive_posts_cache.author, hive_posts_cache.permlink,
+ hive_posts_cache.title, hive_posts_cache.body, hive_posts_cache.category, hive_posts_cache.depth,
+ hive_posts_cache.promoted, hive_posts_cache.payout, hive_posts_cache.payout_at,
+ hive_posts_cache.is_paidout, hive_posts_cache.children, hive_posts_cache.votes,
+ hive_posts_cache.created_at, hive_posts_cache.updated_at, hive_posts_cache.rshares,
+ hive_posts_cache.raw_json, hive_posts_cache.json, hive_accounts.reputation AS author_rep
+ FROM hive_posts_cache JOIN hive_accounts ON (hive_posts_cache.author = hive_accounts.name)
+ JOIN hive_posts ON (hive_posts_cache.post_id = hive_posts.id)
+ WHERE NOT hive_posts.is_deleted AND hive_posts_cache.post_id IN
+ (SELECT post_id FROM hive_feed_cache JOIN hive_accounts ON (hive_feed_cache.account_id = hive_accounts.id) WHERE hive_accounts.name = :author)
+ """
+ if start_author and start_permlink != '':
+ sql += """
+ AND hive_posts_cache.created_at <= (SELECT created_at from hive_posts_cache where author = :start_author AND permlink = :start_permlink)
+ """
+
+ sql += """
+ ORDER BY hive_posts_cache.created_at DESC
+ LIMIT :limit
+ """
+ db = context['db']
+ result = await db.query_all(sql, author=tag, start_author=start_author, start_permlink=start_permlink, limit=limit)
+ posts_by_id = []
+
+ for row in result:
+ row = dict(row)
+ post = _condenser_post_object(row, truncate_body=truncate_body)
+ post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
+ #posts_by_id[row['post_id']] = post
+ posts_by_id.append(post);
+
+ return posts_by_id
@return_error_info
@nested_query_compat
@@ -258,6 +327,18 @@ async def get_discussions_by_feed(context, tag: str = None, start_author: str =
valid_limit(limit, 100))
return await load_posts_reblogs(context['db'], res, truncate_body=truncate_body)
+ #valid_account(start_author, allow_empty=True)
+ #valid_account(tag)
+ #valid_permlink(start_permlink, allow_empty=True)
+ #valid_limit(limit, 100)
+
+ #sql = """
+ #
+ #"""
+
+ #if start_permlink and start_author:
+
+
@return_error_info
@nested_query_compat
@@ -267,12 +348,45 @@ async def get_discussions_by_comments(context, start_author: str = None, start_p
"""Get comments by made by author."""
assert start_author, '`start_author` cannot be blank'
assert not filter_tags, 'filter_tags not supported'
- ids = await cursor.pids_by_account_comments(
- context['db'],
- valid_account(start_author),
- valid_permlink(start_permlink, allow_empty=True),
- valid_limit(limit, 100))
- return await load_posts(context['db'], ids, truncate_body=truncate_body)
+ valid_account(start_author)
+ valid_permlink(start_permlink, allow_empty=True)
+ valid_limit(limit, 100)
+
+ sql = """
+ ---get_discussions_by_comments
+ SELECT hive_posts_cache.post_id, hive_posts_cache.author, hive_posts_cache.permlink,
+ hive_posts_cache.title, hive_posts_cache.body, hive_posts_cache.category, hive_posts_cache.depth,
+ hive_posts_cache.promoted, hive_posts_cache.payout, hive_posts_cache.payout_at,
+ hive_posts_cache.is_paidout, hive_posts_cache.children, hive_posts_cache.votes,
+ hive_posts_cache.created_at, hive_posts_cache.updated_at, hive_posts_cache.rshares,
+ hive_posts_cache.raw_json, hive_posts_cache.json, hive_accounts.reputation AS author_rep
+ FROM hive_posts_cache JOIN hive_accounts ON (hive_posts_cache.author = hive_accounts.name)
+ JOIN hive_posts ON (hive_posts_cache.post_id = hive_posts.id)
+ WHERE hive_posts_cache.author = :start_author AND hive_posts_cache.depth > 0
+ AND NOT hive_posts.is_deleted
+ """
+
+ if start_permlink:
+ sql += """
+ AND hive_posts_cache.post_id <= (SELECT hive_posts_cache.post_id FROM
+ hive_posts_cache WHERE permlink = :start_permlink AND author=:start_author)
+ """
+
+ sql += """
+ ORDER BY hive_posts_cache.post_id DESC, depth LIMIT :limit
+ """
+
+ posts = []
+ db = context['db']
+ result = await db.query_all(sql, start_author=start_author, start_permlink=start_permlink, limit=limit)
+
+ for row in result:
+ row = dict(row)
+ post = _condenser_post_object(row, truncate_body=truncate_body)
+ post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
+ posts.append(post)
+
+ return posts
@return_error_info
@@ -281,6 +395,22 @@ async def get_replies_by_last_update(context, start_author: str = None, start_pe
limit: int = 20, truncate_body: int = 0):
"""Get all replies made to any of author's posts."""
assert start_author, '`start_author` cannot be blank'
+ #valid_account(start_author)
+ #valid_permlink(start_permlink, allow_empty=True)
+ #valid_limit(limit, 100)
+
+ #sql = """
+ # SELECT hive_posts_cache.post_id, hive_posts_cache.author, hive_posts_cache.permlink,
+ # hive_posts_cache.title, hive_posts_cache.body, hive_posts_cache.category, hive_posts_cache.depth,
+ # hive_posts_cache.promoted, hive_posts_cache.payout, hive_posts_cache.payout_at,
+ # hive_posts_cache.is_paidout, hive_posts_cache.children, hive_posts_cache.votes,
+ # hive_posts_cache.created_at, hive_posts_cache.updated_at, hive_posts_cache.rshares,
+ # hive_posts_cache.raw_json, hive_posts_cache.json, hive_accounts.reputation AS author_rep
+ # FROM hive_posts_cache JOIN hive_accounts ON (hive_posts_cache.author = hive_accounts.name)
+ # JOIN hive_posts ON (hive_posts_cache.post_id = hive_posts.id)
+ #
+ #"""
+
ids = await cursor.pids_by_replies_to_account(
context['db'],
valid_account(start_author),
@@ -315,14 +445,9 @@ async def get_post_discussions_by_payout(context, start_author: str = '', start_
limit: int = 20, tag: str = None,
truncate_body: int = 0):
"""Query top-level posts, sorted by payout."""
- ids = await cursor.pids_by_query(
- context['db'],
- 'payout',
- valid_account(start_author, allow_empty=True),
- valid_permlink(start_permlink, allow_empty=True),
- valid_limit(limit, 100),
- valid_tag(tag, allow_empty=True))
- return await load_posts(context['db'], ids, truncate_body=truncate_body)
+ results = await get_discussions_by('payout', context, start_author, start_permlink,
+ limit, tag, truncate_body)
+ return results
@return_error_info
@@ -332,14 +457,9 @@ async def get_comment_discussions_by_payout(context, start_author: str = '', sta
truncate_body: int = 0):
"""Query comments, sorted by payout."""
# pylint: disable=invalid-name
- ids = await cursor.pids_by_query(
- context['db'],
- 'payout_comments',
- valid_account(start_author, allow_empty=True),
- valid_permlink(start_permlink, allow_empty=True),
- valid_limit(limit, 100),
- valid_tag(tag, allow_empty=True))
- return await load_posts(context['db'], ids, truncate_body=truncate_body)
+ results = await get_discussions_by('payout_comments', context, start_author, start_permlink,
+ limit, tag, truncate_body)
+ return results
@return_error_info
diff --git a/hive/server/db.py b/hive/server/db.py
index e7608b4b6b12fb96ec02eeeeff4b67f81ebfd4f3..d6d81e73f22c8de8b51f5cea3823ad3bab5d0a35 100644
--- a/hive/server/db.py
+++ b/hive/server/db.py
@@ -10,6 +10,7 @@ from aiopg.sa import create_engine
from hive.utils.stats import Stats
logging.getLogger('sqlalchemy.engine').setLevel(logging.WARNING)
+logging.FileHandler('hive_database_timer.log')
log = logging.getLogger(__name__)
def sqltimer(function):
diff --git a/hive/utils/account.py b/hive/utils/account.py
index caf38307c378d39473ae3d997d762462679bc768..6502f5aa61ba1b427b630b621da17ad50182a420 100644
--- a/hive/utils/account.py
+++ b/hive/utils/account.py
@@ -41,6 +41,8 @@ def safe_profile_metadata(account):
website = None
if website and not _valid_url_proto(website):
website = 'http://' + website
+ if website and len(website) > 100:
+ website = None
if profile_image and not _valid_url_proto(profile_image):
profile_image = None
diff --git a/hive/utils/stats.py b/hive/utils/stats.py
index fc7d3a5543196e5d08658efeb25726a3b5a14a8a..e5705007496df317c21c36c79860ce854707e41c 100644
--- a/hive/utils/stats.py
+++ b/hive/utils/stats.py
@@ -6,7 +6,10 @@ import logging
from time import perf_counter as perf
from hive.utils.system import colorize, peak_usage_mb
+file_handler = logging.FileHandler('database_timer.log')
+file_handler.setLevel(logging.INFO)
log = logging.getLogger(__name__)
+log.addHandler(file_handler)
def _normalize_sql(sql, maxlen=180):
"""Collapse whitespace and middle-truncate if needed."""
@@ -116,6 +119,7 @@ class DbStats(StatsAbstract):
def check_timing(self, call, ms, batch_size):
"""Warn if any query is slower than defined threshold."""
+ log.warning("[SQL][%dms] %s", ms, call)
if ms > self.SLOW_QUERY_MS:
out = "[SQL][%dms] %s" % (ms, call[:250])
log.warning(colorize(out))
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b7dd598b2b2833b437cf02861d2dc32202a6026d
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,79 @@
+aiocache==0.11.1
+aiohttp==3.6.2
+aiopg==1.0.0
+apply-defaults==0.1.4
+asn1crypto==0.24.0
+async-timeout==3.0.1
+attrs==19.3.0
+Automat==0.6.0
+certifi==2020.4.5.1
+chardet==3.0.4
+click==6.7
+colorama==0.3.7
+command-not-found==0.3
+ConfigArgParse==1.2
+configobj==5.0.6
+constantly==15.1.0
+cryptography==2.1.4
+dateparser==0.7.4
+distro-info===0.18ubuntu0.18.04.1
+funcsigs==1.0.2
+funcy==1.14
+-e git+git@gitlab.syncad.com:blocktrades/hivemind.git@ca1cccc0dbc4d78fdfe009afc251f7ee760b6847#egg=hivemind
+httplib2==0.9.2
+humanize==2.3.0
+hyperlink==17.3.1
+idna==2.9
+idna-ssl==1.1.0
+incremental==16.10.1
+iotop==0.6
+Jinja2==2.10
+jsonrpcserver==4.0.1
+jsonschema==2.6.0
+keyring==10.6.0
+keyrings.alt==3.0
+language-selector==0.1
+Mako==1.1.2
+Markdown==2.4.1
+MarkupSafe==1.1.1
+maya==0.6.1
+multidict==4.7.5
+netifaces==0.10.4
+PAM==0.4.2
+pdoc==0.3.2
+pendulum==2.1.0
+psycopg2-binary==2.8.5
+pyasn1==0.4.2
+pyasn1-modules==0.2.1
+pycrypto==2.6.1
+pygobject==3.26.1
+pyOpenSSL==17.5.0
+pyserial==3.4
+python-apt==1.6.5+ubuntu0.2
+python-dateutil==2.8.1
+python-debian==0.1.32
+pytz==2019.3
+pytzdata==2019.3
+pyxdg==0.25
+PyYAML==3.12
+regex==2020.4.4
+requests==2.18.4
+requests-unixsocket==0.1.5
+SecretStorage==2.3.1
+service-identity==16.0.0
+six==1.14.0
+snaptime==0.2.4
+SQLAlchemy==1.3.15
+ssh-import-id==5.7
+systemd-python==234
+toolz==0.10.0
+Twisted==17.9.0
+typing-extensions==3.7.4.2
+tzlocal==2.0.0
+ufw==0.36
+ujson==2.0.3
+unattended-upgrades==0.1
+urllib3==1.25.8
+virtualenv==15.1.0
+yarl==1.4.2
+zope.interface==4.3.2