.gitignore

@@ -130,3 +130,6 @@ tests/failed_blocks/
 
 # pyrest tests
 *.out.json
+
+# version.py
+version.py

.gitlab-ci.yaml

@@ -36,7 +36,7 @@ before_script:
 hivemind_build:
   stage: build
   script:
-    - pip3 install --user --upgrade pip setuptools
+    - pip3 install --user --upgrade pip setuptools gitpython
     - git fetch --tags
     - git tag -f ci_implicit_tag
     - echo $PYTHONUSERBASE
@@ -71,7 +71,7 @@ hivemind_sync:
     PYTHONUSERBASE: ./local-site
 
   script:
-    - pip3 install --user --upgrade pip setuptools
+    - pip3 install --user --upgrade pip setuptools gitpython
     - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK $HIVEMIND_HTTP_PORT
 
   artifacts:

Pipfile

@@ -16,6 +16,7 @@ aiohttp = "*"
 aiopg = "*"
 "psycopg2-binary" = "*"
 "diff-match-patch" = "*"
+"gitpython" = "*"
 
 [dev-packages]
 

hive/conf.py

@@ -61,8 +61,13 @@ class Conf():
         # configure logger and print config
         root = logging.getLogger()
         root.setLevel(conf.log_level())
-        root.info("loaded configuration:\n%s",
-                  _sanitized_conf(parser))
+
+        from sys import argv
+        root.info("Used command line args: %s", " ".join(argv[1:]))
+
+        # uncomment for full list of program args
+        #args_list = ["--" + k + " " + str(v) for k,v in vars(args).items()]
+        #root.info("Full command line args: %s", " ".join(args_list))
 
         if conf.mode() == 'server':
             #DbStats.SLOW_QUERY_MS = 750

hive/indexer/blocks.py

@@ -207,7 +207,7 @@ class Blocks:
                     if not is_initial_sync:
                         Accounts.dirty(op['author']) # lite - rep
                         Accounts.dirty(op['voter']) # lite - stats
-                    Votes.vote_op(op)
+                    Votes.vote_op(op, cls._head_block_date)
 
                 # misc ops
                 elif op_type == 'transfer_operation':
@@ -238,7 +238,7 @@ class Blocks:
 
         if vote_ops is not None:
           for k, v in vote_ops.items():
-              Votes.effective_comment_vote_op(k, v, cls._head_block_date)
+            Votes.effective_comment_vote_op(k, v)
 
         if Posts.comment_payout_ops:
             cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, comment_payout_stats)

hive/indexer/posts.py

@@ -161,7 +161,6 @@ class Posts:
                   payout                = COALESCE( CAST( data_source.payout as DECIMAL ),              ihp.payout ),
                   pending_payout        = COALESCE( CAST( data_source.pending_payout as DECIMAL ),      ihp.pending_payout ),
                   payout_at             = COALESCE( CAST( data_source.payout_at as TIMESTAMP ),         ihp.payout_at ),
-                  updated_at            = data_source.updated_at,
                   last_payout           = COALESCE( CAST( data_source.last_payout as TIMESTAMP ),       ihp.last_payout ),
                   cashout_time          = COALESCE( CAST( data_source.cashout_time as TIMESTAMP ),      ihp.cashout_time ),
                   is_paidout            = COALESCE( CAST( data_source.is_paidout as BOOLEAN ),          ihp.is_paidout )
@@ -177,7 +176,6 @@ class Posts:
                       t.payout,
                       t.pending_payout,
                       t.payout_at,
-                      t.updated_at,
                       t.last_payout,
                       t.cashout_time,
                       t.is_paidout
@@ -196,7 +194,6 @@ class Posts:
                       payout,
                       pending_payout,
                       payout_at,
-                      updated_at,
                       last_payout,
                       cashout_time,
                       is_paidout)
@@ -310,7 +307,7 @@ class Posts:
                 payout_at = date  #Here should be `cashout_time`
                 last_payout = date
 
-            cls._comment_payout_ops.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, '{}'::timestamp, {}, {}, {})".format(
+            cls._comment_payout_ops.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {})".format(
               author,
               permlink,
               "NULL" if ( total_payout_value is None ) else ( "'{}'".format( legacy_amount(total_payout_value) ) ),
@@ -323,7 +320,6 @@ class Posts:
               "NULL" if ( pending_payout is None ) else pending_payout,
 
               "NULL" if ( payout_at is None ) else ( "'{}'::timestamp".format( payout_at ) ),
-              date,#updated_at
               "NULL" if ( last_payout is None ) else ( "'{}'::timestamp".format( last_payout ) ),
               "NULL" if ( cashout_time is None ) else ( "'{}'::timestamp".format( cashout_time ) ),
 

hive/indexer/votes.py

@@ -49,11 +49,12 @@ class Votes:
     inside_flush = False
 
     @classmethod
-    def vote_op(cls, vop):
+    def vote_op(cls, vote_operation, date):
         """ Process vote_operation """
-        voter = vop['voter']
-        author = vop['author']
-        permlink = vop['permlink']
+        voter     = vote_operation['voter']
+        author    = vote_operation['author']
+        permlink  = vote_operation['permlink']
+        weight    = vote_operation['weight']
 
         if(cls.inside_flush):
             log.info("Adding new vote-info into '_votes_data' dict")
@@ -61,16 +62,21 @@ class Votes:
 
         key = voter + "/" + author + "/" + permlink
 
-        cls._votes_data[key] = dict(voter=voter,
-                                    author=author,
-                                    permlink=permlink,
-                                    vote_percent=0,
-                                    weight=0,
-                                    rshares=0,
-                                    last_update="1969-12-31T23:59:59")
+        if key in cls._votes_data:
+            cls._votes_data[key]["vote_percent"]=weight
+            cls._votes_data[key]["last_update"]=date
+        else:
+            cls._votes_data[key] = dict(voter=voter,
+                                        author=author,
+                                        permlink=permlink,
+                                        vote_percent=weight,
+                                        weight=0,
+                                        rshares=0,
+                                        last_update=date,
+                                        is_effective=False)
 
     @classmethod
-    def effective_comment_vote_op(cls, key, vop, date):
+    def effective_comment_vote_op(cls, key, vop):
         """ Process effective_comment_vote_operation """
 
         if(cls.inside_flush):
@@ -79,10 +85,9 @@ class Votes:
 
         assert key in cls._votes_data
 
-        cls._votes_data[key]["vote_percent"]  = vop["vote_percent"]
-        cls._votes_data[key]["weight"]        = vop["weight"]
-        cls._votes_data[key]["rshares"]       = vop["rshares"]
-        cls._votes_data[key]["last_update"]   = date
+        cls._votes_data[key]["weight"]       = vop["weight"]
+        cls._votes_data[key]["rshares"]      = vop["rshares"]
+        cls._votes_data[key]["is_effective"] = True
 
     @classmethod
     def flush(cls):
@@ -90,52 +95,65 @@ class Votes:
         cls.inside_flush = True
         if cls._votes_data:
             sql = """
-                    INSERT INTO hive_votes
-                    (post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update) 
-                    select data_source.post_id, data_source.voter_id, data_source.author_id, data_source.permlink_id, data_source.weight, data_source.rshares, data_source.vote_percent, data_source.last_update
-                    from 
-                    (
-                    SELECT hp.id as post_id, ha_v.id as voter_id, ha_a.id as author_id, hpd_p.id as permlink_id, t.weight, t.rshares, t.vote_percent, t.last_update
-                    from
-                    (
-                    VALUES
-                    --   voter, author, permlink, weight, rshares, vote_percent, last_update
-                      {}
-                    ) AS T(voter, author, permlink, weight, rshares, vote_percent, last_update)
-                    INNER JOIN hive_accounts ha_v ON ha_v.name = t.voter
-                    INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
-                    INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
-                    INNER JOIN hive_posts hp ON hp.author_id = ha_a.id AND hp.permlink_id = hpd_p.id
-                    ) as data_source(post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update)
-                    ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
-                      UPDATE
-                        SET
-                          weight = EXCLUDED.weight,
-                          rshares = EXCLUDED.rshares,
-                          vote_percent = EXCLUDED.vote_percent,
-                          last_update = EXCLUDED.last_update,
-                          num_changes = hive_votes.num_changes + 1
-                      WHERE hive_votes.voter_id = EXCLUDED.voter_id and hive_votes.author_id = EXCLUDED.author_id and hive_votes.permlink_id = EXCLUDED.permlink_id;
-                      """
-
-            values = []
+                INSERT INTO hive_votes
+                (post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update) 
+                SELECT hp.id as post_id, ha_v.id as voter_id, ha_a.id as author_id, hpd_p.id as permlink_id, t.weight, t.rshares, t.vote_percent, t.last_update
+                FROM
+                (
+                VALUES
+                  -- voter, author, permlink, weight, rshares, vote_percent, last_update
+                  {}
+                ) AS T(voter, author, permlink, weight, rshares, vote_percent, last_update)
+                INNER JOIN hive_accounts ha_v ON ha_v.name = t.voter
+                INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
+                INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
+                INNER JOIN hive_posts hp ON hp.author_id = ha_a.id AND hp.permlink_id = hpd_p.id
+                ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
+                UPDATE
+                  SET
+                    weight = {}.weight,
+                    rshares = {}.rshares,
+                    vote_percent = EXCLUDED.vote_percent,
+                    last_update = EXCLUDED.last_update,
+                    num_changes = hive_votes.num_changes + 1
+                  WHERE hive_votes.voter_id = EXCLUDED.voter_id and hive_votes.author_id = EXCLUDED.author_id and hive_votes.permlink_id = EXCLUDED.permlink_id;
+                """
+            # WHERE clause above seems superfluous (and works all the same without it, at least up to 5mln)
+
+            values_skip = []
+            values_override = []
             values_limit = 1000
 
             for _, vd in cls._votes_data.items():
+                values = None
+                on_conflict_data_source = None
+
+                if vd['is_effective']:
+                    values = values_override
+                    on_conflict_data_source = 'EXCLUDED'
+                else:
+                    values = values_skip
+                    on_conflict_data_source = 'hive_votes'
+
                 values.append("('{}', '{}', '{}', {}, {}, {}, '{}'::timestamp)".format(
                     vd['voter'], vd['author'], vd['permlink'], vd['weight'], vd['rshares'], vd['vote_percent'], vd['last_update']))
 
                 if len(values) >= values_limit:
                     values_str = ','.join(values)
-                    actual_query = sql.format(values_str)
+                    actual_query = sql.format(values_str,on_conflict_data_source,on_conflict_data_source)
                     DB.query(actual_query)
                     values.clear()
 
-            if len(values) > 0:
-                values_str = ','.join(values)
-                actual_query = sql.format(values_str)
+            if len(values_skip) > 0:
+                values_str = ','.join(values_skip)
+                actual_query = sql.format(values_str,'hive_votes','hive_votes')
+                DB.query(actual_query)
+                values_skip.clear()
+            if len(values_override) > 0:
+                values_str = ','.join(values_override)
+                actual_query = sql.format(values_str,'EXCLUDED','EXCLUDED')
                 DB.query(actual_query)
-                values.clear()
+                values_override.clear()
 
             cls._votes_data.clear()
         cls.inside_flush = False

hive/server/hive_api/public.py

@@ -10,6 +10,7 @@ from hive.server.condenser_api.cursor import get_followers, get_following
 from hive.server.bridge_api.cursor import (
     pids_by_blog, pids_by_comments, pids_by_feed_with_reblog)
 
+from hive.db.schema import DB_VERSION as SCHEMA_DB_VERSION
 
 log = logging.getLogger(__name__)
 
@@ -108,3 +109,20 @@ async def list_account_feed(context, account, limit=10, observer=None, last_post
         if rby: post['reblogged_by'] = list(rby)
 
     return posts
+
+async def get_info(context):
+    db = context['db']
+
+    sql = "SELECT num FROM hive_blocks ORDER BY num DESC LIMIT 1"
+    database_head_block = await db.query_one(sql)
+
+    from hive.version import VERSION, GIT_REVISION
+
+    ret = {
+        "hivemind_version" : VERSION,
+        "hivemind_git_rev" : GIT_REVISION,
+        "database_schema_version" : SCHEMA_DB_VERSION,
+        "database_head_block" : database_head_block
+    }
+
+    return ret

hive/server/serve.py

@@ -27,6 +27,7 @@ from hive.server.bridge_api.support import get_post_header as bridge_api_get_pos
 from hive.server.hive_api import community as hive_api_community
 from hive.server.hive_api import notify as hive_api_notify
 from hive.server.hive_api import stats as hive_api_stats
+from hive.server.hive_api.public import get_info as hive_api_get_info
 
 from hive.server.follow_api import methods as follow_api
 from hive.server.tags_api import methods as tags_api
@@ -62,6 +63,8 @@ def build_methods():
         db_head_state,
     )})
 
+    methods.add(**{'hive.get_info' : hive_api_get_info})
+
     methods.add(**{'condenser_api.' + method.__name__: method for method in (
         condenser_api.get_followers,
         condenser_api.get_following,
@@ -220,7 +223,24 @@ def run_server(conf):
         app['db'].close()
         await app['db'].wait_closed()
 
+    async def show_info(app):
+        sql = "SELECT num FROM hive_blocks ORDER BY num DESC LIMIT 1"
+        database_head_block = await app['db'].query_one(sql)
+
+        import pkg_resources
+        hivemind_version, hivemind_git_rev = pkg_resources.get_distribution("hivemind").version.split("+")
+
+        from hive.version import VERSION, GIT_REVISION
+        log.info("hivemind_version : %s", VERSION)
+        log.info("hivemind_git_rev : %s", GIT_REVISION)
+
+        from hive.db.schema import DB_VERSION as SCHEMA_DB_VERSION
+        log.info("database_schema_version : %s", SCHEMA_DB_VERSION)
+        
+        log.info("database_head_block : %s", database_head_block)
+
     app.on_startup.append(init_db)
+    app.on_startup.append(show_info)
     app.on_cleanup.append(close_db)
 
     async def head_age(request):

hive/version.py

+# generated by setup.py
+# contents will be overwritten
+VERSION = '0.0.1'
+GIT_REVISION = '5c0b832'

scripts/ci_start_server.sh

@@ -38,6 +38,7 @@ sleep 5
 ls -l dist/*
 rm -rf ./local-site
 mkdir -p `python3 -m site --user-site`
+pip3 install --user gitpython
 python3 setup.py install --user --force
 ln -sf ./local-site/bin/hive $HIVE_NAME
 ./$HIVE_NAME -h

scripts/ci_sync.sh

@@ -43,6 +43,7 @@ sleep 5
 ls -l dist/*
 rm -rf ./local-site
 mkdir -p `python3 -m site --user-site`
+pip3 install --user gitpython
 python3 setup.py install --user --force
 ln -sf ./local-site/bin/hive $HIVE_NAME
 ./$HIVE_NAME -h

setup.py

 # coding=utf-8
 import sys
+import os
 
 from setuptools import find_packages
 from setuptools import setup
 
 assert sys.version_info[0] == 3 and sys.version_info[1] >= 6, "hive requires Python 3.6 or newer"
 
+def get_git_version():
+    from git import Repo
+    repo = Repo(os.path.abspath("."))
+    return repo.git.rev_parse('--short', 'HEAD')
+
+VERSION = '0.0.1'
+GIT_REVISION = get_git_version()
+
 tests_require = [
     'pytest',
     'pytest-cov',
@@ -17,10 +26,16 @@ tests_require = [
     'yapf',
 ]
 
+with open("hive/version.py", 'w') as version_file:
+    version_file.write("# generated by setup.py\n")
+    version_file.write("# contents will be overwritten\n")
+    version_file.write("VERSION = '{}'\n".format(VERSION))
+    version_file.write("GIT_REVISION = '{}'\n".format(GIT_REVISION))
+
 # yapf: disable
 setup(
     name='hivemind',
-    version_format='0.0.1+{gitsha}',
+    version_format=VERSION + "+" + GIT_REVISION,
     description='Developer-friendly microservice powering social networks on the Steem blockchain.',
     long_description=open('README.md').read(),
     packages=find_packages(exclude=['scripts']),

tests_api @ 855f525d

-Subproject commit 113cb05cc2dc35fe6e6e149fc3ecb8ae286d4cc4
+Subproject commit 855f525d497119092d24bec91c1d29648fc04c2f