conf.py 9.1 KB
Newer Older
roadscape's avatar
roadscape committed
1
2
"""Conf handles reading run-time config and app-level settings."""

roadscape's avatar
roadscape committed
3
import re
4
5
6
import logging
import configargparse

roadscape's avatar
roadscape committed
7
from hive.steem.client import SteemClient
roadscape's avatar
roadscape committed
8
from hive.db.adapter import Db
roadscape's avatar
roadscape committed
9
from hive.utils.normalize import strtobool, int_log_level
roadscape's avatar
roadscape committed
10
11
from hive.utils.stats import DbStats

12
13
log = logging.getLogger(__name__)

roadscape's avatar
roadscape committed
14
15
16
17
def _sanitized_conf(parser):
    """Formats parser config, redacting database url password."""
    out = parser.format_values()
    return re.sub(r'(?<=:)\w+(?=@)', '<redacted>', out)
roadscape's avatar
roadscape committed
18

19
class Conf():
roadscape's avatar
roadscape committed
20
    """ Manages sync/server configuration via args, ENVs, and hive.conf. """
21

22
23
24
25
26
27
28
29
    def __init__(self):
        self._args = None
        self._env = None
        self._db = None
        self._steem = None
        self.arguments = None

    def init_argparse(self, strict=True, **kwargs):
roadscape's avatar
roadscape committed
30
        """Read hive config (CLI arg > ENV var > config)"""
31

32
33
        #pylint: disable=line-too-long
        parser = configargparse.get_arg_parser(
roadscape's avatar
roadscape committed
34
35
            default_config_files=['./hive.conf'],
            **kwargs)
36
        add = parser.add
37

roadscape's avatar
roadscape committed
38
        # runmodes: sync, server, status
39
        add('mode', nargs='*', default=['sync'])
roadscape's avatar
roadscape committed
40

41
        # common
roadscape's avatar
roadscape committed
42
        add('--database-url', env_var='DATABASE_URL', required=False, help='database connection url', default='')
Jolly Pirate's avatar
Jolly Pirate committed
43
        add('--steemd-url', env_var='STEEMD_URL', required=False, help='steemd/jussi endpoint', default='{"default" : "https://api.hive.blog"}')
44
        add('--muted-accounts-url', env_var='MUTED_ACCOUNTS_URL', required=False, help='url to flat list of muted accounts', default='https://raw.githubusercontent.com/hivevectordefense/irredeemables/master/full.txt')
45
        add('--blacklist-api-url', env_var='BLACKLIST_API_URL', required=False, help='url to access blacklist api', default='https://blacklist.usehive.com')
46

roadscape's avatar
roadscape committed
47
48
        # server
        add('--http-server-port', type=int, env_var='HTTP_SERVER_PORT', default=8080)
49
        add('--prometheus-port', type=int, env_var='PROMETHEUS_PORT', required=False, help='if specified, runs prometheus deamon on specified port, which provide statistic and performance data')
roadscape's avatar
roadscape committed
50
51

        # sync
52
53
        add('--max-workers', type=int, env_var='MAX_WORKERS', help='max workers for batch requests', default=6)
        add('--max-batch', type=int, env_var='MAX_BATCH', help='max chunk size for batch requests', default=35)
54
        add('--trail-blocks', type=int, env_var='TRAIL_BLOCKS', help='number of blocks to trail head by', default=2)
roadscape's avatar
roadscape committed
55
        add('--sync-to-s3', type=strtobool, env_var='SYNC_TO_S3', help='alternative healthcheck for background sync service', default=False)
56

roadscape's avatar
roadscape committed
57
58
        # test/debug
        add('--log-level', env_var='LOG_LEVEL', default='INFO')
roadscape's avatar
roadscape committed
59
60
        add('--test-disable-sync', type=strtobool, env_var='TEST_DISABLE_SYNC', help='(debug) skip sync and sweep; jump to block streaming', default=False)
        add('--test-max-block', type=int, env_var='TEST_MAX_BLOCK', help='(debug) only sync to given block, for running sync test', default=None)
61
        add('--test-skip-ais-phase', env_var='TEST_SKIP_AIS_PHASE', help='(debug) Allows to skip After-Initial-Sync phase. Useful to go into live sync or exit if TEST_MAX_BLOCK is used', action='store_true')
roadscape's avatar
roadscape committed
62
        add('--test-profile', type=strtobool, env_var='TEST_PROFILE', help='(debug) profile execution', default=False)
63
64
        add('--log-request-times', env_var='LOG_REQUEST_TIMES', help='(debug) allows to generate log containing request processing times', action='store_true')
        add('--log-virtual-op-calls', env_var='LOG_VIRTUAL_OP_CALLS', help='(debug) log virtual op calls and responses', default=False)
65
        add('--mock-block-data-path', type=str, nargs='+', env_var='MOCK_BLOCK_DATA_PATH', help='(debug/testing) load additional data from block data file')
Dariusz Kędzierski's avatar
Dariusz Kędzierski committed
66
        add('--mock-vops-data-path', type=str, env_var='MOCK_VOPS_DATA_PATH', help='(debug/testing) load additional data from virtual operations data file')
Mariusz's avatar
Mariusz committed
67
        add('--community-start-block', type=int, env_var='COMMUNITY_START_BLOCK', default=37500000)
roadscape's avatar
roadscape committed
68

69
70
71
72
73
        # logging
        add('--log-timestamp', help='Output timestamp in log', action='store_true')
        add('--log-epoch', help='Output unix epoch in log', action='store_true')
        add('--log-mask-sensitive-data', help='Mask sensitive data, e.g. passwords', action='store_true')

Bartek Wrona's avatar
Bartek Wrona committed
74
75
        add('--pid-file', type=str, env_var='PID_FILE', help='Allows to dump current process pid into specified file', default=None)

76
77
        add('--auto-http-server-port', nargs='+', type=int, help='Hivemind will listen on first available port from this range')

roadscape's avatar
roadscape committed
78
79
80
        # needed for e.g. tests - other args may be present
        args = (parser.parse_args() if strict
                else parser.parse_known_args()[0])
81

82
83
        self._args = vars(args)
        self.arguments = parser._actions
84

roadscape's avatar
roadscape committed
85
        # configure logger and print config
roadscape's avatar
roadscape committed
86
        root = logging.getLogger()
87
        root.setLevel(self.log_level())
Dariusz Kędzierski's avatar
Dariusz Kędzierski committed
88

89
90
91
92
93
94
95
96
97
98
99
100
        try:
            if 'auto_http_server_port' in vars(args) and vars(args)['auto_http_server_port'] is not None:
                port_range = vars(args)['auto_http_server_port']
                port_range_len = len(port_range)
                if port_range_len == 0 or port_range_len > 2:
                    raise ValueError("auto-http-server-port expect maximum two values, minimum one")
                if port_range_len == 2 and port_range[0] > port_range[1]:
                    raise ValueError("port min value is greater than port max value")
        except Exception as ex:
            root.error("Value error: {}".format(ex))
            exit(1)

101
102
        # Print command line args, but on continuous integration server
        # hide db connection string.
Dariusz Kędzierski's avatar
Dariusz Kędzierski committed
103
        from sys import argv
104
        if self.get('log_mask_sensitive_data'):
105
106
107
108
109
110
111
112
113
114
115
116
117
            my_args = []
            upcoming_connection_string = False
            for elem in argv[1:]:
                if upcoming_connection_string:
                    upcoming_connection_string = False
                    my_args.append('MASKED')
                    continue
                if elem == '--database-url':
                    upcoming_connection_string = True
                my_args.append(elem)
            root.info("Used command line args: %s", " ".join(my_args))
        else:
            root.info("Used command line args: %s", " ".join(argv[1:]))
Dariusz Kędzierski's avatar
Dariusz Kędzierski committed
118

119
120
121
        # uncomment for full list of program args
        #args_list = ["--" + k + " " + str(v) for k,v in vars(args).items()]
        #root.info("Full command line args: %s", " ".join(args_list))
roadscape's avatar
roadscape committed
122

123
        if self.mode() == 'server':
roadscape's avatar
roadscape committed
124
125
            #DbStats.SLOW_QUERY_MS = 750
            DbStats.SLOW_QUERY_MS = 200 # TODO
126

127
128
    def __enter__(self):
        return self
roadscape's avatar
roadscape committed
129

130
131
    def __exit__(self, exc_type, value, traceback):
        self.disconnect()
roadscape's avatar
roadscape committed
132

roadscape's avatar
roadscape committed
133
    def args(self):
roadscape's avatar
roadscape committed
134
        """Get the raw Namespace object as generated by configargparse"""
roadscape's avatar
roadscape committed
135
        return self._args
136

roadscape's avatar
roadscape committed
137
    def steem(self):
roadscape's avatar
roadscape committed
138
        """Get a SteemClient instance, lazily initialized"""
roadscape's avatar
roadscape committed
139
        if not self._steem:
140
            from json import loads
roadscape's avatar
roadscape committed
141
            self._steem = SteemClient(
142
                url=loads(self.get('steemd_url')),
roadscape's avatar
roadscape committed
143
144
145
146
147
148
                max_batch=self.get('max_batch'),
                max_workers=self.get('max_workers'))
        return self._steem

    def db(self):
        """Get a configured instance of Db."""
149
        if self._db is None:
roadscape's avatar
roadscape committed
150
            url = self.get('database_url')
roadscape's avatar
roadscape committed
151
152
            assert url, ('--database-url (or DATABASE_URL env) not specified; '
                         'e.g. postgresql://user:pass@localhost:5432/hive')
153
154
            self._db = Db(url, "root db creation")
            log.info("The database created...")
roadscape's avatar
roadscape committed
155
        return self._db
roadscape's avatar
roadscape committed
156

roadscape's avatar
roadscape committed
157
    def get(self, param):
roadscape's avatar
roadscape committed
158
        """Reads a single property, e.g. `database_url`."""
roadscape's avatar
roadscape committed
159
160
        assert self._args, "run init_argparse()"
        return self._args[param]
161

162
163
164
165
166
167
168
169
170
    def mode(self):
        """Get the CLI runmode.

        - `server`: API server
        - `sync`: db sync process
        - `status`: status info dump
        """
        return '/'.join(self.get('mode'))

roadscape's avatar
roadscape committed
171
    def log_level(self):
roadscape's avatar
roadscape committed
172
        """Get `logger`s internal int level from config string."""
roadscape's avatar
roadscape committed
173
        return int_log_level(self.get('log_level'))
Bartek Wrona's avatar
Bartek Wrona committed
174
175
176
177

    def pid_file(self):
        """Get optional pid_file name to put current process pid in"""
        return self._args.get("pid_file", None)
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194

    def generate_completion(self):
        arguments = []
        for arg in self.arguments:
            arguments.extend(arg.option_strings)
        arguments = " ".join(arguments)
        with open('hive-completion.bash', 'w') as file:
            file.writelines([
                "#!/bin/bash\n",
                "# to run type: source hive-completion.bash\n\n",
                "# if you want to have completion everywhere, execute theese commands\n",
                "# ln $PWD/hive-completion.bash $HOME/.local/\n",
                '# echo "source $HOME/.local/hive-completion.bash" >> $HOME/.bashrc\n',
                "# source $HOME/.bashrc\n\n"
                f'complete -f -W "{arguments}" hive\n',
                "\n"
            ])
195
196
197
198
199
200
201

    def disconnect(self):
        if self._db is not None:
            self._db.close()
            self._db.close_engine()
            self._db = None
            log.info("The database is disconnected...")