[Pydantic schemas] Poor performance of schemas import
FYI: @bwrona @kmochocki @jziebinski
From the profiling log below, we can observer that schemas mainly spend time in
import time: 3668053 | 3773275 | schemas.__private.operations
and since all the child modules does not count up to this large value, it means the problem probably is inside schemas/__private/operations/__init__.py file.
which is confirmed by:
>> time python schemas/__private/operations/__init__.py
7,93s user 0,09s system 94% cpu 8,493 totalpart related to schemas, problem was initially observed in clive, output of time python -X importtime clive/main.py --help:
import time: self [us] | cumulative | imported package
import time: 146 | 146 | schemas
import time: 135 | 281 | schemas.__private
import time: 638 | 638 | pydantic.generics
import time: 111 | 111 | schemas.__private.hive_constants
import time: 583 | 583 | schemas.__private.preconfigured_base_model
import time: 7834 | 9444 | schemas.__private.hive_fields_basic_schemas
import time: 8598 | 8598 | schemas.__private.hive_fields_custom_schemas
import time: 1433 | 1433 | schemas.__private.operations.account_create_operation
import time: 1084 | 1084 | schemas.__private.operations.account_update2_operation
import time: 1077 | 1077 | schemas.__private.operations.account_update_operation
import time: 644 | 644 | schemas.__private.operations.account_witness_proxy_operation
import time: 686 | 686 | schemas.__private.operations.account_witness_vote_operation
import time: 570 | 570 | schemas.__private.operations.cancel_transfer_from_savings_operation
import time: 589 | 589 | schemas.__private.operations.change_recovery_account_operation
import time: 1463 | 1463 | schemas.__private.operations.claim_account_operation
import time: 1328 | 1328 | schemas.__private.operations.claim_reward_balance_operation
import time: 853 | 853 | schemas.__private.operations.collateralized_convert_operation
import time: 1115 | 1115 | schemas.__private.operations.comment_operation
import time: 1251 | 1251 | schemas.__private.operations.comment_options_operation
import time: 866 | 866 | schemas.__private.operations.convert_operation
import time: 1014 | 1014 | schemas.__private.operations.create_claimed_account_operation
import time: 1162 | 1162 | schemas.__private.operations.create_proposal_operation
import time: 1164 | 1164 | schemas.__private.operations.custom_binary_operation
import time: 967 | 967 | schemas.__private.operations.custom_json_operation
import time: 659 | 659 | schemas.__private.operations.custom_operation
import time: 497 | 497 | schemas.__private.operations.decline_voting_rights_operation
import time: 879 | 879 | schemas.__private.operations.delegate_vesting_shares_operation
import time: 503 | 503 | schemas.__private.operations.delete_comment_operation
import time: 1499 | 1499 | schemas.__private.operations.escrow_approve_operation
import time: 1017 | 1017 | schemas.__private.operations.escrow_dispute_operation
import time: 1822 | 1822 | schemas.__private.operations.escrow_release_operation
import time: 2353 | 2353 | schemas.__private.operations.escrow_transfer_operation
import time: 2229 | 2229 | schemas.__private.operations.feed_publish_operation
import time: 639 | 639 | schemas.__private.operations.limit_order_cancel_operation
import time: 1363 | 1363 | schemas.__private.operations.limit_order_create2_operation
import time: 1810 | 1810 | schemas.__private.operations.limit_order_create_operation
import time: 2424 | 2424 | schemas.__private.operations.pow_operation
import time: 580 | 580 | schemas.__private.operations.recover_account_operation
import time: 1479 | 1479 | schemas.__private.operations.recurrent_transfer_operation
import time: 567 | 567 | schemas.__private.operations.remove_proposal_operation
import time: 650 | 650 | schemas.__private.operations.request_account_recovery_operation
import time: 901 | 901 | schemas.__private.operations.reset_account_operation
import time: 812 | 812 | schemas.__private.operations.set_reset_account_operation
import time: 767 | 767 | schemas.__private.operations.set_withdraw_vesting_route_operation
import time: 1214 | 1214 | schemas.__private.operations.transfer_from_savings_operation
import time: 1382 | 1382 | schemas.__private.operations.transfer_operation
import time: 1438 | 1438 | schemas.__private.operations.transfer_to_savings_operation
import time: 1054 | 1054 | schemas.__private.operations.transfer_to_vesting_operation
import time: 1080 | 1080 | schemas.__private.operations.update_proposal_operation
import time: 633 | 633 | schemas.__private.operations.update_proposal_votes_operation
import time: 1233 | 1233 | schemas.__private.operations.virtual.account_created_operation
import time: 1632 | 1632 | schemas.__private.operations.virtual.author_reward_operation
import time: 744 | 744 | schemas.__private.operations.virtual.changed_recovery_account_operation
import time: 1197 | 1197 | schemas.__private.operations.virtual.clear_null_account_balance_operation
import time: 860 | 860 | schemas.__private.operations.virtual.collateralized_convert_immediate_conversion_operation
import time: 1531 | 1531 | schemas.__private.operations.virtual.comment_benefactor_reward_operation
import time: 531 | 531 | schemas.__private.operations.virtual.comment_payout_update_operation
import time: 2356 | 2356 | schemas.__private.operations.virtual.comment_reward_operation
import time: 1108 | 1108 | schemas.__private.operations.virtual.consolidate_treasury_balance_operation
import time: 1075 | 1075 | schemas.__private.operations.virtual.curation_reward_operation
import time: 448 | 448 | schemas.__private.operations.virtual.declined_voting_rights_operation
import time: 551 | 551 | schemas.__private.operations.virtual.delayed_voting_operation
import time: 1879 | 1879 | schemas.__private.operations.virtual.dhf_conversion_operation
import time: 792 | 792 | schemas.__private.operations.virtual.dhf_funding_operation
import time: 3785 | 3785 | schemas.__private.operations.virtual.effective_comment_vote_operation
import time: 1604 | 1604 | schemas.__private.operations.virtual.escrow_approved_operation
import time: 2050 | 2050 | schemas.__private.operations.virtual.escrow_rejected_operation
import time: 447 | 447 | schemas.__private.operations.virtual.expired_account_notification_operation
import time: 1566 | 1566 | schemas.__private.operations.virtual.failed_recurrent_transfer_operation
import time: 1418 | 1418 | schemas.__private.operations.virtual.fill_collateralized_convert_request_operation
import time: 1111 | 1111 | schemas.__private.operations.virtual.fill_convert_request_operation
import time: 1897 | 1897 | schemas.__private.operations.virtual.fill_order_operation
import time: 1400 | 1400 | schemas.__private.operations.virtual.fill_recurrent_transfer_operation
import time: 1620 | 1620 | schemas.__private.operations.virtual.fill_transfer_from_savings_operation
import time: 1462 | 1462 | schemas.__private.operations.virtual.fill_vesting_withdraw_operation
import time: 2026 | 2026 | schemas.__private.operations.virtual.hardfork_hive_operation
import time: 1170 | 1170 | schemas.__private.operations.virtual.hardfork_hive_restore_operation
import time: 393 | 393 | schemas.__private.operations.virtual.hardfork_operation
import time: 523 | 523 | schemas.__private.operations.virtual.ineffective_delete_comment_operation
import time: 816 | 816 | schemas.__private.operations.virtual.interest_operation
import time: 1195 | 1195 | schemas.__private.operations.virtual.limit_order_cancelled_operation
import time: 741 | 741 | schemas.__private.operations.virtual.liquidity_reward_operation
import time: 1057 | 1057 | schemas.__private.operations.virtual.pow_reward_operation
import time: 437 | 437 | schemas.__private.operations.virtual.producer_missed_operation
import time: 983 | 983 | schemas.__private.operations.virtual.producer_reward_operation
import time: 1115 | 1115 | schemas.__private.operations.virtual.proposal_fee_operation
import time: 1276 | 1276 | schemas.__private.operations.virtual.proposal_pay_operation
import time: 607 | 607 | schemas.__private.operations.virtual.proxy_cleared_operation
import time: 739 | 739 | schemas.__private.operations.virtual.return_vesting_delegation_operation
import time: 517 | 517 | schemas.__private.operations.virtual.shutdown_witness_operation
import time: 350 | 350 | schemas.__private.operations.virtual.system_warning_operation
import time: 1199 | 1199 | schemas.__private.operations.virtual.transfer_to_vesting_completed_operation
import time: 995 | 995 | schemas.__private.operations.virtual.vesting_shares_split_operation
import time: 2325 | 52739 | schemas.__private.operations.virtual
import time: 855 | 855 | schemas.__private.operations.vote_operation
import time: 737 | 737 | schemas.__private.operations.withdraw_vesting_operation
import time: 545 | 545 | schemas.__private.operations.witness_block_approve_operation
import time: 1790 | 1790 | schemas.__private.operations.witness_set_properties_operation
import time: 1061 | 1061 | schemas.__private.operations.witness_update_operation
import time: 3668053 | 3773275 | schemas.__private.operations
import time: 31383 | 3804657 | schemas.__private.operation_objectsAnd full log of the suspected file only time python -X importtime schemas/__private/operations/__init__.py:
import time: self [us] | cumulative | imported package
import time: 107 | 107 | _io
import time: 27 | 27 | marshal
import time: 229 | 229 | posix
import time: 389 | 750 | _frozen_importlib_external
import time: 61 | 61 | time
import time: 141 | 202 | zipimport
import time: 82 | 82 | _codecs
import time: 299 | 380 | codecs
import time: 273 | 273 | encodings.aliases
import time: 425 | 1077 | encodings
import time: 156 | 156 | encodings.utf_8
import time: 66 | 66 | _signal
import time: 36 | 36 | _abc
import time: 161 | 197 | abc
import time: 193 | 389 | io
import time: 39 | 39 | _stat
import time: 126 | 164 | stat
import time: 657 | 657 | _collections_abc
import time: 102 | 102 | genericpath
import time: 143 | 245 | posixpath
import time: 448 | 1513 | os
import time: 124 | 124 | _sitebuiltins
import time: 297 | 297 | sitecustomize
import time: 925 | 2857 | site
import time: 182 | 182 | __future__
import time: 69 | 69 | itertools
import time: 102 | 102 | keyword
import time: 87 | 87 | _operator
import time: 662 | 749 | operator
import time: 225 | 225 | reprlib
import time: 80 | 80 | _collections
import time: 631 | 1854 | collections
import time: 163 | 163 | collections.abc
import time: 196 | 196 | types
import time: 40 | 40 | _functools
import time: 423 | 658 | functools
import time: 379 | 1037 | contextlib
import time: 541 | 541 | enum
import time: 47 | 47 | _sre
import time: 243 | 243 | sre_constants
import time: 229 | 472 | sre_parse
import time: 199 | 717 | sre_compile
import time: 52 | 52 | _locale
import time: 127 | 127 | copyreg
import time: 383 | 1818 | re
import time: 1342 | 6213 | typing
import time: 105 | 105 | backports_abc
import time: 152 | 152 | _weakrefset
import time: 319 | 471 | weakref
import time: 75 | 75 | org
import time: 14 | 88 | org.python
import time: 14 | 102 | org.python.core
import time: 158 | 730 | copy
import time: 850 | 850 | _ast
import time: 900 | 1749 | ast
import time: 101 | 101 | _opcode
import time: 191 | 292 | opcode
import time: 314 | 606 | dis
import time: 273 | 273 | warnings
import time: 153 | 425 | importlib
import time: 149 | 573 | importlib.machinery
import time: 123 | 123 | token
import time: 655 | 777 | tokenize
import time: 113 | 890 | linecache
import time: 1143 | 4958 | inspect
import time: 405 | 5362 | dataclasses
import time: 1464 | 1464 | typing_extensions
import time: 83 | 83 | backports_abc
import time: 80 | 80 | backports_abc
import time: 245 | 245 | numbers
import time: 455 | 700 | _decimal
import time: 108 | 807 | decimal
import time: 127 | 127 | fnmatch
import time: 76 | 76 | _winapi
import time: 73 | 73 | nt
import time: 66 | 66 | nt
import time: 64 | 64 | nt
import time: 63 | 63 | nt
import time: 174 | 514 | ntpath
import time: 42 | 42 | errno
import time: 98 | 98 | urllib
import time: 712 | 810 | urllib.parse
import time: 532 | 2023 | pathlib
import time: 90 | 90 | backports_abc
import time: 595 | 684 | pydantic.typing
import time: 1307 | 4900 | pydantic.errors
import time: 82 | 82 | backports_abc
import time: 80 | 80 | backports_abc
import time: 182 | 262 | pydantic.version
import time: 915 | 1259 | pydantic.utils
import time: 542 | 6783 | pydantic.class_validators
import time: 166 | 166 | _json
import time: 317 | 482 | json.scanner
import time: 326 | 808 | json.decoder
import time: 333 | 333 | json.encoder
import time: 157 | 1296 | json
import time: 320 | 1616 | pydantic.config
import time: 82 | 82 | backports_abc
import time: 184 | 184 | math
import time: 170 | 170 | _datetime
import time: 504 | 857 | datetime
import time: 853 | 853 | ipaddress
import time: 601 | 601 | signal
import time: 397 | 397 | threading
import time: 115 | 115 | fcntl
import time: 115 | 115 | msvcrt
import time: 115 | 115 | _posixsubprocess
import time: 115 | 115 | select
import time: 354 | 354 | selectors
import time: 405 | 2214 | subprocess
import time: 1638 | 3852 | platform
import time: 161 | 161 | _uuid
import time: 332 | 4344 | uuid
import time: 81 | 81 | backports_abc
import time: 104 | 104 | colorsys
import time: 507 | 691 | pydantic.color
import time: 80 | 80 | backports_abc
import time: 80 | 80 | backports_abc
import time: 1880 | 1880 | pydantic.datetime_parse
import time: 507 | 2465 | pydantic.validators
import time: 881 | 3425 | pydantic.networks
import time: 82 | 82 | backports_abc
import time: 1912 | 1994 | pydantic.types
import time: 227 | 12389 | pydantic.json
import time: 532 | 13002 | pydantic.error_wrappers
import time: 84 | 84 | backports_abc
import time: 672 | 756 | pydantic.fields
import time: 82 | 82 | backports_abc
import time: 137 | 137 | _struct
import time: 105 | 241 | struct
import time: 181 | 181 | _compat_pickle
import time: 172 | 172 | _pickle
import time: 73 | 73 | org
import time: 13 | 86 | org.python
import time: 15 | 100 | org.python.core
import time: 666 | 1357 | pickle
import time: 254 | 1611 | pydantic.parse
import time: 84 | 84 | backports_abc
import time: 592 | 676 | pydantic.schema
import time: 1393 | 3760 | pydantic.main
import time: 636 | 34209 | pydantic.dataclasses
import time: 111 | 111 | backports_abc
import time: 314 | 424 | pydantic.annotated_types
import time: 336 | 336 | pydantic.decorator
import time: 83 | 83 | backports_abc
import time: 692 | 774 | pydantic.env_settings
import time: 227 | 227 | pydantic.tools
import time: 314 | 36282 | pydantic
import time: 156 | 156 | schemas
import time: 156 | 312 | schemas.__private
import time: 572 | 572 | pydantic.generics
import time: 98 | 98 | schemas.__private.hive_constants
import time: 514 | 514 | schemas.__private.preconfigured_base_model
import time: 8016 | 9508 | schemas.__private.hive_fields_basic_schemas
import time: 1368 | 1368 | schemas.__private.operations.account_create_operation
import time: 1041 | 1041 | schemas.__private.operations.account_update2_operation
import time: 1031 | 1031 | schemas.__private.operations.account_update_operation
import time: 606 | 606 | schemas.__private.operations.account_witness_proxy_operation
import time: 670 | 670 | schemas.__private.operations.account_witness_vote_operation
import time: 573 | 573 | schemas.__private.operations.cancel_transfer_from_savings_operation
import time: 591 | 591 | schemas.__private.operations.change_recovery_account_operation
import time: 1081 | 1081 | schemas.__private.operations.claim_account_operation
import time: 1904 | 1904 | schemas.__private.operations.claim_reward_balance_operation
import time: 880 | 880 | schemas.__private.operations.collateralized_convert_operation
import time: 1203 | 1203 | schemas.__private.operations.comment_operation
import time: 1218 | 1218 | schemas.__private.operations.comment_options_operation
import time: 923 | 923 | schemas.__private.operations.convert_operation
import time: 1032 | 1032 | schemas.__private.operations.create_claimed_account_operation
import time: 1254 | 1254 | schemas.__private.operations.create_proposal_operation
import time: 1120 | 1120 | schemas.__private.operations.custom_binary_operation
import time: 1027 | 1027 | schemas.__private.operations.custom_json_operation
import time: 677 | 677 | schemas.__private.operations.custom_operation
import time: 540 | 540 | schemas.__private.operations.decline_voting_rights_operation
import time: 911 | 911 | schemas.__private.operations.delegate_vesting_shares_operation
import time: 520 | 520 | schemas.__private.operations.delete_comment_operation
import time: 1516 | 1516 | schemas.__private.operations.escrow_approve_operation
import time: 1855 | 1855 | schemas.__private.operations.escrow_dispute_operation
import time: 1800 | 1800 | schemas.__private.operations.escrow_release_operation
import time: 2478 | 2478 | schemas.__private.operations.escrow_transfer_operation
import time: 2157 | 2157 | schemas.__private.operations.feed_publish_operation
import time: 590 | 590 | schemas.__private.operations.limit_order_cancel_operation
import time: 1428 | 1428 | schemas.__private.operations.limit_order_create2_operation
import time: 1841 | 1841 | schemas.__private.operations.limit_order_create_operation
import time: 8751 | 8751 | schemas.__private.hive_fields_custom_schemas
import time: 2077 | 10827 | schemas.__private.operations.pow_operation
import time: 635 | 635 | schemas.__private.operations.recover_account_operation
import time: 1512 | 1512 | schemas.__private.operations.recurrent_transfer_operation
import time: 561 | 561 | schemas.__private.operations.remove_proposal_operation
import time: 662 | 662 | schemas.__private.operations.request_account_recovery_operation
import time: 655 | 655 | schemas.__private.operations.reset_account_operation
import time: 778 | 778 | schemas.__private.operations.set_reset_account_operation
import time: 831 | 831 | schemas.__private.operations.set_withdraw_vesting_route_operation
import time: 1403 | 1403 | schemas.__private.operations.transfer_from_savings_operation
import time: 1247 | 1247 | schemas.__private.operations.transfer_operation
import time: 1282 | 1282 | schemas.__private.operations.transfer_to_savings_operation
import time: 1092 | 1092 | schemas.__private.operations.transfer_to_vesting_operation
import time: 1079 | 1079 | schemas.__private.operations.update_proposal_operation
import time: 642 | 642 | schemas.__private.operations.update_proposal_votes_operation
import time: 1324 | 1324 | schemas.__private.operations.virtual.account_created_operation
import time: 1684 | 1684 | schemas.__private.operations.virtual.author_reward_operation
import time: 834 | 834 | schemas.__private.operations.virtual.changed_recovery_account_operation
import time: 1124 | 1124 | schemas.__private.operations.virtual.clear_null_account_balance_operation
import time: 916 | 916 | schemas.__private.operations.virtual.collateralized_convert_immediate_conversion_operation
import time: 1560 | 1560 | schemas.__private.operations.virtual.comment_benefactor_reward_operation
import time: 523 | 523 | schemas.__private.operations.virtual.comment_payout_update_operation
import time: 1753 | 1753 | schemas.__private.operations.virtual.comment_reward_operation
import time: 1095 | 1095 | schemas.__private.operations.virtual.consolidate_treasury_balance_operation
import time: 1055 | 1055 | schemas.__private.operations.virtual.curation_reward_operation
import time: 434 | 434 | schemas.__private.operations.virtual.declined_voting_rights_operation
import time: 564 | 564 | schemas.__private.operations.virtual.delayed_voting_operation
import time: 1043 | 1043 | schemas.__private.operations.virtual.dhf_conversion_operation
import time: 790 | 790 | schemas.__private.operations.virtual.dhf_funding_operation
import time: 4386 | 4386 | schemas.__private.operations.virtual.effective_comment_vote_operation
import time: 1620 | 1620 | schemas.__private.operations.virtual.escrow_approved_operation
import time: 2019 | 2019 | schemas.__private.operations.virtual.escrow_rejected_operation
import time: 447 | 447 | schemas.__private.operations.virtual.expired_account_notification_operation
import time: 1583 | 1583 | schemas.__private.operations.virtual.failed_recurrent_transfer_operation
import time: 1406 | 1406 | schemas.__private.operations.virtual.fill_collateralized_convert_request_operation
import time: 1185 | 1185 | schemas.__private.operations.virtual.fill_convert_request_operation
import time: 1972 | 1972 | schemas.__private.operations.virtual.fill_order_operation
import time: 1427 | 1427 | schemas.__private.operations.virtual.fill_recurrent_transfer_operation
import time: 1422 | 1422 | schemas.__private.operations.virtual.fill_transfer_from_savings_operation
import time: 1488 | 1488 | schemas.__private.operations.virtual.fill_vesting_withdraw_operation
import time: 1906 | 1906 | schemas.__private.operations.virtual.hardfork_hive_operation
import time: 1159 | 1159 | schemas.__private.operations.virtual.hardfork_hive_restore_operation
import time: 431 | 431 | schemas.__private.operations.virtual.hardfork_operation
import time: 517 | 517 | schemas.__private.operations.virtual.ineffective_delete_comment_operation
import time: 831 | 831 | schemas.__private.operations.virtual.interest_operation
import time: 1112 | 1112 | schemas.__private.operations.virtual.limit_order_cancelled_operation
import time: 793 | 793 | schemas.__private.operations.virtual.liquidity_reward_operation
import time: 1058 | 1058 | schemas.__private.operations.virtual.pow_reward_operation
import time: 440 | 440 | schemas.__private.operations.virtual.producer_missed_operation
import time: 1003 | 1003 | schemas.__private.operations.virtual.producer_reward_operation
import time: 1003 | 1003 | schemas.__private.operations.virtual.proposal_fee_operation
import time: 1342 | 1342 | schemas.__private.operations.virtual.proposal_pay_operation
import time: 589 | 589 | schemas.__private.operations.virtual.proxy_cleared_operation
import time: 1057 | 1057 | schemas.__private.operations.virtual.return_vesting_delegation_operation
import time: 506 | 506 | schemas.__private.operations.virtual.shutdown_witness_operation
import time: 432 | 432 | schemas.__private.operations.virtual.system_warning_operation
import time: 1272 | 1272 | schemas.__private.operations.virtual.transfer_to_vesting_completed_operation
import time: 1034 | 1034 | schemas.__private.operations.virtual.vesting_shares_split_operation
import time: 2334 | 52451 | schemas.__private.operations.virtual
import time: 829 | 829 | schemas.__private.operations.vote_operation
import time: 836 | 836 | schemas.__private.operations.withdraw_vesting_operation
import time: 547 | 547 | schemas.__private.operations.witness_block_approve_operation
import time: 1697 | 1697 | schemas.__private.operations.witness_set_properties_operation
import time: 1204 | 1204 | schemas.__private.operations.witness_update_operation
import time: 4073868 | 4188449 | schemas.__private.operations
import time: 31 | 4188479 | schemas.__private.operations.account_create_operation
python -X importtime 8,27s user 0,13s system 100% cpu 8,402 totalEdited by Mateusz Żebrak